Generate new key when nonce reaches max value

https://github.com/XTLS/Xray-core/pull/4952#issuecomment-3179685937

proxy/vless/encryption/client.go

@@ -5,7 +5,6 @@ import (
 	"crypto/cipher"
 	"crypto/mlkem"
 	"crypto/rand"
-	"crypto/sha256"
 	"io"
 	"net"
 	"sync"
@@ -14,7 +13,6 @@ import (
 	"github.com/xtls/xray-core/common/crypto"
 	"github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/common/protocol"
-	"golang.org/x/crypto/hkdf"
 )
 
 var ClientCipher byte
@@ -91,7 +89,7 @@ func (i *ClientInstance) Handshake(conn net.Conn) (net.Conn, error) {
 	clientHello[0] = ClientCipher
 	copy(clientHello[1:], pfsEKeyBytes)
 	copy(clientHello[1185:], encapsulatedNfsKey)
-	encodeHeader(clientHello[2273:], int(paddingLen))
+	EncodeHeader(clientHello[2273:], int(paddingLen))
 	rand.Read(clientHello[2278:])
 
 	if _, err := c.Conn.Write(clientHello); err != nil {
@@ -112,11 +110,9 @@ func (i *ClientInstance) Handshake(conn net.Conn) (net.Conn, error) {
 	}
 	c.baseKey = append(pfsKey, nfsKey...)
 
-	authKey := make([]byte, 32)
-	hkdf.New(sha256.New, c.baseKey, encapsulatedPfsKey, encapsulatedNfsKey).Read(authKey)
 	nonce := [12]byte{ClientCipher}
-	VLESS, _ := newAead(ClientCipher, authKey).Open(nil, nonce[:], c.ticket, pfsEKeyBytes)
+	VLESS, _ := NewAead(ClientCipher, c.baseKey, encapsulatedPfsKey, encapsulatedNfsKey).Open(nil, nonce[:], c.ticket, pfsEKeyBytes)
-	if !bytes.Equal(VLESS, []byte("VLESS")) { // TODO: more message
+	if !bytes.Equal(VLESS, []byte("VLESS")) { // TODO: more messages
 		return nil, errors.New("invalid server").AtError()
 	}
 
@@ -135,32 +131,32 @@ func (c *ClientConn) Write(b []byte) (int, error) {
 	if len(b) == 0 {
 		return 0, nil
 	}
+	var data []byte
 	for n := 0; n < len(b); {
 		b := b[n:]
 		if len(b) > 8192 {
 			b = b[:8192] // for avoiding another copy() in server's Read()
 		}
 		n += len(b)
-		var data []byte
 		if c.aead == nil {
 			c.random = make([]byte, 32)
 			rand.Read(c.random)
-			key := make([]byte, 32)
+			c.aead = NewAead(ClientCipher, c.baseKey, c.random, c.ticket)
-			hkdf.New(sha256.New, c.baseKey, c.random, c.ticket).Read(key)
-			c.aead = newAead(ClientCipher, key)
 			c.nonce = make([]byte, 12)
-
 			data = make([]byte, 21+32+5+len(b)+16)
 			copy(data, c.ticket)
 			copy(data[21:], c.random)
-			encodeHeader(data[53:], len(b)+16)
+			EncodeHeader(data[53:], len(b)+16)
 			c.aead.Seal(data[:58], c.nonce, b, data[53:58])
 		} else {
 			data = make([]byte, 5+len(b)+16)
-			encodeHeader(data, len(b)+16)
+			EncodeHeader(data, len(b)+16)
 			c.aead.Seal(data[:5], c.nonce, b, data[:5])
+			if bytes.Equal(c.nonce, MaxNonce) {
+				c.aead = NewAead(ClientCipher, c.baseKey, data[5:], data[:5])
+			}
 		}
-		increaseNonce(c.nonce)
+		IncreaseNonce(c.nonce)
 		if _, err := c.Conn.Write(data); err != nil {
 			return 0, err
 		}
@@ -179,7 +175,7 @@ func (c *ClientConn) Read(b []byte) (int, error) {
 				if _, err := io.ReadFull(c.Conn, peerHeader); err != nil {
 					return 0, err
 				}
-				peerPaddingLen, _ := decodeHeader(peerHeader)
+				peerPaddingLen, _ := DecodeHeader(peerHeader)
 				if peerPaddingLen == 0 {
 					break
 				}
@@ -200,9 +196,7 @@ func (c *ClientConn) Read(b []byte) (int, error) {
 		if c.random == nil {
 			return 0, errors.New("empty c.random")
 		}
-		peerKey := make([]byte, 32)
+		c.peerAead = NewAead(ClientCipher, c.baseKey, peerRandom, c.random)
-		hkdf.New(sha256.New, c.baseKey, peerRandom, c.random).Read(peerKey)
-		c.peerAead = newAead(ClientCipher, peerKey)
 		c.peerNonce = make([]byte, 12)
 	}
 	if len(c.peerCache) != 0 {
@@ -213,7 +207,7 @@ func (c *ClientConn) Read(b []byte) (int, error) {
 	if _, err := io.ReadFull(c.Conn, peerHeader); err != nil {
 		return 0, err
 	}
-	peerLength, err := decodeHeader(peerHeader) // 17~17000
+	peerLength, err := DecodeHeader(peerHeader) // 17~17000
 	if err != nil {
 		if c.instance != nil {
 			c.instance.Lock()
@@ -232,8 +226,15 @@ func (c *ClientConn) Read(b []byte) (int, error) {
 	if len(dst) <= len(b) {
 		dst = b[:len(dst)] // avoids another copy()
 	}
+	var peerAead cipher.AEAD
+	if bytes.Equal(c.peerNonce, MaxNonce) {
+		peerAead = NewAead(ClientCipher, c.baseKey, peerData, peerHeader)
+	}
 	_, err = c.peerAead.Open(dst[:0], c.peerNonce, peerData, peerHeader)
-	increaseNonce(c.peerNonce)
+	if peerAead != nil {
+		c.peerAead = peerAead
+	}
+	IncreaseNonce(c.peerNonce)
 	if err != nil {
 		return 0, err
 	}

proxy/vless/encryption/common.go

@@ -1,15 +1,20 @@
 package encryption
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/sha256"
 	"strconv"
 
 	"github.com/xtls/xray-core/common/errors"
 	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/hkdf"
 )
 
-func encodeHeader(b []byte, l int) {
+var MaxNonce = bytes.Repeat([]byte{255}, 12)
+
+func EncodeHeader(b []byte, l int) {
 	b[0] = 23
 	b[1] = 3
 	b[2] = 3
@@ -17,10 +22,10 @@ func encodeHeader(b []byte, l int) {
 	b[4] = byte(l)
 }
 
-func decodeHeader(b []byte) (int, error) {
+func DecodeHeader(b []byte) (int, error) {
 	if b[0] == 23 && b[1] == 3 && b[2] == 3 {
 		l := int(b[3])<<8 | int(b[4])
-		if l < 17 || l > 17000 { // TODO
+		if l < 17 || l > 17000 { // TODO: TLSv1.3 max length
 			return 0, errors.New("invalid length in record's header: " + strconv.Itoa(l))
 		}
 		return l, nil
@@ -28,24 +33,23 @@ func decodeHeader(b []byte) (int, error) {
 	return 0, errors.New("invalid record's header")
 }
 
-func newAead(c byte, k []byte) (aead cipher.AEAD) {
+func NewAead(c byte, secret, salt, info []byte) (aead cipher.AEAD) {
+	key := make([]byte, 32)
+	hkdf.New(sha256.New, secret, salt, info).Read(key)
 	if c&1 == 1 {
-		block, _ := aes.NewCipher(k)
+		block, _ := aes.NewCipher(key)
 		aead, _ = cipher.NewGCM(block)
 	} else {
-		aead, _ = chacha20poly1305.New(k)
+		aead, _ = chacha20poly1305.New(key)
 	}
 	return
 }
 
-func increaseNonce(nonce []byte) {
+func IncreaseNonce(nonce []byte) {
 	for i := range 12 {
 		nonce[11-i]++
 		if nonce[11-i] != 0 {
 			break
 		}
-		if i == 11 {
-			// TODO
-		}
 	}
 }

proxy/vless/encryption/server.go

@@ -5,7 +5,6 @@ import (
 	"crypto/cipher"
 	"crypto/mlkem"
 	"crypto/rand"
-	"crypto/sha256"
 	"io"
 	"net"
 	"sync"
@@ -13,7 +12,6 @@ import (
 
 	"github.com/xtls/xray-core/common/crypto"
 	"github.com/xtls/xray-core/common/errors"
-	"golang.org/x/crypto/hkdf"
 )
 
 type ServerSession struct {
@@ -103,7 +101,7 @@ func (i *ServerInstance) Handshake(conn net.Conn) (net.Conn, error) {
 	if _, err := io.ReadFull(c.Conn, peerHeader); err != nil {
 		return nil, err
 	}
-	if l, _ := decodeHeader(peerHeader); l != 0 {
+	if l, _ := DecodeHeader(peerHeader); l != 0 {
 		noise := make([]byte, crypto.RandBetween(100, 1000))
 		rand.Read(noise)
 		c.Conn.Write(noise) // make client do new handshake
@@ -131,17 +129,15 @@ func (i *ServerInstance) Handshake(conn net.Conn) (net.Conn, error) {
 	pfsKey, encapsulatedPfsKey := pfsEKey.Encapsulate()
 	c.baseKey = append(pfsKey, nfsKey...)
 
-	authKey := make([]byte, 32)
-	hkdf.New(sha256.New, c.baseKey, encapsulatedPfsKey, encapsulatedNfsKey).Read(authKey)
 	nonce := [12]byte{c.cipher}
-	c.ticket = newAead(c.cipher, authKey).Seal(nil, nonce[:], []byte("VLESS"), pfsEKeyBytes)
+	c.ticket = NewAead(c.cipher, c.baseKey, encapsulatedPfsKey, encapsulatedNfsKey).Seal(nil, nonce[:], []byte("VLESS"), pfsEKeyBytes)
 
 	paddingLen := crypto.RandBetween(100, 1000)
 
 	serverHello := make([]byte, 1088+21+5+paddingLen)
 	copy(serverHello, encapsulatedPfsKey)
 	copy(serverHello[1088:], c.ticket)
-	encodeHeader(serverHello[1109:], int(paddingLen))
+	EncodeHeader(serverHello[1109:], int(paddingLen))
 	rand.Read(serverHello[1114:])
 
 	if _, err := c.Conn.Write(serverHello); err != nil {
@@ -173,7 +169,7 @@ func (c *ServerConn) Read(b []byte) (int, error) {
 				if _, err := io.ReadFull(c.Conn, peerHeader); err != nil {
 					return 0, err
 				}
-				peerPaddingLen, _ := decodeHeader(peerHeader)
+				peerPaddingLen, _ := DecodeHeader(peerHeader)
 				if peerPaddingLen == 0 {
 					break
 				}
@@ -194,9 +190,7 @@ func (c *ServerConn) Read(b []byte) (int, error) {
 				return 0, err
 			}
 		}
-		peerKey := make([]byte, 32)
+		c.peerAead = NewAead(c.cipher, c.baseKey, c.peerRandom, c.ticket)
-		hkdf.New(sha256.New, c.baseKey, c.peerRandom, c.ticket).Read(peerKey)
-		c.peerAead = newAead(c.cipher, peerKey)
 		c.peerNonce = make([]byte, 12)
 	}
 	if len(c.peerCache) != 0 {
@@ -207,7 +201,7 @@ func (c *ServerConn) Read(b []byte) (int, error) {
 	if _, err := io.ReadFull(c.Conn, peerHeader); err != nil {
 		return 0, err
 	}
-	peerLength, err := decodeHeader(peerHeader) // 17~17000
+	peerLength, err := DecodeHeader(peerHeader) // 17~17000
 	if err != nil {
 		return 0, err
 	}
@@ -219,8 +213,15 @@ func (c *ServerConn) Read(b []byte) (int, error) {
 	if len(dst) <= len(b) {
 		dst = b[:len(dst)] // avoids another copy()
 	}
+	var peerAead cipher.AEAD
+	if bytes.Equal(c.peerNonce, MaxNonce) {
+		peerAead = NewAead(ClientCipher, c.baseKey, peerData, peerHeader)
+	}
 	_, err = c.peerAead.Open(dst[:0], c.peerNonce, peerData, peerHeader)
-	increaseNonce(c.peerNonce)
+	if peerAead != nil {
+		c.peerAead = peerAead
+	}
+	IncreaseNonce(c.peerNonce)
 	if err != nil {
 		return 0, errors.New("error")
 	}
@@ -235,31 +236,32 @@ func (c *ServerConn) Write(b []byte) (int, error) {
 	if len(b) == 0 {
 		return 0, nil
 	}
+	var data []byte
 	for n := 0; n < len(b); {
 		b := b[n:]
 		if len(b) > 8192 {
 			b = b[:8192] // for avoiding another copy() in client's Read()
 		}
 		n += len(b)
-		var data []byte
 		if c.aead == nil {
 			if c.peerRandom == nil {
 				return 0, errors.New("empty c.peerRandom")
 			}
 			data = make([]byte, 32+5+len(b)+16)
 			rand.Read(data[:32])
-			key := make([]byte, 32)
+			c.aead = NewAead(c.cipher, c.baseKey, data[:32], c.peerRandom)
-			hkdf.New(sha256.New, c.baseKey, data[:32], c.peerRandom).Read(key)
-			c.aead = newAead(c.cipher, key)
 			c.nonce = make([]byte, 12)
-			encodeHeader(data[32:], len(b)+16)
+			EncodeHeader(data[32:], len(b)+16)
 			c.aead.Seal(data[:37], c.nonce, b, data[32:37])
 		} else {
 			data = make([]byte, 5+len(b)+16)
-			encodeHeader(data, len(b)+16)
+			EncodeHeader(data, len(b)+16)
 			c.aead.Seal(data[:5], c.nonce, b, data[:5])
+			if bytes.Equal(c.nonce, MaxNonce) {
+				c.aead = NewAead(ClientCipher, c.baseKey, data[5:], data[:5])
+			}
 		}
-		increaseNonce(c.nonce)
+		IncreaseNonce(c.nonce)
 		if _, err := c.Conn.Write(data); err != nil {
 			return 0, err
 		}