WireGuard, MASQUE proxies: Use Xray-core's built-in lightweight TCP/IP stack instead of gVisor as well (#7156)

https://github.com/XTLS/Xray-core/pull/7130#issuecomment-6116258188 https://github.com/XTLS/Xray-core/pull/7130#issuecomment-6120742739 https://github.com/XTLS/Xray-core/pull/7130#issuecomment-6137264458 https://github.com/XTLS/Xray-core/pull/7156#issuecomment-6202933664 https://github.com/XTLS/Xray-core/pull/7156#issuecomment-6203239736
2026-10-17 13:04:05 +00:00 · 2026-10-15 23:38:14 +00:00
parent 10879a4a8b
commit 8703fdb101
24 changed files with 3053 additions and 1430 deletions
@@ -3,7 +3,12 @@ package tun
 import (
 	"context"
 	"encoding/binary"
+	"io"
+	"math"
+	"math/rand/v2"
+	gonet "net"
 	"net/netip"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -15,8 +20,16 @@ import (

 // Stack interface implement ip protocol stack, bridging raw network packets and data streams
 type Stack interface {
+	// Start makes the stack read the packets of its tun
 	Start() error
 	Close() error
+	// Input takes a packet of a tun that hands them over instead of being read. The packet is not kept,
+	// but it may be written to. The packets of one connection have to come one after another
+	Input(packet []byte)
+	// DialTCP opens a connection to address from the one of StackOptions.Addresses that is of its family
+	DialTCP(ctx context.Context, address netip.AddrPort) (net.Conn, error)
+	// DialUDP opens a port on StackOptions.Addresses. What anyone sends to the port is read from it
+	DialUDP() (net.PacketConn, error)
 }

 // StackOptions for the stack implementation
@@ -24,6 +37,12 @@ type StackOptions struct {
 	Tun         Tun
 	MTU         uint32
 	IdleTimeout time.Duration
+	// Addresses are what the stack opens connections from
+	Addresses []netip.Addr
+	// Remote is set if the tun is a tunnel to another system and not the tun of this one: what comes through it
+	// has come a long way, so its checksums are looked at, nothing but that way limits the windows, and the
+	// segments of TCP carry the times of both sides if the other side has them
+	Remote bool
 }

 // flow is what the packets of one connection have in common, src is the side of the system
@@ -33,7 +52,8 @@ type flow struct {

 // stack turns the packets that the system sends to the tun into connections and back.
 // It has only what the two need to talk through a tun, and a local link when the system passes on
-// what others send, see tcpConn
+// what others send, see tcpConn. The tun may also be a tunnel to a system that is far away, and the stack
+// the one that opens the connections
 type stack struct {
 	ctx context.Context
 	tun Tun
@@ -42,32 +62,58 @@ type stack struct {
 	segmenter segmenter
 	// window is the window of every connection
 	window int32
-	// handle is given every new connection
+	// verify is set if the checksums of what comes are looked at
+	verify bool
+	// stamps is set if connections carry the times of RFC 7323 when the other side does
+	stamps bool
+	// handle is given every new connection that the other side opens. Without it those are refused
 	handle func(conn net.Conn, destination net.Destination)
 	udp    *udpConnectionHandler
+	// addresses are what connections are opened from
+	addresses []netip.Addr

-	mu     sync.Mutex
-	conns  map[flow]*tcpConn
-	closed bool
-	// opening is how many of conns wait for the answer to their SYN-ACK
+	mu    sync.Mutex
+	conns map[flow]*tcpConn
+	// sockets are the ports that DialUDP has opened
+	sockets map[netip.AddrPort]*udpSocket
+	closed  bool
+	// opening is how many of conns wait for the answer to their SYN or SYN-ACK
 	opening atomic.Int32
+	// fragments are the packets that are larger than a link on their way and come in parts.
+	// identification numbers the ones that leave like that: a counter for each of a few groups of destinations,
+	// each from a start of its own, so that no two of them that go one way meet, and no destination learns how much
+	// goes to the others
+	fragments      reassembly
+	identification [64]atomic.Uint32
 }

-// NewStack builds new ip stack
-func NewStack(ctx context.Context, options StackOptions, handler *Handler) (Stack, error) {
+// NewStack builds new ip stack. Without a handler it takes no connections, only opens them
+func NewStack(ctx context.Context, options StackOptions, handler ConnectionHandler) (Stack, error) {
 	s := &stack{
-		ctx:    ctx,
-		tun:    options.Tun,
-		mtu:    int(options.MTU),
-		window: tcpWindow,
-		handle: handler.HandleConnection,
-		conns:  make(map[flow]*tcpConn),
+		ctx:       ctx,
+		tun:       options.Tun,
+		mtu:       int(options.MTU),
+		window:    tcpWindow,
+		addresses: options.Addresses,
+		verify:    options.Remote,
+		stamps:    options.Remote,
+		conns:     make(map[flow]*tcpConn),
+		sockets:   make(map[netip.AddrPort]*udpSocket),
 	}
 	if t, ok := options.Tun.(segmenter); ok && t.Segments() {
 		s.segmenter = t
 		s.window = tcpLargeWindow
 	}
-	s.udp = newUdpConnectionHandler(s.handle, s.writeUDP)
+	if options.Remote {
+		s.window = tcpLargeWindow
+	}
+	for i := range s.identification {
+		s.identification[i].Store(rand.Uint32())
+	}
+	if handler != nil {
+		s.handle = handler.HandleConnection
+		s.udp = newUdpConnectionHandler(s.handle, s.writeUDP)
+	}
 	return s, nil
 }

@@ -85,6 +131,8 @@ func (s *stack) Close() error {
 	for _, c := range s.conns {
 		conns = append(conns, c)
 	}
+	sockets := s.sockets
+	s.sockets = nil
 	s.mu.Unlock()
 	for _, c := range conns {
 		c.mu.Lock()
@@ -93,6 +141,9 @@ func (s *stack) Close() error {
 		}
 		c.mu.Unlock()
 	}
+	for _, u := range sockets {
+		u.queue.close()
+	}
 	return nil
 }

@@ -111,19 +162,59 @@ func (s *stack) read() {
 			}
 			return
 		}
-		p, ok := parsePacket(b)
-		if !ok {
-			continue
-		}
-		switch p.proto {
-		case protoTCP:
-			s.handleTCP(p)
-		case protoUDP:
-			s.handleUDP(p)
-		case protoICMPv4, protoICMPv6:
-			s.handleICMP(p)
+		s.Input(b)
+	}
+}
+
+// Input implements Stack. Several tunnels may hand over their packets at once
+func (s *stack) Input(b []byte) {
+	p, ok := parsePacket(b)
+	if !ok || p.src.Is4In6() || p.dst.Is4In6() || p.src.IsLoopback() || p.dst.IsLoopback() || p.dst.IsUnspecified() || p.src.IsMulticast() {
+		// nothing comes from many at once, and an address that stands for this machine itself is not one
+		// that the other side of a tun can ask for. Nor is an IPv6 address that stands for an IPv4 one, which
+		// would be dialed as that: ::ffff:0.0.0.0 is this machine as well
+		return
+	}
+	if s.verify && b[0]>>4 == 4 && checksum(sum(b[:int(b[0]&0xf)*4], 0)) != 0 {
+		return
+	}
+	// what was put together may be of two packets with one number, its checksum tells
+	verify := s.verify || p.fragment
+	if p.fragment {
+		p, ok = s.fragments.add(p)
+	}
+	if !ok || verify && !p.verified() {
+		return
+	}
+	switch p.proto {
+	case protoTCP:
+		s.handleTCP(p)
+	case protoUDP:
+		s.handleUDP(p)
+	case protoICMPv4, protoICMPv6:
+		s.handleICMP(p)
+	}
+}
+
+// local returns the address that packets to remote are sent from
+func (s *stack) local(remote netip.Addr) (netip.Addr, error) {
+	for _, a := range s.addresses {
+		if a.Is4() == remote.Is4() {
+			return a, nil
 		}
 	}
+	return netip.Addr{}, errors.New("no address to reach ", remote, " from")
+}
+
+// ephemeral returns a port for what is opened from here: the first that free takes, from a random one on
+func ephemeral(free func(port uint16) bool) (uint16, bool) {
+	start := rand.Uint32()
+	for i := range uint32(1 << 15) {
+		if port := uint16(1<<15 | (start+i)&(1<<15-1)); free(port) {
+			return port, true
+		}
+	}
+	return 0, false
 }

 func (s *stack) handleTCP(p packet) {
@@ -154,7 +245,7 @@ func (s *stack) handleTCP(p packet) {
 	if c != nil && c.input(seg) {
 		return
 	}
-	if seg.flags&(tcpSyn|tcpAck|tcpRst) != tcpSyn {
+	if seg.flags&(tcpSyn|tcpAck|tcpRst) != tcpSyn || s.handle == nil {
 		if seg.flags&tcpRst == 0 {
 			s.refuse(f, seg)
 		}
@@ -164,7 +255,8 @@ func (s *stack) handleTCP(p packet) {
 	if s.opening.Load() >= tcpOpening {
 		return
 	}
-	c = newTCPConn(s, f, seg)
+	c = newTCPConn(s, f)
+	c.synchronize(seg)
 	s.mu.Lock()
 	if s.closed {
 		s.mu.Unlock()
@@ -174,7 +266,7 @@ func (s *stack) handleTCP(p packet) {
 	s.mu.Unlock()
 	s.opening.Add(1)
 	c.mu.Lock()
-	c.sendSynAck()
+	c.sendSyn()
 	c.arm(tcpTick)
 	c.mu.Unlock()
 }
@@ -224,51 +316,189 @@ func (s *stack) handleUDP(p packet) {
 	if size < udpHeaderSize || size > len(u) {
 		return
 	}
+	var socket *udpSocket
+	if s.handle == nil {
+		// only for a port that was opened from here
+		s.mu.Lock()
+		socket = s.sockets[netip.AddrPortFrom(p.dst, binary.BigEndian.Uint16(u[2:]))]
+		s.mu.Unlock()
+		if socket == nil {
+			return
+		}
+	}
 	// the packet waits for its reader, longer than read keeps it
 	b := buf.NewWithSize(int32(size - udpHeaderSize))
 	b.Write(u[udpHeaderSize:size])
 	src := net.UDPDestination(net.IPAddress(p.src.AsSlice()), net.Port(binary.BigEndian.Uint16(u)))
-	dst := net.UDPDestination(net.IPAddress(p.dst.AsSlice()), net.Port(binary.BigEndian.Uint16(u[2:])))
-	s.udp.HandlePacket(src, dst, b)
+	if socket == nil {
+		s.udp.HandlePacket(src, net.UDPDestination(net.IPAddress(p.dst.AsSlice()), net.Port(binary.BigEndian.Uint16(u[2:]))), b)
+	} else if b.UDP = &src; !socket.queue.push(b) {
+		b.Release()
+	}
 }

 // writeUDP writes a packet with the payload from src to dst to the tun
 func (s *stack) writeUDP(payload []byte, src net.Destination, dst net.Destination) error {
 	srcIP, _ := netip.AddrFromSlice(src.Address.IP())
 	dstIP, _ := netip.AddrFromSlice(dst.Address.IP())
-	srcIP, dstIP = srcIP.Unmap(), dstIP.Unmap()
-	if !srcIP.IsValid() || srcIP.Is4() != dstIP.Is4() {
-		return errors.New("failed to write udp packet from ", src, " to ", dst)
+	return s.sendUDP(payload, netip.AddrPortFrom(srcIP, uint16(src.Port)), netip.AddrPortFrom(dstIP, uint16(dst.Port)))
+}
+
+func (s *stack) sendUDP(payload []byte, src, dst netip.AddrPort) error {
+	srcIP, dstIP := src.Addr().Unmap(), dst.Addr().Unmap()
+	if !srcIP.IsValid() || srcIP.Is4() != dstIP.Is4() || len(payload) > math.MaxUint16-udpHeaderSize {
+		return errors.New("failed to write udp packet of ", len(payload), " bytes from ", src, " to ", dst)
 	}
 	var b [ipv6HeaderSize + udpHeaderSize]byte
 	n := putIPHeader(b[:], protoUDP, srcIP, dstIP)
 	u := b[n : n+udpHeaderSize]
-	binary.BigEndian.PutUint16(u, uint16(src.Port))
-	binary.BigEndian.PutUint16(u[2:], uint16(dst.Port))
+	binary.BigEndian.PutUint16(u, src.Port())
+	binary.BigEndian.PutUint16(u[2:], dst.Port())
 	binary.BigEndian.PutUint16(u[4:], uint16(udpHeaderSize+len(payload)))
-	setIPLength(b[:n], n+udpHeaderSize+len(payload))
 	check := transportChecksum(b[:n], protoUDP, u, payload)
 	if check == 0 {
 		// zero stands for no checksum
 		check = 0xffff
 	}
 	binary.BigEndian.PutUint16(u[6:], check)
-	return s.tun.WritePacket(b[:n+udpHeaderSize], payload)
+	return s.write(b[:n+udpHeaderSize], payload)
 }

-// handleICMP answers an echo request as its destination would
+// write completes the IP header of a packet that is in two parts and writes it to the tun, in fragments if it is
+// larger than the MTU. A segment of a connection never is, and is not written by this
+func (s *stack) write(header, payload []byte) error {
+	size, last := ipv6HeaderSize, 39
+	if header[0]>>4 == 4 {
+		size, last = ipv4HeaderSize, 19
+	}
+	// the group is chosen by the last bytes of the destination
+	id := s.identification[(header[last]^header[last-1]<<3)%byte(len(s.identification))].Add(1)
+	if header[0]>>4 == 4 {
+		// a link on the way may split it as well, and what is put together there is told apart by its number
+		binary.BigEndian.PutUint16(header[4:], uint16(id))
+	}
+	if len(header)+len(payload) > s.mtu {
+		return writeFragments(s.tun, s.mtu, id, header, payload)
+	}
+	setIPLength(header[:size], len(header)+len(payload))
+	return s.tun.WritePacket(header, payload)
+}
+
+// DialUDP implements Stack
+func (s *stack) DialUDP() (net.PacketConn, error) {
+	u := &udpSocket{stack: s, queue: newPacketQueue(udpQueueLimit)}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return nil, io.ErrClosedPipe
+	}
+	port, ok := ephemeral(func(port uint16) bool {
+		for _, a := range s.addresses {
+			if s.sockets[netip.AddrPortFrom(a, port)] != nil {
+				return false
+			}
+		}
+		return true
+	})
+	if !ok || len(s.addresses) == 0 {
+		return nil, errors.New("no port left on ", s.addresses)
+	}
+	u.port = port
+	for _, a := range s.addresses {
+		s.sockets[netip.AddrPortFrom(a, port)] = u
+	}
+	return u, nil
+}
+
+// udpSocket is a UDP port that was opened from here, on every address of the stack. It is not bound to one
+// other side: what anyone sends to it is read from it, with who sent it
+type udpSocket struct {
+	stack *stack
+	port  uint16
+	queue *packetQueue
+}
+
+func (u *udpSocket) ReadFrom(p []byte) (int, net.Addr, error) {
+	b, ok := u.queue.pop()
+	if !ok {
+		return 0, nil, gonet.ErrClosed
+	}
+	defer b.Release()
+	return copy(p, b.Bytes()), b.UDP.RawNetAddr(), nil
+}
+
+func (u *udpSocket) WriteTo(p []byte, addr net.Addr) (int, error) {
+	to, ok := addr.(*net.UDPAddr)
+	if !ok {
+		return 0, errors.New("not a udp address: ", addr)
+	}
+	local, err := u.stack.local(to.AddrPort().Addr().Unmap())
+	if err != nil {
+		return 0, err
+	}
+	if err := u.stack.sendUDP(p, netip.AddrPortFrom(local, u.port), to.AddrPort()); err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
+func (u *udpSocket) Close() error {
+	u.stack.mu.Lock()
+	for _, a := range u.stack.addresses {
+		if local := netip.AddrPortFrom(a, u.port); u.stack.sockets[local] == u {
+			delete(u.stack.sockets, local)
+		}
+	}
+	u.stack.mu.Unlock()
+	u.queue.close()
+	return nil
+}
+
+// LocalAddr implements net.PacketConn with the first address of the stack
+func (u *udpSocket) LocalAddr() net.Addr {
+	return gonet.UDPAddrFromAddrPort(netip.AddrPortFrom(u.stack.addresses[0], u.port))
+}
+
+func (u *udpSocket) SetDeadline(time.Time) error {
+	return nil
+}
+
+func (u *udpSocket) SetReadDeadline(time.Time) error {
+	return nil
+}
+
+func (u *udpSocket) SetWriteDeadline(time.Time) error {
+	return nil
+}
+
+// handleICMP answers an echo request as its destination would, and takes the word that a packet of a connection
+// was too large for a link on its way or found no way at all
 func (s *stack) handleICMP(p packet) {
 	m := p.payload
 	request, reply := byte(8), byte(0)
 	if p.proto == protoICMPv6 {
 		request, reply = 128, 129
 	}
-	if len(m) < 8 || m[0] != request || m[1] != 0 {
+	if len(m) < 8 {
+		return
+	}
+	switch {
+	case p.proto == protoICMPv4 && m[0] == 3 && m[1] == 4:
+		s.undelivered(m[8:], max(int(binary.BigEndian.Uint16(m[6:])), ipv4SmallestMTU))
+		return
+	case p.proto == protoICMPv6 && m[0] == 2:
+		s.undelivered(m[8:], max(int(binary.BigEndian.Uint32(m[4:])), ipv6SmallestMTU))
+		return
+	case p.proto == protoICMPv4 && m[0] == 3, p.proto == protoICMPv6 && m[0] == 1:
+		s.undelivered(m[8:], 0)
+		return
+	}
+	if m[0] != request || m[1] != 0 || s.handle == nil && !slices.Contains(s.addresses, p.dst) {
+		// a stack that takes no connections is one host, not all of them
 		return
 	}
 	var b [ipv6HeaderSize]byte
 	n := putIPHeader(b[:], p.proto, p.dst, p.src)
-	setIPLength(b[:n], n+len(m))
 	// the reply is the request with another type
 	m[0], m[2], m[3] = reply, 0, 0
 	if p.proto == protoICMPv6 {
@@ -276,5 +506,39 @@ func (s *stack) handleICMP(p packet) {
 	} else {
 		binary.BigEndian.PutUint16(m[2:], checksum(sum(m, 0)))
 	}
-	_ = s.tun.WritePacket(b[:n], m)
+	_ = s.write(b[:n], m)
+}
+
+// undelivered takes the start of a packet that was sent from here and did not get to where it goes. If it did not fit
+// a link on its way, which takes packets of mtu bytes, the connection that sent it goes on with smaller segments.
+// Without an mtu there is no way at all, and a connection that is being opened is not waited for
+func (s *stack) undelivered(b []byte, mtu int) {
+	var proto byte
+	var local, remote netip.Addr
+	switch {
+	case len(b) >= ipv4HeaderSize && b[0]>>4 == 4:
+		proto, local, remote = b[9], netip.AddrFrom4([4]byte(b[12:16])), netip.AddrFrom4([4]byte(b[16:20]))
+		b = b[min(int(b[0]&0xf)*4, len(b)):]
+	case len(b) >= ipv6HeaderSize && b[0]>>4 == 6:
+		proto, local, remote = b[6], netip.AddrFrom16([16]byte(b[8:24])), netip.AddrFrom16([16]byte(b[24:40]))
+		b = b[ipv6HeaderSize:]
+	}
+	// of the segment there are the ports and the sequence number at least
+	if proto != protoTCP || len(b) < 8 {
+		return
+	}
+	s.mu.Lock()
+	c := s.conns[flow{
+		src: netip.AddrPortFrom(remote, binary.BigEndian.Uint16(b[2:])),
+		dst: netip.AddrPortFrom(local, binary.BigEndian.Uint16(b)),
+	}]
+	s.mu.Unlock()
+	if c == nil {
+		return
+	}
+	if seq := binary.BigEndian.Uint32(b[4:]); mtu != 0 {
+		c.shrink(seq, mtu)
+	} else {
+		c.unreachable(seq)
+	}
 }