tstest/natlab/vnet: treat network wan/lan interface separately

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/tta/tta.go

@@ -0,0 +1,206 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The tta server is the Tailscale Test Agent.
+//
+// It runs on each Tailscale node being integration tested and permits the test
+// harness to control the node. It connects out to the test drver (rather than
+// accepting any TCP connections inbound, which might be blocked depending on
+// the scenario being tested) and then the test driver turns the TCP connection
+// around and sends request back.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/hostinfo"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+	"tailscale.com/version/distro"
+)
+
+var (
+	driverAddr = flag.String("driver", "test-driver.tailscale:8008", "address of the test driver; by default we use the DNS name test-driver.tailscale which is special cased in the emulated network's DNS server")
+)
+
+func absify(cmd string) string {
+	if distro.Get() == distro.Gokrazy && !strings.Contains(cmd, "/") {
+		return "/user/" + cmd
+	}
+	return cmd
+}
+
+func serveCmd(w http.ResponseWriter, cmd string, args ...string) {
+	log.Printf("Got serveCmd for %q %v", cmd, args)
+	out, err := exec.Command(absify(cmd), args...).CombinedOutput()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if err != nil {
+		w.Header().Set("Exec-Err", err.Error())
+		w.WriteHeader(500)
+		log.Printf("Err on serveCmd for %q %v, %d bytes of output: %v", cmd, args, len(out), err)
+	} else {
+		log.Printf("Did serveCmd for %q %v, %d bytes of output", cmd, args, len(out))
+	}
+	w.Write(out)
+}
+
+type localClientRoundTripper struct {
+	lc tailscale.LocalClient
+}
+
+func (rt *localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.RequestURI = ""
+	return rt.lc.DoLocalRequest(req)
+}
+
+func main() {
+	if distro.Get() == distro.Gokrazy {
+		if !hostinfo.IsNATLabGuestVM() {
+			// "Exiting immediately with status code 0 when the
+			// GOKRAZY_FIRST_START=1 environment variable is set means “don’t
+			// start the program on boot”"
+			return
+		}
+	}
+	flag.Parse()
+
+	if distro.Get() == distro.Gokrazy {
+		nsRx := regexp.MustCompile(`(?m)^nameserver (.*)`)
+		for t := time.Now(); time.Since(t) < 10*time.Second; time.Sleep(10 * time.Millisecond) {
+			all, _ := os.ReadFile("/etc/resolv.conf")
+			if nsRx.Match(all) {
+				break
+			}
+		}
+	}
+
+	logc, err := net.Dial("tcp", "9.9.9.9:124")
+	if err == nil {
+		log.SetOutput(logc)
+	}
+
+	log.Printf("Tailscale Test Agent running.")
+
+	gokRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+	gokRP.Transport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if network != "tcp" {
+				return nil, errors.New("unexpected network")
+			}
+			if addr != "gokrazy:80" {
+				return nil, errors.New("unexpected addr")
+			}
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", "/run/gokrazy-http.sock")
+		},
+	}
+
+	var ttaMux http.ServeMux // agent mux
+	var serveMux http.ServeMux
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("X-TTA-GoKrazy") == "1" {
+			gokRP.ServeHTTP(w, r)
+			return
+		}
+		ttaMux.ServeHTTP(w, r)
+	})
+	var hs http.Server
+	hs.Handler = &serveMux
+	var (
+		stMu   sync.Mutex
+		newSet = set.Set[net.Conn]{} // conns in StateNew
+	)
+	needConnCh := make(chan bool, 1)
+	hs.ConnState = func(c net.Conn, s http.ConnState) {
+		stMu.Lock()
+		defer stMu.Unlock()
+		switch s {
+		case http.StateNew:
+			newSet.Add(c)
+		default:
+			newSet.Delete(c)
+		}
+		if len(newSet) == 0 {
+			select {
+			case needConnCh <- true:
+			default:
+			}
+		}
+	}
+	conns := make(chan net.Conn, 1)
+
+	lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock")))
+	lcRP.Transport = new(localClientRoundTripper)
+	ttaMux.Handle("/localapi/", lcRP)
+
+	ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "TTA\n")
+		return
+	})
+	ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale")
+	})
+	go hs.Serve(chanListener(conns))
+
+	var lastErr string
+	needConnCh <- true
+	for {
+		<-needConnCh
+		c, err := connect()
+		if err != nil {
+			s := err.Error()
+			if s != lastErr {
+				log.Printf("Connect failure: %v", s)
+			}
+			lastErr = s
+			time.Sleep(time.Second)
+			continue
+		}
+		conns <- c
+	}
+}
+
+func connect() (net.Conn, error) {
+	c, err := net.Dial("tcp", *driverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+type chanListener <-chan net.Conn
+
+func (cl chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl
+	if !ok {
+		return nil, errors.New("closed")
+	}
+	return c, nil
+}
+
+func (cl chanListener) Close() error {
+	return nil
+}
+
+func (cl chanListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("52.0.0.34"), // TS..DR(iver)
+		Port: 123,
+	}
+}

cmd/vnet/run-krazy.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "Type 'C-a c' to enter monitor; q to quit."
+
+set -eux
+qemu-system-x86_64 -M microvm,isa-serial=off \
+    -m 1G \
+    -nodefaults -no-user-config -nographic \
+    -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
+    -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1" \
+    -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/tsapp.img,format=raw \
+    -device virtio-blk-device,drive=blk0 \
+    -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \
+    -device virtio-serial-device \
+    -device virtio-net-device,netdev=net0,mac=52:cc:cc:cc:cc:00 \
+    -chardev stdio,id=virtiocon0,mux=on \
+    -device virtconsole,chardev=virtiocon0 \
+    -mon chardev=virtiocon0,mode=readline \
+    -audio none
+

cmd/vnet/vnet-main.go

@@ -0,0 +1,118 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The vnet binary runs a virtual network stack in userspace for qemu instances
+// to connect to and simulate various network conditions.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+)
+
+var (
+	listen  = flag.String("listen", "/tmp/qemu.sock", "path to listen on")
+	nat     = flag.String("nat", "easy", "type of NAT to use")
+	nat2    = flag.String("nat2", "hard", "type of NAT to use for second network")
+	portmap = flag.Bool("portmap", false, "enable portmapping")
+	dgram   = flag.Bool("dgram", false, "enable datagram mode; for use with macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment")
+)
+
+func main() {
+	flag.Parse()
+
+	if _, err := os.Stat(*listen); err == nil {
+		os.Remove(*listen)
+	}
+
+	var srv net.Listener
+	var err error
+	var conn *net.UnixConn
+	if *dgram {
+		addr, err := net.ResolveUnixAddr("unixgram", *listen)
+		if err != nil {
+			log.Fatalf("ResolveUnixAddr: %v", err)
+		}
+		conn, err = net.ListenUnixgram("unixgram", addr)
+		if err != nil {
+			log.Fatalf("ListenUnixgram: %v", err)
+		}
+		defer conn.Close()
+	} else {
+		srv, err = net.Listen("unix", *listen)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var c vnet.Config
+	node1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.NAT(*nat)))
+	c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", vnet.NAT(*nat2)))
+	if *portmap {
+		node1.Network().AddService(vnet.NATPMP)
+	}
+
+	s, err := vnet.New(&c)
+	if err != nil {
+		log.Fatalf("newServer: %v", err)
+	}
+
+	if err := s.PopulateDERPMapIPs(); err != nil {
+		log.Printf("warning: ignoring failure to populate DERP map: %v", err)
+	}
+
+	s.WriteStartingBanner(os.Stdout)
+	nc := s.NodeAgentClient(node1)
+	go func() {
+		rp := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+		d := rp.Director
+		rp.Director = func(r *http.Request) {
+			d(r)
+			r.Header.Set("X-TTA-GoKrazy", "1")
+		}
+		rp.Transport = nc.HTTPClient.Transport
+		http.ListenAndServe(":8080", rp)
+	}()
+	go func() {
+		getStatus := func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			st, err := nc.Status(ctx)
+			if err != nil {
+				log.Printf("NodeStatus: %v", err)
+				return
+			}
+			log.Printf("NodeStatus: %v", logger.AsJSON(st))
+		}
+		for {
+			time.Sleep(5 * time.Second)
+			//continue
+			getStatus()
+		}
+	}()
+
+	if conn != nil {
+		s.ServeUnixConn(conn, vnet.ProtocolUnixDGRAM)
+		return
+	}
+
+	for {
+		c, err := srv.Accept()
+		if err != nil {
+			log.Printf("Accept: %v", err)
+			continue
+		}
+		go s.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+	}
+}

derp/derphttp/derphttp_server.go

@@ -18,6 +18,7 @@ import (
 // following its HTTP request.
 const fastStartHeader = "Derp-Fast-Start"
 
+// Handler returns an http.Handler to be mounted at /derp, serving s.
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// These are installed both here and in cmd/derper. The check here
@@ -79,3 +80,29 @@ func ProbeHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
 	}
 }
+
+// ServeNoContent generates the /generate_204 response used by Tailscale's
+// captive portal detection.
+func ServeNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(NoContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(NoContentResponseHeader, "response "+challenge)
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
+const (
+	NoContentChallengeHeader = "X-Tailscale-Challenge"
+	NoContentResponseHeader  = "X-Tailscale-Response"
+)

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/golangci/golangci-lint v1.52.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.18.0
+	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
 	github.com/goreleaser/nfpm/v2 v2.33.1

go.sum

@@ -477,6 +477,8 @@ github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=

go.toolchain.rev

@@ -1 +1 @@
-2f152a4eff5875655a9a84fce8f8d329f8d9a321
+22ef9eb38e9a2d21b4a45f7adc75addb05f3efb8

gokrazy/Makefile

@@ -6,3 +6,6 @@ image:
 
 qemu: image
 	qemu-system-x86_64 -m 1G -drive file=tsapp.img,format=raw -boot d -netdev user,id=user.0 -device virtio-net-pci,netdev=user.0 -serial mon:stdio -audio none
+
+qcow2: image
+	qemu-img convert -O qcow2 tsapp.img tsapp.qcow2

gokrazy/go.mod

@@ -2,12 +2,12 @@ module tailscale.com/gokrazy
 
 go 1.22
 
-require github.com/gokrazy/tools v0.0.0-20240510170341-34b02e215bc2
+require github.com/gokrazy/tools v0.0.0-20240730192548-9f81add3a91e
 
 require (
 	github.com/breml/rootcerts v0.2.10 // indirect
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 // indirect
-	github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
 	github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 // indirect
 	github.com/google/renameio/v2 v2.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -20,6 +20,4 @@ require (
 
 replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
 
-replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf
-
-replace github.com/gokrazy/internal => github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd
+replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e

gokrazy/go.sum

@@ -3,6 +3,8 @@ github.com/breml/rootcerts v0.2.10/go.mod h1:24FDtzYMpqIeYC7QzaE8VPRQaFZU5TIUDly
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 h1:kBY5R1tSf+EYZ+QaSrofLaVJtBqYsVNVBWkdMq3Smcg=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2/go.mod h1:PYOvzGOL4nlBmuxu7IyKQTFLaxr61+WPRNRzVtuYOHw=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
@@ -17,10 +19,8 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd h1:ZJplHHhYSzxYmrXuDPCNChGRZbLkPqRkYqRBM7KNyng=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf h1:lmAGqLbIVoMK1TYWqJvxKFsu+Tb1OecgvXTmypZGAZY=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf/go.mod h1:+PSix9a8BHqAz6RV/9+tiE3C1ou0GA1ViR8pqAZVfwI=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e h1:3/xIc1QCvnKL7BCLng9od98HEvxCadjvqiI/bN+Twso=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e/go.mod h1:eTZ0QsugEPFU5UAQ/87bKMkPxQuTNa7+iFAIahOFwRg=
 golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.mod

@@ -2,6 +2,14 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect
+require (
+	github.com/gokrazy/gokrazy v0.0.0-20240802144848-676865a4e84f // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/kenshaw/evdev v0.1.0 // indirect
+	github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
 
-replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
+replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.sum

@@ -2,6 +2,8 @@ github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZ
 github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a h1:FKeN678rNpKTpWRdFbAhYL9mWzPu57R5XPXCR3WmXdI=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/kenshaw/evdev v0.1.0 h1:wmtceEOFfilChgdNT+c/djPJ2JineVsQ0N14kGzFRUo=
@@ -12,5 +14,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a h1:7dnA8x14JihQmKbPr++Y5CCN/XSyDmOB6cXUxcIj6VQ=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f h1:ZSAGWpgs+6dK2oIz5OR+HUul3oJbnhFn8YNgcZ3d9SQ=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+golang.org/x/sys v0.0.0-20201005065044-765f4ea38db3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod

@@ -2,4 +2,4 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 // indirect
+require github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e // indirect

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.sum

@@ -1,2 +1,4 @@
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 h1:xzf+cMvBJBcA/Av7OTWBa0Tjrbfcy00TeatJeJt6zrY=
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e h1:tyUUgeRPGHjCZWycRnhdx8Lx9DRkjl3WsVUxYMrVBOw=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=

gokrazy/tsapp/builddir/tailscale.com/go.sum

@@ -122,8 +122,12 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1 h1:ycpNCSYwzZ7x4G4ioPNtKQmIY0G/3o4pVf8wCZq6blY=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98 h1:RNpJrXfI5u6e+uzyIzvmnXbhmhdRkVf//90sMBH3lso=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -170,6 +174,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3 h1:/8/t5pz/mgdRXhYOIeqqYhFAQLE4DDGegc0Y4ZjyFJM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=

gokrazy/tsapp/config.json

@@ -1,16 +1,21 @@
 {
     "Hostname": "tsapp",
-    "Update": { "NoPassword": true },
+    "Update": {
+        "NoPassword": true
+    },
     "SerialConsole": "ttyS0,115200",
     "Packages": [
         "github.com/gokrazy/serial-busybox",
         "github.com/gokrazy/breakglass",
         "tailscale.com/cmd/tailscale",
-        "tailscale.com/cmd/tailscaled"
+        "tailscale.com/cmd/tailscaled",
+        "tailscale.com/cmd/tta"
     ],
     "PackageConfig": {
         "github.com/gokrazy/breakglass": {
-            "CommandLineFlags": [ "-authorized_keys=ec2" ]
+            "CommandLineFlags": [
+                "-authorized_keys=ec2"
+            ]
         },
         "tailscale.com/cmd/tailscale": {
             "ExtraFilePaths": {
@@ -21,4 +26,4 @@
     "KernelPackage": "github.com/tailscale/gokrazy-kernel",
     "FirmwarePackage": "github.com/tailscale/gokrazy-kernel",
     "InternalCompatibilityFlags": {}
-}
+}

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/lineread"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 )
 
 var started = time.Now()
@@ -462,3 +463,15 @@ func IsSELinuxEnforcing() bool {
 	out, _ := exec.Command("getenforce").Output()
 	return string(bytes.TrimSpace(out)) == "Enforcing"
 }
+
+// IsNATLabGuestVM reports whether the current host is a NAT Lab guest VM.
+func IsNATLabGuestVM() bool {
+	if runtime.GOOS == "linux" && distro.Get() == distro.Gokrazy {
+		cmdLine, _ := os.ReadFile("/proc/cmdline")
+		return bytes.Contains(cmdLine, []byte("tailscale-tta=1"))
+	}
+	return false
+}
+
+// NAT Lab VMs have a unique MAC address prefix.
+// See

logpolicy/logpolicy.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/envknob"
 	"tailscale.com/health"
+	"tailscale.com/hostinfo"
 	"tailscale.com/log/filelogger"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
@@ -566,7 +567,7 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor,
 		conf.IncludeProcSequence = true
 	}
 
-	if envknob.NoLogsNoSupport() || testenv.InTest() {
+	if envknob.NoLogsNoSupport() || testenv.InTest() || hostinfo.IsNATLabGuestVM() {
 		logf("You have disabled logging. Tailscale will not be able to provide support.")
 		conf.HTTPC = &http.Client{Transport: noopPretendSuccessTransport{}}
 	} else if val := getLogTarget(); val != "" {

tstest/integration/integration.go

@@ -190,6 +190,7 @@ func RunDERPAndSTUN(t testing.TB, logf logger.Logf, ipAddress string) (derpMap *
 	}
 
 	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Listener.Close()
 	httpsrv.Listener = ln
 	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
 	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))

tstest/integration/nat/nat_test.go

@@ -0,0 +1,538 @@
+package nat
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/mod/modfile"
+	"golang.org/x/sync/errgroup"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/syncs"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+var (
+	logTailscaled = flag.Bool("log-tailscaled", false, "log tailscaled output")
+	pcapFile      = flag.String("pcap", "", "write pcap to file")
+)
+
+type natTest struct {
+	tb      testing.TB
+	base    string // base image
+	tempDir string // for qcow2 images
+	vnet    *vnet.Server
+	kernel  string // linux kernel path
+}
+
+func newNatTest(tb testing.TB) *natTest {
+	root, err := os.Getwd()
+	if err != nil {
+		tb.Fatal(err)
+	}
+	modRoot := filepath.Join(root, "../../..")
+
+	linuxKernel, err := findKernelPath(filepath.Join(modRoot, "gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod"))
+	if err != nil {
+		tb.Fatalf("findKernelPath: %v", err)
+	}
+	tb.Logf("found kernel: %v", linuxKernel)
+
+	nt := &natTest{
+		tb:      tb,
+		tempDir: tb.TempDir(),
+		base:    filepath.Join(modRoot, "gokrazy/tsapp.qcow2"),
+		kernel:  linuxKernel,
+	}
+
+	if _, err := os.Stat(nt.base); err != nil {
+		tb.Skipf("skipping test; base image %q not found", nt.base)
+	}
+	return nt
+}
+
+func findKernelPath(goMod string) (string, error) {
+	b, err := os.ReadFile(goMod)
+	if err != nil {
+		return "", err
+	}
+	mf, err := modfile.Parse("go.mod", b, nil)
+	if err != nil {
+		return "", err
+	}
+	goModB, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	for _, r := range mf.Require {
+		if r.Mod.Path == "github.com/tailscale/gokrazy-kernel" {
+			return strings.TrimSpace(string(goModB)) + "/" + r.Mod.String() + "/vmlinuz", nil
+		}
+	}
+	return "", fmt.Errorf("failed to find kernel in %v", goMod)
+}
+
+type addNodeFunc func(c *vnet.Config) *vnet.Node // returns nil to omit test
+
+func easy(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT))
+}
+
+func easyAF(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyAFNAT))
+}
+
+func sameLAN(c *vnet.Config) *vnet.Node {
+	nw := c.FirstNetwork()
+	if nw == nil {
+		return nil
+	}
+	if !nw.CanTakeMoreNodes() {
+		return nil
+	}
+	return c.AddNode(nw)
+}
+
+func one2one(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("172.16.%d.1/24", n), vnet.One2OneNAT))
+}
+
+func easyPMP(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
+}
+
+func hard(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("10.0.%d.1/24", n), vnet.HardNAT))
+}
+
+func hardPMP(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("10.7.%d.1/24", n), vnet.HardNAT, vnet.NATPMP))
+}
+
+func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute {
+	t := nt.tb
+
+	var c vnet.Config
+	c.SetPCAPFile(*pcapFile)
+	nodes := []*vnet.Node{
+		node1(&c),
+		node2(&c),
+	}
+	if nodes[0] == nil || nodes[1] == nil {
+		t.Skip("skipping test; not applicable combination")
+	}
+
+	var err error
+	nt.vnet, err = vnet.New(&c)
+	if err != nil {
+		t.Fatalf("newServer: %v", err)
+	}
+	nt.tb.Cleanup(func() {
+		nt.vnet.Close()
+	})
+
+	var wg sync.WaitGroup // waiting for srv.Accept goroutine
+	defer wg.Wait()
+
+	sockAddr := filepath.Join(nt.tempDir, "qemu.sock")
+	srv, err := net.Listen("unix", sockAddr)
+	if err != nil {
+		t.Fatalf("Listen: %v", err)
+	}
+	defer srv.Close()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			c, err := srv.Accept()
+			if err != nil {
+				return
+			}
+			go nt.vnet.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+		}
+	}()
+
+	for i, node := range nodes {
+		disk := fmt.Sprintf("%s/node-%d.qcow2", nt.tempDir, i)
+		out, err := exec.Command("qemu-img", "create",
+			"-f", "qcow2",
+			"-F", "qcow2",
+			"-b", nt.base,
+			disk).CombinedOutput()
+		if err != nil {
+			t.Fatalf("qemu-img create: %v, %s", err, out)
+		}
+
+		cmd := exec.Command("qemu-system-x86_64",
+			"-M", "microvm,isa-serial=off",
+			"-m", "384M",
+			"-nodefaults", "-no-user-config", "-nographic",
+			"-kernel", nt.kernel,
+			"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1",
+			"-drive", "id=blk0,file="+disk+",format=qcow2",
+			"-device", "virtio-blk-device,drive=blk0",
+			"-netdev", "stream,id=net0,addr.type=unix,addr.path="+sockAddr,
+			"-device", "virtio-serial-device",
+			"-device", "virtio-net-device,netdev=net0,mac="+node.MAC().String(),
+			"-chardev", "stdio,id=virtiocon0,mux=on",
+			"-device", "virtconsole,chardev=virtiocon0",
+			"-mon", "chardev=virtiocon0,mode=readline",
+			"-audio", "none",
+		)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Start(); err != nil {
+			t.Fatalf("qemu: %v", err)
+		}
+		nt.tb.Cleanup(func() {
+			cmd.Process.Kill()
+			cmd.Wait()
+		})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	lc1 := nt.vnet.NodeAgentClient(nodes[0])
+	lc2 := nt.vnet.NodeAgentClient(nodes[1])
+	clients := []*vnet.NodeAgentClient{lc1, lc2}
+
+	var eg errgroup.Group
+	var sts [2]*ipnstate.Status
+	for i, c := range clients {
+		i, c := i, c
+		eg.Go(func() error {
+			if *logTailscaled {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					streamDaemonLogs(ctx, t, c, fmt.Sprintf("node%d:", i))
+				}()
+			}
+			st, err := c.Status(ctx)
+			if err != nil {
+				return fmt.Errorf("node%d status: %w", i, err)
+			}
+			t.Logf("node%d status: %v", i, st)
+			if err := up(ctx, c); err != nil {
+				return fmt.Errorf("node%d up: %w", i, err)
+			}
+			t.Logf("node%d up!", i)
+			st, err = c.Status(ctx)
+			if err != nil {
+				return fmt.Errorf("node%d status: %w", i, err)
+			}
+			sts[i] = st
+
+			if st.BackendState != "Running" {
+				return fmt.Errorf("node%d state = %q", i, st.BackendState)
+			}
+			t.Logf("node%d up with %v", i, sts[i].Self.TailscaleIPs)
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		t.Fatalf("initial setup: %v", err)
+	}
+
+	defer nt.vnet.Close()
+
+	pingRes, err := ping(ctx, lc1, sts[1].Self.TailscaleIPs[0])
+	if err != nil {
+		t.Fatalf("ping failure: %v", err)
+	}
+	route := classifyPing(pingRes)
+	t.Logf("ping route: %v", route)
+	return route
+}
+
+func classifyPing(pr *ipnstate.PingResult) pingRoute {
+	if pr == nil {
+		return routeNil
+	}
+	if pr.Endpoint != "" {
+		ap, err := netip.ParseAddrPort(pr.Endpoint)
+		if err == nil {
+			if ap.Addr().IsPrivate() {
+				return routeLocal
+			}
+			return routeDirect
+		}
+	}
+	return routeDERP // presumably
+}
+
+type pingRoute string
+
+const (
+	routeDERP   pingRoute = "derp"
+	routeLocal  pingRoute = "local"
+	routeDirect pingRoute = "direct"
+	routeNil    pingRoute = "nil" // *ipnstate.PingResult is nil
+)
+
+func streamDaemonLogs(ctx context.Context, t testing.TB, c *vnet.NodeAgentClient, nodeID string) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	r, err := c.TailDaemonLogs(ctx)
+	if err != nil {
+		t.Errorf("tailDaemonLogs: %v", err)
+		return
+	}
+	logger := log.New(os.Stderr, nodeID+" ", log.Lmsgprefix)
+	dec := json.NewDecoder(r)
+	for {
+		// /{"logtail":{"client_time":"2024-08-08T17:42:31.95095956Z","proc_id":2024742977,"proc_seq":232},"text":"magicsock: derp-1 connected; connGen=1\n"}
+		var logEntry struct {
+			LogTail struct {
+				ClientTime time.Time `json:"client_time"`
+			}
+			Text string `json:"text"`
+		}
+		if err := dec.Decode(&logEntry); err != nil {
+			if err == io.EOF || errors.Is(err, context.Canceled) {
+				return
+			}
+			t.Errorf("log entry: %v", err)
+			return
+		}
+		logger.Printf("%s %s", logEntry.LogTail.ClientTime.Format("2006/01/02 15:04:05"), logEntry.Text)
+	}
+}
+
+func ping(ctx context.Context, c *vnet.NodeAgentClient, target netip.Addr) (*ipnstate.PingResult, error) {
+	n := 0
+	var res *ipnstate.PingResult
+	anyPong := false
+	for n < 10 {
+		n++
+		pr, err := c.PingWithOpts(ctx, target, tailcfg.PingDisco, tailscale.PingOpts{})
+		if err != nil {
+			if anyPong {
+				return res, nil
+			}
+			return nil, err
+		}
+		if pr.Err != "" {
+			return nil, errors.New(pr.Err)
+		}
+		if pr.DERPRegionID == 0 {
+			return pr, nil
+		}
+		res = pr
+		select {
+		case <-ctx.Done():
+		case <-time.After(time.Second):
+		}
+	}
+	if res == nil {
+		return nil, errors.New("no ping response")
+	}
+	return res, nil
+}
+
+func up(ctx context.Context, c *vnet.NodeAgentClient) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://unused/up", nil)
+	if err != nil {
+		return err
+	}
+	res, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	all, _ := io.ReadAll(res.Body)
+	if res.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code %v: %s", res.Status, all)
+	}
+	return nil
+}
+
+func TestEasyEasy(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, easy)
+}
+
+func TestEasyHard(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, hard)
+}
+
+func TestEasyAFHard(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easyAF, hard)
+}
+
+func TestEasyHardPMP(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, hardPMP)
+}
+
+func TestEasyPMPHard(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easyPMP, hard)
+}
+
+func TestHardHardPMP(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(hard, hardPMP)
+}
+
+func TestHardHard(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(hard, hard)
+}
+
+func TestEasyOne2One(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, one2one)
+}
+
+func TestHardOne2One(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(hard, one2one)
+}
+
+func TestGrid(t *testing.T) {
+	t.Parallel()
+
+	type nodeType struct {
+		name string
+		fn   addNodeFunc
+	}
+	types := []nodeType{
+		{"easy", easy},
+		{"easyAF", easyAF},
+		{"hard", hard},
+		{"easyPMP", easyPMP},
+		{"hardPMP", hardPMP},
+		{"one2one", one2one},
+		{"sameLAN", sameLAN},
+	}
+
+	sem := syncs.NewSemaphore(2)
+	var (
+		mu  sync.Mutex
+		res = make(map[string]pingRoute)
+	)
+	for _, a := range types {
+		for _, b := range types {
+			key := a.name + "-" + b.name
+			keyBack := b.name + "-" + a.name
+			t.Run(key, func(t *testing.T) {
+				t.Parallel()
+
+				sem.Acquire()
+				defer sem.Release()
+
+				filename := key + ".cache"
+				contents, _ := os.ReadFile(filename)
+				if len(contents) == 0 {
+					filename2 := keyBack + ".cache"
+					contents, _ = os.ReadFile(filename2)
+				}
+				route := pingRoute(strings.TrimSpace(string(contents)))
+
+				if route == "" {
+					nt := newNatTest(t)
+					route = nt.runTest(a.fn, b.fn)
+					if err := os.WriteFile(filename, []byte(string(route)), 0666); err != nil {
+						t.Fatalf("writeFile: %v", err)
+					}
+				}
+
+				mu.Lock()
+				defer mu.Unlock()
+				res[key] = route
+				t.Logf("results: %v", res)
+			})
+		}
+	}
+
+	t.Cleanup(func() {
+		mu.Lock()
+		defer mu.Unlock()
+		var hb bytes.Buffer
+		pf := func(format string, args ...interface{}) {
+			fmt.Fprintf(&hb, format, args...)
+		}
+		rewrite := func(s string) string {
+			return strings.ReplaceAll(s, "PMP", "+pm")
+		}
+		pf("<html><table border=1 cellpadding=5>")
+		pf("<tr><td></td>")
+		for _, a := range types {
+			pf("<td><b>%s</b></td>", rewrite(a.name))
+		}
+		pf("</tr>\n")
+
+		for _, a := range types {
+			if a.name == "sameLAN" {
+				continue
+			}
+			pf("<tr><td><b>%s</b></td>", rewrite(a.name))
+			for _, b := range types {
+				key := a.name + "-" + b.name
+				key2 := b.name + "-" + a.name
+				v := cmp.Or(res[key], res[key2], "-")
+				if v == "derp" {
+					pf("<td><div style='color: red; font-weight: bold'>%s</div></td>", v)
+				} else if v == "local" {
+					pf("<td><div style='color: green; font-weight: bold'>%s</div></td>", v)
+				} else {
+					pf("<td>%s</td>", v)
+				}
+			}
+			pf("</tr>\n")
+		}
+		pf("</table>")
+		pf("<b>easy</b>: Endpoint-Independent Mapping, Address and Port-Dependent Filtering (e.g. Linux, Google Wifi, Unifi, eero)<br>")
+		pf("<b>easyAF</b>: Endpoint-Independent Mapping, Address-Dependent Filtering (James says telephony things or Zyxel type things)<br>")
+		pf("<b>hard</b>: Address and Port-Dependent Mapping, Address and Port-Dependent Filtering (FreeBSD, OPNSense, pfSense)<br>")
+		pf("<b>one2one</b>: One-to-One NAT (e.g. an EC2 instance with a public IPv4)<br>")
+		pf("<b>x+pm</b>: x, with port mapping (NAT-PMP, PCP, UPnP, etc)<br>")
+		pf("<b>sameLAN</b>: a second node in the same LAN as the first<br>")
+		pf("</html>")
+
+		if err := os.WriteFile("grid.html", hb.Bytes(), 0666); err != nil {
+			t.Fatalf("writeFile: %v", err)
+		}
+	})
+}

tstest/natlab/vnet/conf.go

@@ -0,0 +1,279 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"cmp"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"slices"
+
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcapgo"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+)
+
+// Note: the exported Node and Network are the configuration types;
+// the unexported node and network are the runtime types that are actually
+// used once the server is created.
+
+// Config is the requested state of the natlab virtual network.
+//
+// The zero value is a valid empty configuration. Call AddNode
+// and AddNetwork to methods on the returned Node and Network
+// values to modify the config before calling NewServer.
+// Once the NewServer is called, Config is no longer used.
+type Config struct {
+	nodes    []*Node
+	networks []*Network
+	pcapFile string
+}
+
+func (c *Config) SetPCAPFile(file string) {
+	c.pcapFile = file
+}
+
+func (c *Config) NumNodes() int {
+	return len(c.nodes)
+}
+
+func (c *Config) FirstNetwork() *Network {
+	if len(c.networks) == 0 {
+		return nil
+	}
+	return c.networks[0]
+}
+
+// AddNode creates a new node in the world.
+//
+// The opts may be of the following types:
+//   - *Network: zero, one, or more networks to add this node to
+//   - TODO: more
+//
+// On an error or unknown opt type, AddNode returns a
+// node with a carried error that gets returned later.
+func (c *Config) AddNode(opts ...any) *Node {
+	num := len(c.nodes)
+	n := &Node{
+		mac: MAC{0x52, 0xcc, 0xcc, 0xcc, 0xcc, byte(num)}, // 52=TS then 0xcc for ccclient
+	}
+	c.nodes = append(c.nodes, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case *Network:
+			if !slices.Contains(o.nodes, n) {
+				o.nodes = append(o.nodes, n)
+			}
+			n.nets = append(n.nets, o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNode option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// AddNetwork add a new network.
+//
+// The opts may be of the following types:
+//   - string IP address, for the network's WAN IP (if any)
+//   - string netip.Prefix, for the network's LAN IP (defaults to 192.168.0.0/24)
+//   - NAT, the type of NAT to use
+//   - NetworkService, a service to add to the network
+//
+// On an error or unknown opt type, AddNetwork returns a
+// network with a carried error that gets returned later.
+func (c *Config) AddNetwork(opts ...any) *Network {
+	num := len(c.networks)
+	n := &Network{
+		mac: MAC{0x52, 0xee, 0xee, 0xee, 0xee, byte(num)}, // 52=TS then 0xee for 'etwork
+	}
+	c.networks = append(c.networks, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case string:
+			if ip, err := netip.ParseAddr(o); err == nil {
+				n.wanIP = ip
+			} else if ip, err := netip.ParsePrefix(o); err == nil {
+				n.lanIP = ip
+			} else {
+				if n.err == nil {
+					n.err = fmt.Errorf("unknown string option %q", o)
+				}
+			}
+		case NAT:
+			n.natType = o
+		case NetworkService:
+			n.AddService(o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNetwork option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// Node is the configuration of a node in the virtual network.
+type Node struct {
+	err error
+	n   *node // nil until NewServer called
+
+	// TODO(bradfitz): this is halfway converted to supporting multiple NICs
+	// but not done. We need a MAC-per-Network.
+
+	mac  MAC
+	nets []*Network
+}
+
+// MAC returns the MAC address of the node.
+func (n *Node) MAC() MAC {
+	return n.mac
+}
+
+// Network returns the first network this node is connected to,
+// or nil if none.
+func (n *Node) Network() *Network {
+	if len(n.nets) == 0 {
+		return nil
+	}
+	return n.nets[0]
+}
+
+// Network is the configuration of a network in the virtual network.
+type Network struct {
+	mac     MAC // MAC address of the router/gateway
+	natType NAT
+
+	wanIP netip.Addr
+	lanIP netip.Prefix
+	nodes []*Node
+
+	svcs set.Set[NetworkService]
+
+	// ...
+	err error // carried error
+}
+
+func (n *Network) CanTakeMoreNodes() bool {
+	if n.natType == One2OneNAT {
+		return len(n.nodes) == 0
+	}
+	return len(n.nodes) < 150
+}
+
+// NetworkService is a service that can be added to a network.
+type NetworkService string
+
+const (
+	NATPMP NetworkService = "NAT-PMP"
+	PCP    NetworkService = "PCP"
+	UPnP   NetworkService = "UPnP"
+)
+
+// AddService adds a network service (such as port mapping protocols) to a
+// network.
+func (n *Network) AddService(s NetworkService) {
+	if n.svcs == nil {
+		n.svcs = set.Of(s)
+	} else {
+		n.svcs.Add(s)
+	}
+}
+
+// initFromConfig initializes the server from the previous calls
+// to NewNode and NewNetwork and returns an error if
+// there were any configuration issues.
+func (s *Server) initFromConfig(c *Config) error {
+	netOfConf := map[*Network]*network{}
+	if c.pcapFile != "" {
+		pcf, err := os.OpenFile(c.pcapFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		nw, err := pcapgo.NewNgWriter(pcf, layers.LinkTypeEthernet)
+		if err != nil {
+			return err
+		}
+		pw := &pcapWriter{
+			f: pcf,
+			w: nw,
+		}
+		s.pcapWriter = pw
+	}
+	for i, conf := range c.networks {
+		if conf.err != nil {
+			return conf.err
+		}
+		if !conf.lanIP.IsValid() {
+			conf.lanIP = netip.MustParsePrefix("192.168.0.0/24")
+		}
+		n := &network{
+			s:         s,
+			mac:       conf.mac,
+			portmap:   conf.svcs.Contains(NATPMP), // TODO: expand network.portmap
+			wanIP:     conf.wanIP,
+			lanIP:     conf.lanIP,
+			nodesByIP: map[netip.Addr]*node{},
+			logf:      logger.WithPrefix(log.Printf, fmt.Sprintf("[net-%v] ", conf.mac)),
+		}
+		netOfConf[conf] = n
+		s.networks.Add(n)
+		if _, ok := s.networkByWAN[conf.wanIP]; ok {
+			return fmt.Errorf("two networks have the same WAN IP %v; Anycast not (yet?) supported", conf.wanIP)
+		}
+		s.networkByWAN[conf.wanIP] = n
+		n.lanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("network%d-lan", i+1),
+			LinkType: layers.LinkTypeIPv4,
+		}))
+		n.wanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("network%d-wan", i+1),
+			LinkType: layers.LinkTypeIPv4,
+		}))
+	}
+	for i, conf := range c.nodes {
+		if conf.err != nil {
+			return conf.err
+		}
+		n := &node{
+			mac: conf.mac,
+			net: netOfConf[conf.Network()],
+		}
+		n.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("node%d", i+1),
+			LinkType: layers.LinkTypeEthernet,
+		}))
+		conf.n = n
+		if _, ok := s.nodeByMAC[n.mac]; ok {
+			return fmt.Errorf("two nodes have the same MAC %v", n.mac)
+		}
+		s.nodes = append(s.nodes, n)
+		s.nodeByMAC[n.mac] = n
+
+		// Allocate a lanIP for the node. Use the network's CIDR and use final
+		// octet 101 (for first node), 102, etc. The node number comes from the
+		// last octent of the MAC address (0-based)
+		ip4 := n.net.lanIP.Addr().As4()
+		ip4[3] = 101 + n.mac[5]
+		n.lanIP = netip.AddrFrom4(ip4)
+		n.net.nodesByIP[n.lanIP] = n
+	}
+
+	// Now that nodes are populated, set up NAT:
+	for _, conf := range c.networks {
+		n := netOfConf[conf]
+		natType := cmp.Or(conf.natType, EasyNAT)
+		if err := n.InitNAT(natType); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tstest/natlab/vnet/conf_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import "testing"
+
+func TestConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		setup   func(*Config)
+		wantErr string
+	}{
+		{
+			name: "simple",
+			setup: func(c *Config) {
+				c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", EasyNAT, NATPMP))
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", HardNAT))
+			},
+		},
+		{
+			name: "indirect",
+			setup: func(c *Config) {
+				n1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", HardNAT))
+				n1.Network().AddService(NATPMP)
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", NAT("hard")))
+			},
+		},
+		{
+			name: "multi-node-in-net",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+		},
+		{
+			name: "dup-wan-ip",
+			setup: func(c *Config) {
+				c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNetwork("2.1.1.1", "10.2.0.1/16")
+			},
+			wantErr: "two networks have the same WAN IP 2.1.1.1; Anycast not (yet?) supported",
+		},
+		{
+			name: "one-to-one-nat-with-multiple-nodes",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24", One2OneNAT)
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+			wantErr: "error creating NAT type \"one2one\" for network 2.1.1.1: can't use one2one NAT type on networks other than single-node networks",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var c Config
+			tt.setup(&c)
+			_, err := New(&c)
+			if err == nil {
+				if tt.wantErr == "" {
+					return
+				}
+				t.Fatalf("got success; wanted error %q", tt.wantErr)
+			}
+			if err.Error() != tt.wantErr {
+				t.Fatalf("got error %q; want %q", err, tt.wantErr)
+			}
+		})
+	}
+}

tstest/natlab/vnet/easyaf.go

@@ -0,0 +1,88 @@
+package vnet
+
+import (
+	"log"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+// easyAFNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux), but with only address filtering, not address+port
+// filtering.
+//
+// James says these are used by "anyone with “voip helpers” turned on"
+// "which is a lot of home modem routers" ... "probably like most of the zyxel
+// type things".
+type easyAFNAT struct {
+	pool    IPPool
+	wanIP   netip.Addr
+	out     map[netip.Addr]portMappingAndTime
+	in      map[uint16]lanAddrAndTime
+	lastOut map[srcAPDstAddrTuple]time.Time // (lan:port, wan:port) => last packet out time
+}
+
+type srcAPDstAddrTuple struct {
+	src netip.AddrPort
+	dst netip.Addr
+}
+
+func init() {
+	registerNATType(EasyAFNAT, func(p IPPool) (NATTable, error) {
+		return &easyAFNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyAFNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	_, ok := n.in[ap.Port()]
+	return ok
+}
+
+func (n *easyAFNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	mak.Set(&n.lastOut, srcAPDstAddrTuple{src, dst.Addr()}, at)
+	if pm, ok := n.out[src.Addr()]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+			if n.pool.IsPublicPortUsed(wanAddr) {
+				continue
+			}
+
+			// Found a free port.
+			mak.Set(&n.out, src.Addr(), portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyAFNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	lanDst = n.in[dst.Port()].lanAddr
+
+	// Stateful firewall: drop incoming packets that don't have traffic out.
+	// TODO(bradfitz): verify Linux does this in the router code, not in the NAT code.
+	if t, ok := n.lastOut[srcAPDstAddrTuple{lanDst, src.Addr()}]; !ok || at.Sub(t) > 300*time.Second {
+		log.Printf("Drop incoming packet from %v to %v; no recent outgoing packet", src, dst)
+		return netip.AddrPort{}
+	}
+
+	return lanDst
+}

tstest/natlab/vnet/nat.go

@@ -0,0 +1,293 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"errors"
+	"log"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+const (
+	One2OneNAT NAT = "one2one"
+	EasyNAT    NAT = "easy"   // address+port filtering
+	EasyAFNAT  NAT = "easyaf" // address filtering (not port)
+	HardNAT    NAT = "hard"
+)
+
+// IPPool is the interface that a NAT implementation uses to get information
+// about a network.
+//
+// Outside of tests, this is typically a *network.
+type IPPool interface {
+	// WANIP returns the primary WAN IP address.
+	//
+	// TODO: add another method for networks with multiple WAN IP addresses.
+	WANIP() netip.Addr
+
+	// SoleLanIP reports whether this network has a sole LAN client
+	// and if so, its IP address.
+	SoleLANIP() (_ netip.Addr, ok bool)
+
+	// IsPublicPortUsed reports whether the provided WAN IP+port is in use by
+	// anything. (In particular, the NAT-PMP/etc port mappers might have taken
+	// a port.) Implementations should check this before allocating a port,
+	// and then they should report IsPublicPortUsed themselves for that port.
+	IsPublicPortUsed(netip.AddrPort) bool
+}
+
+// newTableFunc is a constructor for a NAT table.
+// The provided IPPool is typically (outside of tests) a *network.
+type newTableFunc func(IPPool) (NATTable, error)
+
+// NAT is a type of NAT that's known to natlab.
+//
+// For example, "easy" for Linux-style NAT, "hard" for FreeBSD-style NAT, etc.
+type NAT string
+
+// natTypes are the known NAT types.
+var natTypes = map[NAT]newTableFunc{}
+
+// registerNATType registers a NAT type.
+func registerNATType(name NAT, f newTableFunc) {
+	if _, ok := natTypes[name]; ok {
+		panic("duplicate NAT type: " + name)
+	}
+	natTypes[name] = f
+}
+
+// NATTable is what a NAT implementation is expected to do.
+//
+// This project tests Tailscale as it faces various combinations various NAT
+// implementations (e.g. Linux easy style NAT vs FreeBSD hard/endpoint dependent
+// NAT vs Cloud 1:1 NAT, etc)
+//
+// Implementations of NATTable need not handle concurrency; the natlab serializes
+// all calls into a NATTable.
+//
+// The provided `at` value will typically be time.Now, except for tests.
+// Implementations should not use real time and should only compare
+// previously provided time values.
+type NATTable interface {
+	// PickOutgoingSrc returns the source address to use for an outgoing packet.
+	//
+	// The result should either be invalid (to drop the packet) or a WAN (not
+	// private) IP address.
+	//
+	// Typically, the src is a LAN source IP address, but it might also be a WAN
+	// IP address if the packet is being forwarded for a source machine that has
+	// a public IP address.
+	PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort)
+
+	// PickIncomingDst returns the destination address to use for an incoming
+	// packet. The incoming src address is always a public WAN IP.
+	//
+	// The result should either be invalid (to drop the packet) or the IP
+	// address of a machine on the local network address, usually a private
+	// LAN IP.
+	PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort)
+
+	// IsPublicPortUsed reports whether the provided WAN IP+port is in use by
+	// anything. The port mapper uses this to avoid grabbing an in-use port.
+	IsPublicPortUsed(netip.AddrPort) bool
+}
+
+// oneToOneNAT is a 1:1 NAT, like a typical EC2 VM.
+type oneToOneNAT struct {
+	lanIP netip.Addr
+	wanIP netip.Addr
+}
+
+func init() {
+	registerNATType(One2OneNAT, func(p IPPool) (NATTable, error) {
+		lanIP, ok := p.SoleLANIP()
+		if !ok {
+			return nil, errors.New("can't use one2one NAT type on networks other than single-node networks")
+		}
+		return &oneToOneNAT{lanIP: lanIP, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *oneToOneNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	return netip.AddrPortFrom(n.wanIP, src.Port())
+}
+
+func (n *oneToOneNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	return netip.AddrPortFrom(n.lanIP, dst.Port())
+}
+
+func (n *oneToOneNAT) IsPublicPortUsed(netip.AddrPort) bool {
+	return true // all ports are owned by the 1:1 NAT
+}
+
+type srcDstTuple struct {
+	src netip.AddrPort
+	dst netip.AddrPort
+}
+
+type hardKeyIn struct {
+	wanPort uint16
+	src     netip.AddrPort
+}
+
+type portMappingAndTime struct {
+	port uint16
+	at   time.Time
+}
+
+type lanAddrAndTime struct {
+	lanAddr netip.AddrPort
+	at      time.Time
+}
+
+// hardNAT is an "Endpoint Dependent" NAT, like FreeBSD/pfSense/OPNsense.
+// This is shown as "MappingVariesByDestIP: true" by netcheck, and what
+// Tailscale calls "Hard NAT".
+type hardNAT struct {
+	pool  IPPool
+	wanIP netip.Addr
+
+	out map[srcDstTuple]portMappingAndTime
+	in  map[hardKeyIn]lanAddrAndTime
+}
+
+func init() {
+	registerNATType(HardNAT, func(p IPPool) (NATTable, error) {
+		return &hardNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *hardNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	for k := range n.in {
+		if k.wanPort == ap.Port() {
+			return true
+		}
+	}
+	return false
+}
+
+func (n *hardNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	ko := srcDstTuple{src, dst}
+	if pm, ok := n.out[ko]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// No existing mapping exists. Create one.
+
+	// TODO: clean up old expired mappings
+
+	// Instead of proper data structures that would be efficient, we instead
+	// just loop a bunch and look for a free port. This project is only used
+	// by tests and doesn't care about performance, this is good enough.
+	for {
+		port := rand.N(uint16(32<<10)) + 32<<10 // pick some "ephemeral" port
+		if n.pool.IsPublicPortUsed(netip.AddrPortFrom(n.wanIP, port)) {
+			continue
+		}
+
+		ki := hardKeyIn{wanPort: port, src: dst}
+		if _, ok := n.in[ki]; ok {
+			// Port already in use.
+			continue
+		}
+		mak.Set(&n.in, ki, lanAddrAndTime{lanAddr: src, at: at})
+		mak.Set(&n.out, ko, portMappingAndTime{port: port, at: at})
+		return netip.AddrPortFrom(n.wanIP, port)
+	}
+}
+
+func (n *hardNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	ki := hardKeyIn{wanPort: dst.Port(), src: src}
+	if pm, ok := n.in[ki]; ok {
+		// Existing flow.
+		return pm.lanAddr
+	}
+	return netip.AddrPort{} // drop; no mapping
+}
+
+// easyNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux).
+//
+// This is shown as "MappingVariesByDestIP: false" by netcheck, and what
+// Tailscale calls "Easy NAT".
+//
+// Unlike Linux, this implementation is capped at 32k entries and doesn't resort
+// to other allocation strategies when all 32k WAN ports are taken.
+type easyNAT struct {
+	pool    IPPool
+	wanIP   netip.Addr
+	out     map[netip.AddrPort]portMappingAndTime
+	in      map[uint16]lanAddrAndTime
+	lastOut map[srcDstTuple]time.Time // (lan:port, wan:port) => last packet out time
+}
+
+func init() {
+	registerNATType(EasyNAT, func(p IPPool) (NATTable, error) {
+		return &easyNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	_, ok := n.in[ap.Port()]
+	return ok
+}
+
+func (n *easyNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	mak.Set(&n.lastOut, srcDstTuple{src, dst}, at)
+	if pm, ok := n.out[src]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+			if n.pool.IsPublicPortUsed(wanAddr) {
+				continue
+			}
+
+			// Found a free port.
+			mak.Set(&n.out, src, portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	lanDst = n.in[dst.Port()].lanAddr
+
+	// Stateful firewall: drop incoming packets that don't have traffic out.
+	// TODO(bradfitz): verify Linux does this in the router code, not in the NAT code.
+	if t, ok := n.lastOut[srcDstTuple{lanDst, src}]; !ok || at.Sub(t) > 300*time.Second {
+		log.Printf("Drop incoming packet from %v to %v; no recent outgoing packet", src, dst)
+		return netip.AddrPort{}
+	}
+
+	return lanDst
+}

tstest/natlab/vnet/pcap.go

@@ -0,0 +1,51 @@
+package vnet
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/pcapgo"
+)
+
+type pcapWriter struct {
+	f *os.File
+
+	mu sync.Mutex
+	w  *pcapgo.NgWriter
+}
+
+func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if p == nil {
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.w == nil {
+		return io.ErrClosedPipe
+	}
+	return p.w.WritePacket(ci, data)
+}
+
+func (p *pcapWriter) AddInterface(i pcapgo.NgInterface) (int, error) {
+	if p == nil {
+		return 0, nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.w.AddInterface(i)
+}
+
+func (p *pcapWriter) Close() error {
+	if p == nil {
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.w != nil {
+		p.w.Flush()
+		p.w = nil
+	}
+	return p.f.Close()
+}