build(deps): bump github.com/docker/docker

Bumps [github.com/docker/docker](https://github.com/docker/docker) from 26.1.4+incompatible to 26.1.5+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](https://github.com/docker/docker/compare/v26.1.4...v26.1.5)

---
updated-dependencies:
- dependency-name: github.com/docker/docker
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/stunstamp/stunstamp.go

@@ -42,7 +42,9 @@ var (
 	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
 	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
 	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
-	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+	flagSTUNDstPorts   = flag.String("stun-dst-ports", "", "comma-separated list of STUN destination ports to monitor")
+	flagHTTPSDstPorts  = flag.String("https-dst-ports", "", "comma-separated list of HTTPS destination ports to monitor")
+	flagICMP           = flag.Bool("icmp", false, "probe ICMP")
 )
 
 const (
@@ -89,12 +91,21 @@ func (t timestampSource) String() string {
 	}
 }
 
+type protocol string
+
+const (
+	protocolSTUN  protocol = "stun"
+	protocolICMP  protocol = "icmp"
+	protocolHTTPS protocol = "https"
+)
+
 // resultKey contains the stable dimensions and their values for a given
 // timeseries, i.e. not time and not rtt/timeout.
 type resultKey struct {
 	meta            nodeMeta
 	timestampSource timestampSource
 	connStability   connStability
+	protocol        protocol
 	dstPort         int
 }
 
@@ -104,7 +115,7 @@ type result struct {
 	rtt *time.Duration // nil signifies failure, e.g. timeout
 }
 
-func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTT(conn io.ReadWriteCloser, dst netip.AddrPort) (rtt time.Duration, err error) {
 	uconn, ok := conn.(*net.UDPConn)
 	if !ok {
 		return 0, fmt.Errorf("unexpected conn type: %T", conn)
@@ -116,7 +127,10 @@ func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, e
 	txID := stun.NewTxID()
 	req := stun.Request(txID)
 	txAt := time.Now()
-	_, err = uconn.WriteToUDP(req, dst)
+	_, err = uconn.WriteToUDP(req, &net.UDPAddr{
+		IP:   dst.Addr().AsSlice(),
+		Port: int(dst.Port()),
+	})
 	if err != nil {
 		return 0, fmt.Errorf("error writing to udp socket: %w", err)
 	}
@@ -153,11 +167,11 @@ type nodeMeta struct {
 	addr       netip.Addr
 }
 
-type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
+type measureFn func(conn io.ReadWriteCloser, dst netip.AddrPort) (rtt time.Duration, err error)
 
-// probe measures STUN round trip time for the node described by meta over
-// conn against dstPort. It may return a nil duration and nil error if the
-// STUN request timed out. A non-nil error indicates an unrecoverable or
+// probe measures round trip time for the node described by meta over
+// conn against dstPort using fn. It may return a nil duration and nil error in
+// the event of a timeout. A non-nil error indicates an unrecoverable or
 // non-temporary error.
 func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
 	ua := &net.UDPAddr{
@@ -166,7 +180,7 @@ func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*
 	}
 
 	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
-	rtt, err := fn(conn, ua)
+	rtt, err := fn(conn, netip.AddrPortFrom(meta.addr, uint16(dstPort)))
 	if err != nil {
 		if isTemporaryOrTimeoutErr(err) {
 			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
@@ -237,43 +251,71 @@ func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]node
 	return stale, nil
 }
 
-func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
-	conns := [2]io.ReadWriteCloser{}
-	byDstPort, ok := stableConns[addr]
-	if ok {
-		conns, ok = byDstPort[dstPort]
-		if ok {
-			return conns, nil
+func newConn(source timestampSource, protocol protocol) (io.ReadWriteCloser, error) {
+	switch protocol {
+	case protocolSTUN:
+		if source == timestampSourceKernel {
+			return getUDPConnKernelTimestamp()
+		} else {
+			return net.ListenUDP("udp", &net.UDPAddr{})
 		}
+	case protocolICMP:
+		// TODO(jwhited): implement
+		return nil, errors.New("unimplemented protocol")
+	case protocolHTTPS:
+		// TODO(jwhited): implement
+		return nil, errors.New("unimplemented protocol")
 	}
-	if supportsKernelTS() {
-		kconn, err := getConnKernelTimestamp()
+	return nil, errors.New("unknown protocol")
+}
+
+type stableConnKey struct {
+	node     netip.Addr
+	protocol protocol
+	port     int
+}
+
+func getStableConns(stableConns map[stableConnKey][2]io.ReadWriteCloser, addr netip.Addr, protocol protocol, dstPort int) ([2]io.ReadWriteCloser, error) {
+	if !protocolSupportsStableConn(protocol) {
+		return [2]io.ReadWriteCloser{}, nil
+	}
+	conns, ok := stableConns[stableConnKey{addr, protocol, dstPort}]
+	if ok {
+		return conns, nil
+	}
+
+	if protocolSupportsKernelTS(protocol) {
+		kconn, err := newConn(timestampSourceKernel, protocol)
 		if err != nil {
 			return conns, err
 		}
 		conns[timestampSourceKernel] = kconn
 	}
-	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
+	uconn, err := newConn(timestampSourceUserspace, protocol)
 	if err != nil {
-		if supportsKernelTS() {
+		if protocolSupportsKernelTS(protocol) {
 			conns[timestampSourceKernel].Close()
 		}
 		return conns, err
 	}
 	conns[timestampSourceUserspace] = uconn
-	if byDstPort == nil {
-		byDstPort = make(map[int][2]io.ReadWriteCloser)
-	}
-	byDstPort[dstPort] = conns
-	stableConns[addr] = byDstPort
 	return conns, nil
 }
 
-// probeNodes measures the round-trip time for STUN binding requests against the
-// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
-// UDP sockets that should be recycled across runs. It returns the results or
-// an error if one occurs.
-func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+func protocolSupportsStableConn(p protocol) bool {
+	if p == protocolICMP {
+		// no value for ICMP
+		return false
+	}
+	return true
+}
+
+// probeNodes measures the round-trip time for the protocols and ports described
+// by portsByProtocol against the DERP nodes described by nodeMetaByAddr.
+// stableConns are used to recycle connections across calls to probeNodes.
+// probeNodes is also responsible for trimming stableConns based on node
+// lifetime in nodeMetaByAddr. It returns the results or an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[stableConnKey][2]io.ReadWriteCloser, portsByProtocol map[protocol][]int) ([]result, error) {
 	wg := sync.WaitGroup{}
 	results := make([]result, 0)
 	resultsCh := make(chan result)
@@ -283,23 +325,20 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 	at := time.Now()
 	addrsToProbe := make(map[netip.Addr]bool)
 
-	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, protocol protocol, dstPort int) {
 		defer wg.Done()
 		r := result{
 			key: resultKey{
 				meta:            meta,
 				timestampSource: source,
 				dstPort:         dstPort,
+				protocol:        protocol,
 			},
 			at: at,
 		}
 		if conn == nil {
 			var err error
-			if source == timestampSourceKernel {
-				conn, err = getConnKernelTimestamp()
-			} else {
-				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
-			}
+			conn, err = newConn(source, protocol)
 			if err != nil {
 				select {
 				case <-doneCh:
@@ -312,9 +351,17 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 		} else {
 			r.key.connStability = stableConn
 		}
-		fn := measureRTT
-		if source == timestampSourceKernel {
-			fn = measureRTTKernel
+		var fn measureFn
+		switch protocol {
+		case protocolSTUN:
+			fn = measureSTUNRTT
+			if source == timestampSourceKernel {
+				fn = measureSTUNRTTKernel
+			}
+		case protocolICMP:
+			// TODO(jwhited): implement
+		case protocolHTTPS:
+			// TODO(jwhited): implement
 		}
 		rtt, err := probe(meta, conn, fn, dstPort)
 		if err != nil {
@@ -334,37 +381,47 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 
 	for _, meta := range nodeMetaByAddr {
 		addrsToProbe[meta.addr] = true
-		for _, port := range dstPorts {
-			stable, err := getStableConns(stableConns, meta.addr, port)
-			if err != nil {
-				close(doneCh)
-				wg.Wait()
-				return nil, err
-			}
+		for p, ports := range portsByProtocol {
+			for _, port := range ports {
+				stable, err := getStableConns(stableConns, meta.addr, p, port)
+				if err != nil {
+					close(doneCh)
+					wg.Wait()
+					return nil, err
+				}
 
-			wg.Add(2)
-			numProbes += 2
-			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
-			go doProbe(nil, meta, timestampSourceUserspace, port)
-			if supportsKernelTS() {
-				wg.Add(2)
-				numProbes += 2
-				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
-				go doProbe(nil, meta, timestampSourceKernel, port)
+				if protocolSupportsStableConn(p) {
+					wg.Add(1)
+					numProbes++
+					go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, p, port)
+				}
+				wg.Add(1)
+				numProbes++
+				go doProbe(nil, meta, timestampSourceUserspace, p, port)
+
+				if protocolSupportsKernelTS(p) {
+					if protocolSupportsStableConn(p) {
+						wg.Add(1)
+						numProbes++
+						go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, p, port)
+					}
+
+					wg.Add(1)
+					numProbes++
+					go doProbe(nil, meta, timestampSourceKernel, p, port)
+				}
 			}
 		}
 	}
 
 	// cleanup conns we no longer need
-	for k, byDstPort := range stableConns {
-		if !addrsToProbe[k] {
-			for _, conns := range byDstPort {
-				if conns[timestampSourceKernel] != nil {
-					conns[timestampSourceKernel].Close()
-				}
-				conns[timestampSourceUserspace].Close()
-				delete(stableConns, k)
+	for k, conns := range stableConns {
+		if !addrsToProbe[k.node] {
+			if conns[timestampSourceKernel] != nil {
+				conns[timestampSourceKernel].Close()
 			}
+			conns[timestampSourceUserspace].Close()
+			delete(stableConns, k)
 		}
 	}
 
@@ -391,11 +448,11 @@ const (
 )
 
 const (
-	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
-	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+	rttMetricName      = "stunstamp_derp_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_timeouts_total"
 )
 
-func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, protocol protocol, dstPort int) []prompb.Label {
 	addressFamily := "ipv4"
 	if meta.addr.Is6() {
 		addressFamily = "ipv6"
@@ -425,6 +482,10 @@ func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source
 		Name:  "hostname",
 		Value: meta.hostname,
 	})
+	labels = append(labels, prompb.Label{
+		Name:  "protocol",
+		Value: string(protocol),
+	})
 	labels = append(labels, prompb.Label{
 		Name:  "dst_port",
 		Value: strconv.Itoa(dstPort),
@@ -453,53 +514,61 @@ const (
 	staleNaN uint64 = 0x7ff0000000000002
 )
 
-func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, portsByProtocol map[protocol][]int) []prompb.TimeSeries {
 	staleMarkers := make([]prompb.TimeSeries, 0)
 	now := time.Now()
-	for _, s := range stale {
-		for _, dstPort := range dstPorts {
-			samples := []prompb.Sample{
-				{
-					Timestamp: now.UnixMilli(),
-					Value:     math.Float64frombits(staleNaN),
-				},
-			}
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			if supportsKernelTS() {
+
+	for p, ports := range portsByProtocol {
+		for _, port := range ports {
+			for _, s := range stale {
+				samples := []prompb.Sample{
+					{
+						Timestamp: now.UnixMilli(),
+						Value:     math.Float64frombits(staleNaN),
+					},
+				}
 				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
+					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, p, port),
 					Samples: samples,
 				})
 				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
+					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, p, port),
 					Samples: samples,
 				})
+				if protocolSupportsStableConn(p) {
+					staleMarkers = append(staleMarkers, prompb.TimeSeries{
+						Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, p, port),
+						Samples: samples,
+					})
+					staleMarkers = append(staleMarkers, prompb.TimeSeries{
+						Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, p, port),
+						Samples: samples,
+					})
+				}
+				if protocolSupportsKernelTS(p) {
+					staleMarkers = append(staleMarkers, prompb.TimeSeries{
+						Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, p, port),
+						Samples: samples,
+					})
+					staleMarkers = append(staleMarkers, prompb.TimeSeries{
+						Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, p, port),
+						Samples: samples,
+					})
+					if protocolSupportsStableConn(p) {
+						staleMarkers = append(staleMarkers, prompb.TimeSeries{
+							Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, p, port),
+							Samples: samples,
+						})
+						staleMarkers = append(staleMarkers, prompb.TimeSeries{
+							Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, p, port),
+							Samples: samples,
+						})
+					}
+				}
 			}
 		}
 	}
+
 	return staleMarkers
 }
 
@@ -513,7 +582,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 	for _, r := range results {
 		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
 		seenKeys[r.key] = true
-		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		rttSamples := make([]prompb.Sample, 1)
 		rttSamples[0].Timestamp = r.at.UnixMilli()
 		if r.rtt != nil {
@@ -528,7 +597,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 		}
 		all = append(all, rttTS)
 		timeouts[r.key] = timeoutsCount
-		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		timeoutsSamples := make([]prompb.Sample, 1)
 		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
 		timeoutsSamples[0].Value = float64(timeoutsCount)
@@ -620,22 +689,56 @@ func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSer
 	}
 }
 
+func getPortsFromFlag(f string) ([]int, error) {
+	if len(f) == 0 {
+		return nil, nil
+	}
+	split := strings.Split(f, ",")
+	slices.Sort(split)
+	split = slices.Compact(split)
+	ports := make([]int, 0)
+	for _, portStr := range split {
+		port, err := strconv.ParseUint(portStr, 10, 16)
+		if err != nil {
+			return nil, err
+		}
+		ports = append(ports, int(port))
+	}
+	return ports, nil
+}
+
 func main() {
 	flag.Parse()
-	if len(*flagDstPorts) == 0 {
-		log.Fatal("dst-ports flag is unset")
+
+	portsByProtocol := make(map[protocol][]int)
+	stunPorts, err := getPortsFromFlag(*flagSTUNDstPorts)
+	if err != nil {
+		log.Fatalf("invalid stun-dst-ports flag value: %v", err)
 	}
-	dstPortsSplit := strings.Split(*flagDstPorts, ",")
-	slices.Sort(dstPortsSplit)
-	dstPortsSplit = slices.Compact(dstPortsSplit)
-	dstPorts := make([]int, 0, len(dstPortsSplit))
-	for _, d := range dstPortsSplit {
-		i, err := strconv.ParseUint(d, 10, 16)
-		if err != nil {
-			log.Fatal("invalid dst-ports")
+	if len(stunPorts) > 0 {
+		portsByProtocol[protocolSTUN] = stunPorts
+	}
+	httpsPorts, err := getPortsFromFlag(*flagHTTPSDstPorts)
+	if err != nil {
+		log.Fatalf("invalid https-dst-ports flag value: %v", err)
+	}
+	if len(httpsPorts) > 0 {
+		portsByProtocol[protocolHTTPS] = httpsPorts
+	}
+	if *flagICMP {
+		portsByProtocol[protocolICMP] = []int{0}
+	}
+	if len(portsByProtocol) == 0 {
+		log.Fatal("nothing to probe")
+	}
+
+	// TODO(jwhited): remove protocol restriction
+	for k := range portsByProtocol {
+		if k != protocolSTUN {
+			log.Fatal("HTTPS & ICMP are not yet supported")
 		}
-		dstPorts = append(dstPorts, int(i))
 	}
+
 	if len(*flagDERPMap) < 1 {
 		log.Fatal("derp-map flag is unset")
 	}
@@ -645,7 +748,7 @@ func main() {
 	if len(*flagRemoteWriteURL) < 1 {
 		log.Fatal("rw-url flag is unset")
 	}
-	_, err := url.Parse(*flagRemoteWriteURL)
+	_, err = url.Parse(*flagRemoteWriteURL)
 	if err != nil {
 		log.Fatalf("invalid rw-url flag value: %v", err)
 	}
@@ -707,7 +810,7 @@ func main() {
 		for _, v := range nodeMetaByAddr {
 			staleMeta = append(staleMeta, v)
 		}
-		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 		if len(staleMarkers) > 0 {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 			rwc.write(ctx, staleMarkers)
@@ -723,8 +826,8 @@ func main() {
 	// in a higher probability of the packets traversing the same underlay path.
 	// Comparison of stable and unstable 5-tuple results can shed light on
 	// differences between paths where hashing (multipathing/load balancing)
-	// comes into play.
-	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+	// comes into play. The inner 2 element array index is timestampSource.
+	stableConns := make(map[stableConnKey][2]io.ReadWriteCloser)
 
 	// timeouts holds counts of timeout events. Values are persisted for the
 	// lifetime of the related node in the DERP map.
@@ -738,7 +841,7 @@ func main() {
 	for {
 		select {
 		case <-probeTicker.C:
-			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			results, err := probeNodes(nodeMetaByAddr, stableConns, portsByProtocol)
 			if err != nil {
 				log.Printf("unrecoverable error while probing: %v", err)
 				shutdown()
@@ -761,7 +864,7 @@ func main() {
 				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
 				continue
 			}
-			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 			if len(staleMarkers) < 1 {
 				continue
 			}

cmd/stunstamp/stunstamp_default.go

@@ -8,18 +8,18 @@ package main
 import (
 	"errors"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unimplemented")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, dst netip.AddrPort) (rtt time.Duration, err error) {
 	return 0, errors.New("unimplemented")
 }
 
-func supportsKernelTS() bool {
+func protocolSupportsKernelTS(_ protocol) bool {
 	return false
 }

cmd/stunstamp/stunstamp_linux.go

@@ -10,7 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 
 	"github.com/mdlayher/socket"
@@ -24,7 +24,7 @@ const (
 		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
 	if err != nil {
 		return nil, err
@@ -56,24 +56,23 @@ func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
 	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, dst netip.AddrPort) (rtt time.Duration, err error) {
 	sconn, ok := conn.(*socket.Conn)
 	if !ok {
 		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
 	}
 
 	var to unix.Sockaddr
-	to4 := dst.IP.To4()
-	if to4 != nil {
+	if dst.Addr().Is4() {
 		to = &unix.SockaddrInet4{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
 	} else {
 		to = &unix.SockaddrInet6{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
 	}
 
 	txID := stun.NewTxID()
@@ -138,6 +137,10 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 
 }
 
-func supportsKernelTS() bool {
-	return true
+func protocolSupportsKernelTS(p protocol) bool {
+	if p == protocolSTUN {
+		return true
+	}
+	// TODO: jwhited support ICMP
+	return false
 }

cmd/tsidp/tsidp.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/rsa"
@@ -16,6 +17,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -25,6 +27,7 @@ import (
 	"net/netip"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +38,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -44,13 +48,22 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 	"tailscale.com/util/rands"
+	"tailscale.com/version"
 )
 
+// ctxConn is a key to look up a net.Conn stored in an HTTP request's context.
+type ctxConn struct{}
+
+// funnelClientsFile is the file where client IDs and secrets for OIDC clients
+// accessing the IDP over Funnel are persisted.
+const funnelClientsFile = "oidc-funnel-clients.json"
+
 var (
 	flagVerbose            = flag.Bool("verbose", false, "be verbose")
 	flagPort               = flag.Int("port", 443, "port to listen on")
 	flagLocalPort          = flag.Int("local-port", -1, "allow requests from localhost")
 	flagUseLocalTailscaled = flag.Bool("use-local-tailscaled", false, "use local tailscaled instead of tsnet")
+	flagFunnel             = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet")
 )
 
 func main() {
@@ -61,9 +74,11 @@ func main() {
 	}
 
 	var (
-		lc  *tailscale.LocalClient
-		st  *ipnstate.Status
-		err error
+		lc          *tailscale.LocalClient
+		st          *ipnstate.Status
+		err         error
+		watcherChan chan error
+		cleanup     func()
 
 		lns []net.Listener
 	)
@@ -90,6 +105,18 @@ func main() {
 		if !anySuccess {
 			log.Fatalf("failed to listen on any of %v", st.TailscaleIPs)
 		}
+
+		// tailscaled needs to be setting an HTTP header for funneled requests
+		// that older versions don't provide.
+		// TODO(naman): is this the correct check?
+		if *flagFunnel && !version.AtLeast(st.Version, "1.71.0") {
+			log.Fatalf("Local tailscaled not new enough to support -funnel. Update Tailscale or use tsnet mode.")
+		}
+		cleanup, watcherChan, err = serveOnLocalTailscaled(ctx, lc, st, uint16(*flagPort), *flagFunnel)
+		if err != nil {
+			log.Fatalf("could not serve on local tailscaled: %v", err)
+		}
+		defer cleanup()
 	} else {
 		ts := &tsnet.Server{
 			Hostname: "idp",
@@ -105,7 +132,15 @@ func main() {
 		if err != nil {
 			log.Fatalf("getting local client: %v", err)
 		}
-		ln, err := ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		var ln net.Listener
+		if *flagFunnel {
+			if err := ipn.CheckFunnelAccess(uint16(*flagPort), st.Self); err != nil {
+				log.Fatalf("%v", err)
+			}
+			ln, err = ts.ListenFunnel("tcp", fmt.Sprintf(":%d", *flagPort))
+		} else {
+			ln, err = ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		}
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -113,13 +148,26 @@ func main() {
 	}
 
 	srv := &idpServer{
-		lc: lc,
+		lc:          lc,
+		funnel:      *flagFunnel,
+		localTSMode: *flagUseLocalTailscaled,
 	}
 	if *flagPort != 443 {
 		srv.serverURL = fmt.Sprintf("https://%s:%d", strings.TrimSuffix(st.Self.DNSName, "."), *flagPort)
 	} else {
 		srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, "."))
 	}
+	if *flagFunnel {
+		f, err := os.Open(funnelClientsFile)
+		if err == nil {
+			srv.funnelClients = make(map[string]*funnelClient)
+			if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil {
+				log.Fatalf("could not parse %s: %v", funnelClientsFile, err)
+			}
+		} else if !errors.Is(err, os.ErrNotExist) {
+			log.Fatalf("could not open %s: %v", funnelClientsFile, err)
+		}
+	}
 
 	log.Printf("Running tsidp at %s ...", srv.serverURL)
 
@@ -134,35 +182,129 @@ func main() {
 	}
 
 	for _, ln := range lns {
-		go http.Serve(ln, srv)
+		server := http.Server{
+			Handler: srv,
+			ConnContext: func(ctx context.Context, c net.Conn) context.Context {
+				return context.WithValue(ctx, ctxConn{}, c)
+			},
+		}
+		go server.Serve(ln)
 	}
-	select {}
+	// need to catch os.Interrupt, otherwise deferred cleanup code doesn't run
+	exitChan := make(chan os.Signal, 1)
+	signal.Notify(exitChan, os.Interrupt)
+	select {
+	case <-exitChan:
+		log.Printf("interrupt, exiting")
+		return
+	case <-watcherChan:
+		if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
+			log.Printf("watcher closed, exiting")
+			return
+		}
+		log.Fatalf("watcher error: %v", err)
+		return
+	}
+}
+
+// serveOnLocalTailscaled starts a serve session using an already-running
+// tailscaled instead of starting a fresh tsnet server, making something
+// listening on clientDNSName:dstPort accessible over serve/funnel.
+func serveOnLocalTailscaled(ctx context.Context, lc *tailscale.LocalClient, st *ipnstate.Status, dstPort uint16, shouldFunnel bool) (cleanup func(), watcherChan chan error, err error) {
+	// In order to support funneling out in local tailscaled mode, we need
+	// to add a serve config to forward the listeners we bound above and
+	// allow those forwarders to be funneled out.
+	sc, err := lc.GetServeConfig(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get serve config: %v", err)
+	}
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+
+	// We watch the IPN bus just to get a session ID. The session expires
+	// when we stop watching the bus, and that auto-deletes the foreground
+	// serve/funnel configs we are creating below.
+	watcher, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not set up ipn bus watcher: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+	n, err := watcher.Next()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get initial state from ipn bus watcher: %v", err)
+	}
+	if n.SessionID == "" {
+		err = fmt.Errorf("missing sessionID in ipn.Notify")
+		return nil, nil, err
+	}
+	watcherChan = make(chan error)
+	go func() {
+		for {
+			_, err = watcher.Next()
+			if err != nil {
+				watcherChan <- err
+				return
+			}
+		}
+	}()
+
+	// Create a foreground serve config that gets cleaned up when tsidp
+	// exits and the session ID associated with this config is invalidated.
+	foregroundSc := new(ipn.ServeConfig)
+	mak.Set(&sc.Foreground, n.SessionID, foregroundSc)
+	serverURL := strings.TrimSuffix(st.Self.DNSName, ".")
+	fmt.Printf("setting funnel for %s:%v\n", serverURL, dstPort)
+
+	foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel)
+	foregroundSc.SetWebHandler(&ipn.HTTPHandler{
+		Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))),
+	}, serverURL, uint16(*flagPort), "/", true)
+	err = lc.SetServeConfig(ctx, sc)
+	if err != nil {
+		return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err)
+	}
+
+	return func() { watcher.Close() }, watcherChan, nil
 }
 
 type idpServer struct {
 	lc          *tailscale.LocalClient
 	loopbackURL string
 	serverURL   string // "https://foo.bar.ts.net"
+	funnel      bool
+	localTSMode bool
 
 	lazyMux        lazy.SyncValue[*http.ServeMux]
 	lazySigningKey lazy.SyncValue[*signingKey]
 	lazySigner     lazy.SyncValue[jose.Signer]
 
-	mu          sync.Mutex              // guards the fields below
-	code        map[string]*authRequest // keyed by random hex
-	accessToken map[string]*authRequest // keyed by random hex
+	mu            sync.Mutex               // guards the fields below
+	code          map[string]*authRequest  // keyed by random hex
+	accessToken   map[string]*authRequest  // keyed by random hex
+	funnelClients map[string]*funnelClient // keyed by client ID
 }
 
 type authRequest struct {
 	// localRP is true if the request is from a relying party running on the
-	// same machine as the idp server. It is mutually exclusive with rpNodeID.
+	// same machine as the idp server. It is mutually exclusive with rpNodeID
+	// and funnelRP.
 	localRP bool
 
 	// rpNodeID is the NodeID of the relying party (who requested the auth, such
 	// as Proxmox or Synology), not the user node who is being authenticated. It
-	// is mutually exclusive with localRP.
+	// is mutually exclusive with localRP and funnelRP.
 	rpNodeID tailcfg.NodeID
 
+	// funnelRP is non-nil if the request is from a relying party outside the
+	// tailnet, via Tailscale Funnel. It is mutually exclusive with rpNodeID
+	// and localRP.
+	funnelRP *funnelClient
+
 	// clientID is the "client_id" sent in the authorized request.
 	clientID string
 
@@ -181,9 +323,12 @@ type authRequest struct {
 	validTill time.Time
 }
 
-func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string, lc *tailscale.LocalClient) error {
+// allowRelyingParty validates that a relying party identified either by a
+// known remoteAddr or a valid client ID/secret pair is allowed to proceed
+// with the authorization flow associated with this authRequest.
+func (ar *authRequest) allowRelyingParty(r *http.Request, lc *tailscale.LocalClient) error {
 	if ar.localRP {
-		ra, err := netip.ParseAddrPort(remoteAddr)
+		ra, err := netip.ParseAddrPort(r.RemoteAddr)
 		if err != nil {
 			return err
 		}
@@ -192,7 +337,18 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 		}
 		return nil
 	}
-	who, err := lc.WhoIs(ctx, remoteAddr)
+	if ar.funnelRP != nil {
+		clientID, clientSecret, ok := r.BasicAuth()
+		if !ok {
+			clientID = r.FormValue("client_id")
+			clientSecret = r.FormValue("client_secret")
+		}
+		if ar.funnelRP.ID != clientID || ar.funnelRP.Secret != clientSecret {
+			return fmt.Errorf("tsidp: invalid client credentials")
+		}
+		return nil
+	}
+	who, err := lc.WhoIs(r.Context(), r.RemoteAddr)
 	if err != nil {
 		return fmt.Errorf("tsidp: error getting WhoIs: %w", err)
 	}
@@ -203,24 +359,60 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 }
 
 func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
-	who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr)
+	// This URL is visited by the user who is being authenticated. If they are
+	// visiting the URL over Funnel, that means they are not part of the
+	// tailnet that they are trying to be authenticated for.
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	uq := r.URL.Query()
+
+	redirectURI := uq.Get("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must specify redirect_uri", http.StatusBadRequest)
+		return
+	}
+
+	var remoteAddr string
+	if s.localTSMode {
+		// in local tailscaled mode, the local tailscaled is forwarding us
+		// HTTP requests, so reading r.RemoteAddr will just get us our own
+		// address.
+		remoteAddr = r.Header.Get("X-Forwarded-For")
+	} else {
+		remoteAddr = r.RemoteAddr
+	}
+	who, err := s.lc.WhoIs(r.Context(), remoteAddr)
 	if err != nil {
 		log.Printf("Error getting WhoIs: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	uq := r.URL.Query()
-
 	code := rands.HexString(32)
 	ar := &authRequest{
 		nonce:       uq.Get("nonce"),
 		remoteUser:  who,
-		redirectURI: uq.Get("redirect_uri"),
+		redirectURI: redirectURI,
 		clientID:    uq.Get("client_id"),
 	}
 
-	if r.URL.Path == "/authorize/localhost" {
+	if r.URL.Path == "/authorize/funnel" {
+		s.mu.Lock()
+		c, ok := s.funnelClients[ar.clientID]
+		s.mu.Unlock()
+		if !ok {
+			http.Error(w, "tsidp: invalid client ID", http.StatusBadRequest)
+			return
+		}
+		if ar.redirectURI != c.RedirectURI {
+			http.Error(w, "tsidp: redirect_uri mismatch", http.StatusBadRequest)
+			return
+		}
+		ar.funnelRP = c
+	} else if r.URL.Path == "/authorize/localhost" {
 		ar.localRP = true
 	} else {
 		var ok bool
@@ -237,8 +429,10 @@ func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
 
 	q := make(url.Values)
 	q.Set("code", code)
-	q.Set("state", uq.Get("state"))
-	u := uq.Get("redirect_uri") + "?" + q.Encode()
+	if state := uq.Get("state"); state != "" {
+		q.Set("state", state)
+	}
+	u := redirectURI + "?" + q.Encode()
 	log.Printf("Redirecting to %q", u)
 
 	http.Redirect(w, r, u, http.StatusFound)
@@ -251,6 +445,7 @@ func (s *idpServer) newMux() *http.ServeMux {
 	mux.HandleFunc("/authorize/", s.authorize)
 	mux.HandleFunc("/userinfo", s.serveUserInfo)
 	mux.HandleFunc("/token", s.serveToken)
+	mux.HandleFunc("/clients/", s.serveClients)
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
 			io.WriteString(w, "<html><body><h1>Tailscale OIDC IdP</h1>")
@@ -284,11 +479,6 @@ func (s *idpServer) serveUserInfo(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: invalid token", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
-		log.Printf("Error allowing relying party: %v", err)
-		http.Error(w, err.Error(), http.StatusForbidden)
-		return
-	}
 
 	if ar.validTill.Before(time.Now()) {
 		http.Error(w, "tsidp: token expired", http.StatusBadRequest)
@@ -348,7 +538,7 @@ func (s *idpServer) serveToken(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: code not found", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
+	if err := ar.allowRelyingParty(r, s.lc); err != nil {
 		log.Printf("Error allowing relying party: %v", err)
 		http.Error(w, err.Error(), http.StatusForbidden)
 		return
@@ -581,7 +771,9 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 	var authorizeEndpoint string
 	rpEndpoint := s.serverURL
-	if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
+	if isFunnelRequest(r) {
+		authorizeEndpoint = fmt.Sprintf("%s/authorize/funnel", s.serverURL)
+	} else if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
 		authorizeEndpoint = fmt.Sprintf("%s/authorize/%d", s.serverURL, who.Node.ID)
 	} else if ap.Addr().IsLoopback() {
 		rpEndpoint = s.loopbackURL
@@ -611,6 +803,148 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// funnelClient represents an OIDC client/relying party that is accessing the
+// IDP over Funnel.
+type funnelClient struct {
+	ID          string `json:"client_id"`
+	Secret      string `json:"client_secret,omitempty"`
+	Name        string `json:"name,omitempty"`
+	RedirectURI string `json:"redirect_uri"`
+}
+
+// /clients is a privileged endpoint that allows the visitor to create new
+// Funnel-capable OIDC clients, so it is only accessible over the tailnet.
+func (s *idpServer) serveClients(w http.ResponseWriter, r *http.Request) {
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	path := strings.TrimPrefix(r.URL.Path, "/clients/")
+
+	if path == "new" {
+		s.serveNewClient(w, r)
+		return
+	}
+
+	if path == "" {
+		s.serveGetClientsList(w, r)
+		return
+	}
+
+	s.mu.Lock()
+	c, ok := s.funnelClients[path]
+	s.mu.Unlock()
+	if !ok {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	switch r.Method {
+	case "DELETE":
+		s.serveDeleteClient(w, r, path)
+	case "GET":
+		json.NewEncoder(w).Encode(&funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	default:
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *idpServer) serveNewClient(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	redirectURI := r.FormValue("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must provide redirect_uri", http.StatusBadRequest)
+		return
+	}
+	clientID := rands.HexString(32)
+	clientSecret := rands.HexString(64)
+	newClient := funnelClient{
+		ID:          clientID,
+		Secret:      clientSecret,
+		Name:        r.FormValue("name"),
+		RedirectURI: redirectURI,
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.funnelClients, clientID, &newClient)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// delete the new client to avoid inconsistent state between memory
+		// and disk
+		delete(s.funnelClients, clientID)
+		return
+	}
+	json.NewEncoder(w).Encode(newClient)
+}
+
+func (s *idpServer) serveGetClientsList(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	redactedClients := make([]funnelClient, 0, len(s.funnelClients))
+	for _, c := range s.funnelClients {
+		redactedClients = append(redactedClients, funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	}
+	s.mu.Unlock()
+	json.NewEncoder(w).Encode(redactedClients)
+}
+
+func (s *idpServer) serveDeleteClient(w http.ResponseWriter, r *http.Request, clientID string) {
+	if r.Method != "DELETE" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.funnelClients == nil {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	if _, ok := s.funnelClients[clientID]; !ok {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	deleted := s.funnelClients[clientID]
+	delete(s.funnelClients, clientID)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// restore the deleted value to avoid inconsistent state between memory
+		// and disk
+		s.funnelClients[clientID] = deleted
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// storeFunnelClientsLocked writes the current mapping of OIDC client ID/secret
+// pairs for RPs that access the IDP over funnel. s.mu must be held while
+// calling this.
+func (s *idpServer) storeFunnelClientsLocked() error {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(s.funnelClients); err != nil {
+		return err
+	}
+	return os.WriteFile(funnelClientsFile, buf.Bytes(), 0600)
+}
+
 const (
 	minimumRSAKeySize = 2048
 )
@@ -700,3 +1034,24 @@ func parseID[T ~int64](input string) (_ T, ok bool) {
 	}
 	return T(i), true
 }
+
+// isFunnelRequest checks if an HTTP request is coming over Tailscale Funnel.
+func isFunnelRequest(r *http.Request) bool {
+	// If we're funneling through the local tailscaled, it will set this HTTP
+	// header.
+	if r.Header.Get("Tailscale-Funnel-Request") != "" {
+		return true
+	}
+
+	// If the funneled connection is from tsnet, then the net.Conn will be of
+	// type ipn.FunnelConn.
+	netConn := r.Context().Value(ctxConn{})
+	// if the conn is wrapped inside TLS, unwrap it
+	if tlsConn, ok := netConn.(*tls.Conn); ok {
+		netConn = tlsConn.NetConn()
+	}
+	if _, ok := netConn.(*ipn.FunnelConn); ok {
+		return true
+	}
+	return false
+}

cmd/tta/tta.go

@@ -0,0 +1,206 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The tta server is the Tailscale Test Agent.
+//
+// It runs on each Tailscale node being integration tested and permits the test
+// harness to control the node. It connects out to the test drver (rather than
+// accepting any TCP connections inbound, which might be blocked depending on
+// the scenario being tested) and then the test driver turns the TCP connection
+// around and sends request back.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/hostinfo"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+	"tailscale.com/version/distro"
+)
+
+var (
+	driverAddr = flag.String("driver", "test-driver.tailscale:8008", "address of the test driver; by default we use the DNS name test-driver.tailscale which is special cased in the emulated network's DNS server")
+)
+
+func absify(cmd string) string {
+	if distro.Get() == distro.Gokrazy && !strings.Contains(cmd, "/") {
+		return "/user/" + cmd
+	}
+	return cmd
+}
+
+func serveCmd(w http.ResponseWriter, cmd string, args ...string) {
+	log.Printf("Got serveCmd for %q %v", cmd, args)
+	out, err := exec.Command(absify(cmd), args...).CombinedOutput()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if err != nil {
+		w.Header().Set("Exec-Err", err.Error())
+		w.WriteHeader(500)
+		log.Printf("Err on serveCmd for %q %v, %d bytes of output: %v", cmd, args, len(out), err)
+	} else {
+		log.Printf("Did serveCmd for %q %v, %d bytes of output", cmd, args, len(out))
+	}
+	w.Write(out)
+}
+
+type localClientRoundTripper struct {
+	lc tailscale.LocalClient
+}
+
+func (rt *localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.RequestURI = ""
+	return rt.lc.DoLocalRequest(req)
+}
+
+func main() {
+	if distro.Get() == distro.Gokrazy {
+		if !hostinfo.IsNATLabGuestVM() {
+			// "Exiting immediately with status code 0 when the
+			// GOKRAZY_FIRST_START=1 environment variable is set means “don’t
+			// start the program on boot”"
+			return
+		}
+	}
+	flag.Parse()
+
+	if distro.Get() == distro.Gokrazy {
+		nsRx := regexp.MustCompile(`(?m)^nameserver (.*)`)
+		for t := time.Now(); time.Since(t) < 10*time.Second; time.Sleep(10 * time.Millisecond) {
+			all, _ := os.ReadFile("/etc/resolv.conf")
+			if nsRx.Match(all) {
+				break
+			}
+		}
+	}
+
+	logc, err := net.Dial("tcp", "9.9.9.9:124")
+	if err == nil {
+		log.SetOutput(logc)
+	}
+
+	log.Printf("Tailscale Test Agent running.")
+
+	gokRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+	gokRP.Transport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if network != "tcp" {
+				return nil, errors.New("unexpected network")
+			}
+			if addr != "gokrazy:80" {
+				return nil, errors.New("unexpected addr")
+			}
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", "/run/gokrazy-http.sock")
+		},
+	}
+
+	var ttaMux http.ServeMux // agent mux
+	var serveMux http.ServeMux
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("X-TTA-GoKrazy") == "1" {
+			gokRP.ServeHTTP(w, r)
+			return
+		}
+		ttaMux.ServeHTTP(w, r)
+	})
+	var hs http.Server
+	hs.Handler = &serveMux
+	var (
+		stMu   sync.Mutex
+		newSet = set.Set[net.Conn]{} // conns in StateNew
+	)
+	needConnCh := make(chan bool, 1)
+	hs.ConnState = func(c net.Conn, s http.ConnState) {
+		stMu.Lock()
+		defer stMu.Unlock()
+		switch s {
+		case http.StateNew:
+			newSet.Add(c)
+		default:
+			newSet.Delete(c)
+		}
+		if len(newSet) == 0 {
+			select {
+			case needConnCh <- true:
+			default:
+			}
+		}
+	}
+	conns := make(chan net.Conn, 1)
+
+	lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock")))
+	lcRP.Transport = new(localClientRoundTripper)
+	ttaMux.Handle("/localapi/", lcRP)
+
+	ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "TTA\n")
+		return
+	})
+	ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale")
+	})
+	go hs.Serve(chanListener(conns))
+
+	var lastErr string
+	needConnCh <- true
+	for {
+		<-needConnCh
+		c, err := connect()
+		if err != nil {
+			s := err.Error()
+			if s != lastErr {
+				log.Printf("Connect failure: %v", s)
+			}
+			lastErr = s
+			time.Sleep(time.Second)
+			continue
+		}
+		conns <- c
+	}
+}
+
+func connect() (net.Conn, error) {
+	c, err := net.Dial("tcp", *driverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+type chanListener <-chan net.Conn
+
+func (cl chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl
+	if !ok {
+		return nil, errors.New("closed")
+	}
+	return c, nil
+}
+
+func (cl chanListener) Close() error {
+	return nil
+}
+
+func (cl chanListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("52.0.0.34"), // TS..DR(iver)
+		Port: 123,
+	}
+}

cmd/vnet/run-krazy.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "Type 'C-a c' to enter monitor; q to quit."
+
+set -eux
+qemu-system-x86_64 -M microvm,isa-serial=off \
+    -m 1G \
+    -nodefaults -no-user-config -nographic \
+    -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
+    -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1" \
+    -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/tsapp.img,format=raw \
+    -device virtio-blk-device,drive=blk0 \
+    -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \
+    -device virtio-serial-device \
+    -device virtio-net-device,netdev=net0,mac=52:cc:cc:cc:cc:00 \
+    -chardev stdio,id=virtiocon0,mux=on \
+    -device virtconsole,chardev=virtiocon0 \
+    -mon chardev=virtiocon0,mode=readline \
+    -audio none
+

cmd/vnet/vnet-main.go

@@ -0,0 +1,118 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The vnet binary runs a virtual network stack in userspace for qemu instances
+// to connect to and simulate various network conditions.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+)
+
+var (
+	listen  = flag.String("listen", "/tmp/qemu.sock", "path to listen on")
+	nat     = flag.String("nat", "easy", "type of NAT to use")
+	nat2    = flag.String("nat2", "hard", "type of NAT to use for second network")
+	portmap = flag.Bool("portmap", false, "enable portmapping")
+	dgram   = flag.Bool("dgram", false, "enable datagram mode; for use with macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment")
+)
+
+func main() {
+	flag.Parse()
+
+	if _, err := os.Stat(*listen); err == nil {
+		os.Remove(*listen)
+	}
+
+	var srv net.Listener
+	var err error
+	var conn *net.UnixConn
+	if *dgram {
+		addr, err := net.ResolveUnixAddr("unixgram", *listen)
+		if err != nil {
+			log.Fatalf("ResolveUnixAddr: %v", err)
+		}
+		conn, err = net.ListenUnixgram("unixgram", addr)
+		if err != nil {
+			log.Fatalf("ListenUnixgram: %v", err)
+		}
+		defer conn.Close()
+	} else {
+		srv, err = net.Listen("unix", *listen)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var c vnet.Config
+	node1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.NAT(*nat)))
+	c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", vnet.NAT(*nat2)))
+	if *portmap {
+		node1.Network().AddService(vnet.NATPMP)
+	}
+
+	s, err := vnet.New(&c)
+	if err != nil {
+		log.Fatalf("newServer: %v", err)
+	}
+
+	if err := s.PopulateDERPMapIPs(); err != nil {
+		log.Printf("warning: ignoring failure to populate DERP map: %v", err)
+	}
+
+	s.WriteStartingBanner(os.Stdout)
+	nc := s.NodeAgentClient(node1)
+	go func() {
+		rp := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+		d := rp.Director
+		rp.Director = func(r *http.Request) {
+			d(r)
+			r.Header.Set("X-TTA-GoKrazy", "1")
+		}
+		rp.Transport = nc.HTTPClient.Transport
+		http.ListenAndServe(":8080", rp)
+	}()
+	go func() {
+		getStatus := func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			st, err := nc.Status(ctx)
+			if err != nil {
+				log.Printf("NodeStatus: %v", err)
+				return
+			}
+			log.Printf("NodeStatus: %v", logger.AsJSON(st))
+		}
+		for {
+			time.Sleep(5 * time.Second)
+			//continue
+			getStatus()
+		}
+	}()
+
+	if conn != nil {
+		s.ServeUnixConn(conn, vnet.ProtocolUnixDGRAM)
+		return
+	}
+
+	for {
+		c, err := srv.Accept()
+		if err != nil {
+			log.Printf("Accept: %v", err)
+			continue
+		}
+		go s.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+	}
+}

derp/derphttp/derphttp_server.go

@@ -18,6 +18,7 @@ import (
 // following its HTTP request.
 const fastStartHeader = "Derp-Fast-Start"
 
+// Handler returns an http.Handler to be mounted at /derp, serving s.
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// These are installed both here and in cmd/derper. The check here
@@ -79,3 +80,29 @@ func ProbeHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
 	}
 }
+
+// ServeNoContent generates the /generate_204 response used by Tailscale's
+// captive portal detection.
+func ServeNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(NoContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(NoContentResponseHeader, "response "+challenge)
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
+const (
+	NoContentChallengeHeader = "X-Tailscale-Challenge"
+	NoContentResponseHeader  = "X-Tailscale-Response"
+)

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/golangci/golangci-lint v1.52.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.18.0
+	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
 	github.com/goreleaser/nfpm/v2 v2.33.1
@@ -198,7 +199,7 @@ require (
 	github.com/denis-tingaikin/go-header v0.4.3 // indirect
 	github.com/docker/cli v25.0.0+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/docker v26.1.4+incompatible // indirect
+	github.com/docker/docker v26.1.5+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect

go.sum

@@ -262,8 +262,8 @@ github.com/docker/cli v25.0.0+incompatible h1:zaimaQdnX7fYWFqzN88exE9LDEvRslexpF
 github.com/docker/cli v25.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU=
-github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g=
+github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo=
 github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@@ -477,6 +477,8 @@ github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=

go.toolchain.rev

@@ -1 +1 @@
-2f152a4eff5875655a9a84fce8f8d329f8d9a321
+22ef9eb38e9a2d21b4a45f7adc75addb05f3efb8

gokrazy/Makefile

@@ -6,3 +6,6 @@ image:
 
 qemu: image
 	qemu-system-x86_64 -m 1G -drive file=tsapp.img,format=raw -boot d -netdev user,id=user.0 -device virtio-net-pci,netdev=user.0 -serial mon:stdio -audio none
+
+qcow2: image
+	qemu-img convert -O qcow2 tsapp.img tsapp.qcow2

gokrazy/go.mod

@@ -2,12 +2,12 @@ module tailscale.com/gokrazy
 
 go 1.22
 
-require github.com/gokrazy/tools v0.0.0-20240510170341-34b02e215bc2
+require github.com/gokrazy/tools v0.0.0-20240730192548-9f81add3a91e
 
 require (
 	github.com/breml/rootcerts v0.2.10 // indirect
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 // indirect
-	github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
 	github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 // indirect
 	github.com/google/renameio/v2 v2.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -20,6 +20,4 @@ require (
 
 replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
 
-replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf
-
-replace github.com/gokrazy/internal => github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd
+replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e

gokrazy/go.sum

@@ -3,6 +3,8 @@ github.com/breml/rootcerts v0.2.10/go.mod h1:24FDtzYMpqIeYC7QzaE8VPRQaFZU5TIUDly
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 h1:kBY5R1tSf+EYZ+QaSrofLaVJtBqYsVNVBWkdMq3Smcg=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2/go.mod h1:PYOvzGOL4nlBmuxu7IyKQTFLaxr61+WPRNRzVtuYOHw=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
@@ -17,10 +19,8 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd h1:ZJplHHhYSzxYmrXuDPCNChGRZbLkPqRkYqRBM7KNyng=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf h1:lmAGqLbIVoMK1TYWqJvxKFsu+Tb1OecgvXTmypZGAZY=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf/go.mod h1:+PSix9a8BHqAz6RV/9+tiE3C1ou0GA1ViR8pqAZVfwI=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e h1:3/xIc1QCvnKL7BCLng9od98HEvxCadjvqiI/bN+Twso=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e/go.mod h1:eTZ0QsugEPFU5UAQ/87bKMkPxQuTNa7+iFAIahOFwRg=
 golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.mod

@@ -2,6 +2,14 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect
+require (
+	github.com/gokrazy/gokrazy v0.0.0-20240802144848-676865a4e84f // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/kenshaw/evdev v0.1.0 // indirect
+	github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
 
-replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
+replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.sum

@@ -2,6 +2,8 @@ github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZ
 github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a h1:FKeN678rNpKTpWRdFbAhYL9mWzPu57R5XPXCR3WmXdI=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/kenshaw/evdev v0.1.0 h1:wmtceEOFfilChgdNT+c/djPJ2JineVsQ0N14kGzFRUo=
@@ -12,5 +14,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a h1:7dnA8x14JihQmKbPr++Y5CCN/XSyDmOB6cXUxcIj6VQ=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f h1:ZSAGWpgs+6dK2oIz5OR+HUul3oJbnhFn8YNgcZ3d9SQ=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+golang.org/x/sys v0.0.0-20201005065044-765f4ea38db3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod

@@ -2,4 +2,4 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 // indirect
+require github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e // indirect

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.sum

@@ -1,2 +1,4 @@
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 h1:xzf+cMvBJBcA/Av7OTWBa0Tjrbfcy00TeatJeJt6zrY=
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e h1:tyUUgeRPGHjCZWycRnhdx8Lx9DRkjl3WsVUxYMrVBOw=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=

gokrazy/tsapp/builddir/tailscale.com/go.sum

@@ -122,8 +122,12 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1 h1:ycpNCSYwzZ7x4G4ioPNtKQmIY0G/3o4pVf8wCZq6blY=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98 h1:RNpJrXfI5u6e+uzyIzvmnXbhmhdRkVf//90sMBH3lso=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -170,6 +174,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3 h1:/8/t5pz/mgdRXhYOIeqqYhFAQLE4DDGegc0Y4ZjyFJM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=

gokrazy/tsapp/config.json

@@ -1,16 +1,21 @@
 {
     "Hostname": "tsapp",
-    "Update": { "NoPassword": true },
+    "Update": {
+        "NoPassword": true
+    },
     "SerialConsole": "ttyS0,115200",
     "Packages": [
         "github.com/gokrazy/serial-busybox",
         "github.com/gokrazy/breakglass",
         "tailscale.com/cmd/tailscale",
-        "tailscale.com/cmd/tailscaled"
+        "tailscale.com/cmd/tailscaled",
+        "tailscale.com/cmd/tta"
     ],
     "PackageConfig": {
         "github.com/gokrazy/breakglass": {
-            "CommandLineFlags": [ "-authorized_keys=ec2" ]
+            "CommandLineFlags": [
+                "-authorized_keys=ec2"
+            ]
         },
         "tailscale.com/cmd/tailscale": {
             "ExtraFilePaths": {
@@ -21,4 +26,4 @@
     "KernelPackage": "github.com/tailscale/gokrazy-kernel",
     "FirmwarePackage": "github.com/tailscale/gokrazy-kernel",
     "InternalCompatibilityFlags": {}
-}
+}

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/lineread"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 )
 
 var started = time.Now()
@@ -462,3 +463,15 @@ func IsSELinuxEnforcing() bool {
 	out, _ := exec.Command("getenforce").Output()
 	return string(bytes.TrimSpace(out)) == "Enforcing"
 }
+
+// IsNATLabGuestVM reports whether the current host is a NAT Lab guest VM.
+func IsNATLabGuestVM() bool {
+	if runtime.GOOS == "linux" && distro.Get() == distro.Gokrazy {
+		cmdLine, _ := os.ReadFile("/proc/cmdline")
+		return bytes.Contains(cmdLine, []byte("tailscale-tta=1"))
+	}
+	return false
+}
+
+// NAT Lab VMs have a unique MAC address prefix.
+// See

ipn/ipnlocal/local.go

@@ -3781,7 +3781,7 @@ func (b *LocalBackend) TCPHandlerForDst(src, dst netip.AddrPort) (handler func(c
 			return nil
 		}, opts
 	}
-	if handler := b.tcpHandlerForServe(dst.Port(), src); handler != nil {
+	if handler := b.tcpHandlerForServe(dst.Port(), src, nil); handler != nil {
 		return handler, opts
 	}
 	return nil, nil

ipn/ipnlocal/serve.go

@@ -56,6 +56,16 @@ var serveHTTPContextKey ctxkey.Key[*serveHTTPContext]
 type serveHTTPContext struct {
 	SrcAddr  netip.AddrPort
 	DestPort uint16
+
+	// provides funnel-specific context, nil if not funneled
+	Funnel *funnelFlow
+}
+
+// funnelFlow represents a funneled connection initiated via IngressPeer
+// to Host.
+type funnelFlow struct {
+	Host        string
+	IngressPeer tailcfg.NodeView
 }
 
 // localListener is the state of host-level net.Listen for a specific (Tailscale IP, port)
@@ -91,7 +101,7 @@ func (b *LocalBackend) newServeListener(ctx context.Context, ap netip.AddrPort,
 
 		handler: func(conn net.Conn) error {
 			srcAddr := conn.RemoteAddr().(*net.TCPAddr).AddrPort()
-			handler := b.tcpHandlerForServe(ap.Port(), srcAddr)
+			handler := b.tcpHandlerForServe(ap.Port(), srcAddr, nil)
 			if handler == nil {
 				b.logf("[unexpected] local-serve: no handler for %v to port %v", srcAddr, ap.Port())
 				conn.Close()
@@ -382,7 +392,7 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 		return
 	}
 
-	_, port, err := net.SplitHostPort(string(target))
+	host, port, err := net.SplitHostPort(string(target))
 	if err != nil {
 		logf("got ingress conn for bad target %q; rejecting", target)
 		sendRST()
@@ -407,9 +417,10 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 			return
 		}
 	}
-	// TODO(bradfitz): pass ingressPeer etc in context to tcpHandlerForServe,
-	// extend serveHTTPContext or similar.
-	handler := b.tcpHandlerForServe(dport, srcAddr)
+	handler := b.tcpHandlerForServe(dport, srcAddr, &funnelFlow{
+		Host:        host,
+		IngressPeer: ingressPeer,
+	})
 	if handler == nil {
 		logf("[unexpected] no matching ingress serve handler for %v to port %v", srcAddr, dport)
 		sendRST()
@@ -424,8 +435,9 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 }
 
 // tcpHandlerForServe returns a handler for a TCP connection to be served via
-// the ipn.ServeConfig.
-func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort) (handler func(net.Conn) error) {
+// the ipn.ServeConfig. The funnelFlow can be nil if this is not a funneled
+// connection.
+func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort, f *funnelFlow) (handler func(net.Conn) error) {
 	b.mu.Lock()
 	sc := b.serveConfig
 	b.mu.Unlock()
@@ -444,6 +456,7 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort)
 			Handler: http.HandlerFunc(b.serveWebHandler),
 			BaseContext: func(_ net.Listener) context.Context {
 				return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{
+					Funnel:   f,
 					SrcAddr:  srcAddr,
 					DestPort: dport,
 				})
@@ -712,15 +725,20 @@ func (b *LocalBackend) addTailscaleIdentityHeaders(r *httputil.ProxyRequest) {
 	r.Out.Header.Del("Tailscale-User-Login")
 	r.Out.Header.Del("Tailscale-User-Name")
 	r.Out.Header.Del("Tailscale-User-Profile-Pic")
+	r.Out.Header.Del("Tailscale-Funnel-Request")
 	r.Out.Header.Del("Tailscale-Headers-Info")
 
 	c, ok := serveHTTPContextKey.ValueOk(r.Out.Context())
 	if !ok {
 		return
 	}
+	if c.Funnel != nil {
+		r.Out.Header.Set("Tailscale-Funnel-Request", "?1")
+		return
+	}
 	node, user, ok := b.WhoIs("tcp", c.SrcAddr)
 	if !ok {
-		return // traffic from outside of Tailnet (funneled)
+		return // traffic from outside of Tailnet (funneled or local machine)
 	}
 	if node.IsTagged() {
 		// 2023-06-14: Not setting identity headers for tagged nodes.

logpolicy/logpolicy.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/envknob"
 	"tailscale.com/health"
+	"tailscale.com/hostinfo"
 	"tailscale.com/log/filelogger"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
@@ -566,7 +567,7 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor,
 		conf.IncludeProcSequence = true
 	}
 
-	if envknob.NoLogsNoSupport() || testenv.InTest() {
+	if envknob.NoLogsNoSupport() || testenv.InTest() || hostinfo.IsNATLabGuestVM() {
 		logf("You have disabled logging. Tailscale will not be able to provide support.")
 		conf.HTTPC = &http.Client{Transport: noopPretendSuccessTransport{}}
 	} else if val := getLogTarget(); val != "" {

net/dns/resolver/forwarder.go

@@ -6,6 +6,8 @@ package resolver
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -498,9 +500,10 @@ var (
 func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDelay) (ret []byte, err error) {
 	if verboseDNSForward() {
 		id := forwarderCount.Add(1)
-		f.logf("forwarder.send(%q) [%d] ...", rr.name.Addr, id)
+		domain, typ, _ := nameFromQuery(fq.packet)
+		f.logf("forwarder.send(%q, %d, %v, %d) [%d] ...", rr.name.Addr, fq.txid, typ, len(domain), id)
 		defer func() {
-			f.logf("forwarder.send(%q) [%d] = %v, %v", rr.name.Addr, id, len(ret), err)
+			f.logf("forwarder.send(%q, %d, %v, %d) [%d] = %v, %v", rr.name.Addr, fq.txid, typ, len(domain), id, len(ret), err)
 		}()
 	}
 	if strings.HasPrefix(rr.name.Addr, "http://") {
@@ -866,7 +869,7 @@ type forwardQuery struct {
 // node DNS proxy queries), otherwise f.resolvers is used.
 func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, responseChan chan<- packet, resolvers ...resolverAndDelay) error {
 	metricDNSFwd.Add(1)
-	domain, err := nameFromQuery(query.bs)
+	domain, typ, err := nameFromQuery(query.bs)
 	if err != nil {
 		metricDNSFwdErrorName.Add(1)
 		return err
@@ -943,6 +946,12 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 	}
 	defer fq.closeOnCtxDone.Close()
 
+	if verboseDNSForward() {
+		domainSha256 := sha256.Sum256([]byte(domain))
+		domainSig := base64.RawStdEncoding.EncodeToString(domainSha256[:3])
+		f.logf("request(%d, %v, %d, %s) %d...", fq.txid, typ, len(domain), domainSig, len(fq.packet))
+	}
+
 	resc := make(chan []byte, 1) // it's fine buffered or not
 	errc := make(chan error, 1)  // it's fine buffered or not too
 	for i := range resolvers {
@@ -982,6 +991,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 				metricDNSFwdErrorContext.Add(1)
 				return fmt.Errorf("waiting to send response: %w", ctx.Err())
 			case responseChan <- packet{v, query.family, query.addr}:
+				if verboseDNSForward() {
+					f.logf("response(%d, %v, %d) = %d, nil", fq.txid, typ, len(domain), len(v))
+				}
 				metricDNSFwdSuccess.Add(1)
 				f.health.SetHealthy(dnsForwarderFailing)
 				return nil
@@ -1009,6 +1021,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 						}
 						f.health.SetUnhealthy(dnsForwarderFailing, health.Args{health.ArgDNSServers: strings.Join(resolverAddrs, ",")})
 					case responseChan <- res:
+						if verboseDNSForward() {
+							f.logf("forwarder response(%d, %v, %d) = %d, %v", fq.txid, typ, len(domain), len(res.bs), firstErr)
+						}
 					}
 				}
 				return firstErr
@@ -1037,24 +1052,28 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 var initListenConfig func(_ *net.ListenConfig, _ *netmon.Monitor, tunName string) error
 
 // nameFromQuery extracts the normalized query name from bs.
-func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
+func nameFromQuery(bs []byte) (dnsname.FQDN, dns.Type, error) {
 	var parser dns.Parser
 
 	hdr, err := parser.Start(bs)
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
 	if hdr.Response {
-		return "", errNotQuery
+		return "", 0, errNotQuery
 	}
 
 	q, err := parser.Question()
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
 
 	n := q.Name.Data[:q.Name.Length]
-	return dnsname.ToFQDN(rawNameToLower(n))
+	fqdn, err := dnsname.ToFQDN(rawNameToLower(n))
+	if err != nil {
+		return "", 0, err
+	}
+	return fqdn, q.Type, nil
 }
 
 // nxDomainResponse returns an NXDomain DNS reply for the provided request.

net/dns/resolver/forwarder_test.go

@@ -201,7 +201,7 @@ func BenchmarkNameFromQuery(b *testing.B) {
 	b.ResetTimer()
 	b.ReportAllocs()
 	for range b.N {
-		_, err := nameFromQuery(msg)
+		_, _, err := nameFromQuery(msg)
 		if err != nil {
 			b.Fatal(err)
 		}

prober/prober.go

@@ -496,7 +496,8 @@ func (p *Prober) RunHandler(w http.ResponseWriter, r *http.Request) error {
 		return nil
 	}
 
-	stats := fmt.Sprintf("Previous runs: success rate %d%%, median latency %v",
+	stats := fmt.Sprintf("Last %d probes: success rate %d%%, median latency %v\n",
+		len(prevInfo.RecentResults),
 		int(prevInfo.RecentSuccessRatio()*100), prevInfo.RecentMedianLatency())
 	if err != nil {
 		return tsweb.Error(respStatus, fmt.Sprintf("Probe failed: %s\n%s", err.Error(), stats), err)

prober/status.go

@@ -86,7 +86,7 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc
 			}
 			s := probeStatus{ProbeInfo: info}
 			if !info.End.IsZero() {
-				s.TimeSinceLast = time.Since(info.End)
+				s.TimeSinceLast = time.Since(info.End).Truncate(time.Second)
 			}
 			for textTpl, urlTpl := range params.probeLinks {
 				text, err := renderTemplate(textTpl, info)

prober/status.html

@@ -71,12 +71,12 @@
     <table class="sortable">
         <thead><tr>
             <th>Name</th>
-            <th>Class & Labels</th>
+            <th>Probe Class & Labels</th>
             <th>Interval</th>
-            <th>Result</th>
+            <th>Last Attempt</th>
             <th>Success</th>
             <th>Latency</th>
-            <th>Error</th>
+            <th>Last Error</th>
         </tr></thead>
         <tbody>
         {{range $name, $probeInfo := .Probes}}
@@ -100,8 +100,8 @@
             <td>{{$probeInfo.Interval}}</td>
             <td data-sort="{{$probeInfo.TimeSinceLast.Milliseconds}}">
                 {{if $probeInfo.TimeSinceLast}}
-                    {{$probeInfo.TimeSinceLast.String}}<br/>
-                    <span class="small">{{$probeInfo.End}}</span>
+                    {{$probeInfo.TimeSinceLast.String}} ago<br/>
+                    <span class="small">{{$probeInfo.End.Format "2006-01-02T15:04:05Z07:00"}}</span>
                 {{else}}
                     Never
                 {{end}}

tstest/integration/integration.go

@@ -190,6 +190,7 @@ func RunDERPAndSTUN(t testing.TB, logf logger.Logf, ipAddress string) (derpMap *
 	}
 
 	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Listener.Close()
 	httpsrv.Listener = ln
 	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
 	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))

tstest/integration/nat/nat_test.go

@@ -0,0 +1,528 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package nat
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/mod/modfile"
+	"golang.org/x/sync/errgroup"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/syncs"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+var (
+	logTailscaled = flag.Bool("log-tailscaled", false, "log tailscaled output")
+	pcapFile      = flag.String("pcap", "", "write pcap to file")
+)
+
+type natTest struct {
+	tb      testing.TB
+	base    string // base image
+	tempDir string // for qcow2 images
+	vnet    *vnet.Server
+	kernel  string // linux kernel path
+}
+
+func newNatTest(tb testing.TB) *natTest {
+	root, err := os.Getwd()
+	if err != nil {
+		tb.Fatal(err)
+	}
+	modRoot := filepath.Join(root, "../../..")
+
+	nt := &natTest{
+		tb:      tb,
+		tempDir: tb.TempDir(),
+		base:    filepath.Join(modRoot, "gokrazy/tsapp.qcow2"),
+	}
+
+	if _, err := os.Stat(nt.base); err != nil {
+		tb.Skipf("skipping test; base image %q not found", nt.base)
+	}
+
+	nt.kernel, err = findKernelPath(filepath.Join(modRoot, "gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod"))
+	if err != nil {
+		tb.Skipf("skipping test; kernel not found: %v", err)
+	}
+	tb.Logf("found kernel: %v", nt.kernel)
+
+	return nt
+}
+
+func findKernelPath(goMod string) (string, error) {
+	b, err := os.ReadFile(goMod)
+	if err != nil {
+		return "", err
+	}
+	mf, err := modfile.Parse("go.mod", b, nil)
+	if err != nil {
+		return "", err
+	}
+	goModB, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	for _, r := range mf.Require {
+		if r.Mod.Path == "github.com/tailscale/gokrazy-kernel" {
+			return strings.TrimSpace(string(goModB)) + "/" + r.Mod.String() + "/vmlinuz", nil
+		}
+	}
+	return "", fmt.Errorf("failed to find kernel in %v", goMod)
+}
+
+type addNodeFunc func(c *vnet.Config) *vnet.Node // returns nil to omit test
+
+func easy(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT))
+}
+
+func easyAF(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyAFNAT))
+}
+
+func sameLAN(c *vnet.Config) *vnet.Node {
+	nw := c.FirstNetwork()
+	if nw == nil {
+		return nil
+	}
+	if !nw.CanTakeMoreNodes() {
+		return nil
+	}
+	return c.AddNode(nw)
+}
+
+func one2one(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("172.16.%d.1/24", n), vnet.One2OneNAT))
+}
+
+func easyPMP(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
+}
+
+func hard(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("10.0.%d.1/24", n), vnet.HardNAT))
+}
+
+func hardPMP(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("10.7.%d.1/24", n), vnet.HardNAT, vnet.NATPMP))
+}
+
+func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute {
+	t := nt.tb
+
+	var c vnet.Config
+	c.SetPCAPFile(*pcapFile)
+	nodes := []*vnet.Node{
+		node1(&c),
+		node2(&c),
+	}
+	if nodes[0] == nil || nodes[1] == nil {
+		t.Skip("skipping test; not applicable combination")
+	}
+
+	var err error
+	nt.vnet, err = vnet.New(&c)
+	if err != nil {
+		t.Fatalf("newServer: %v", err)
+	}
+	nt.tb.Cleanup(func() {
+		nt.vnet.Close()
+	})
+
+	var wg sync.WaitGroup // waiting for srv.Accept goroutine
+	defer wg.Wait()
+
+	sockAddr := filepath.Join(nt.tempDir, "qemu.sock")
+	srv, err := net.Listen("unix", sockAddr)
+	if err != nil {
+		t.Fatalf("Listen: %v", err)
+	}
+	defer srv.Close()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			c, err := srv.Accept()
+			if err != nil {
+				return
+			}
+			go nt.vnet.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+		}
+	}()
+
+	for i, node := range nodes {
+		disk := fmt.Sprintf("%s/node-%d.qcow2", nt.tempDir, i)
+		out, err := exec.Command("qemu-img", "create",
+			"-f", "qcow2",
+			"-F", "qcow2",
+			"-b", nt.base,
+			disk).CombinedOutput()
+		if err != nil {
+			t.Fatalf("qemu-img create: %v, %s", err, out)
+		}
+
+		cmd := exec.Command("qemu-system-x86_64",
+			"-M", "microvm,isa-serial=off",
+			"-m", "384M",
+			"-nodefaults", "-no-user-config", "-nographic",
+			"-kernel", nt.kernel,
+			"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1",
+			"-drive", "id=blk0,file="+disk+",format=qcow2",
+			"-device", "virtio-blk-device,drive=blk0",
+			"-netdev", "stream,id=net0,addr.type=unix,addr.path="+sockAddr,
+			"-device", "virtio-serial-device",
+			"-device", "virtio-net-device,netdev=net0,mac="+node.MAC().String(),
+			"-chardev", "stdio,id=virtiocon0,mux=on",
+			"-device", "virtconsole,chardev=virtiocon0",
+			"-mon", "chardev=virtiocon0,mode=readline",
+			"-audio", "none",
+		)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Start(); err != nil {
+			t.Fatalf("qemu: %v", err)
+		}
+		nt.tb.Cleanup(func() {
+			cmd.Process.Kill()
+			cmd.Wait()
+		})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	lc1 := nt.vnet.NodeAgentClient(nodes[0])
+	lc2 := nt.vnet.NodeAgentClient(nodes[1])
+	clients := []*vnet.NodeAgentClient{lc1, lc2}
+
+	var eg errgroup.Group
+	var sts [2]*ipnstate.Status
+	for i, c := range clients {
+		i, c := i, c
+		eg.Go(func() error {
+			if *logTailscaled {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					streamDaemonLogs(ctx, t, c, fmt.Sprintf("node%d:", i))
+				}()
+			}
+			st, err := c.Status(ctx)
+			if err != nil {
+				return fmt.Errorf("node%d status: %w", i, err)
+			}
+			t.Logf("node%d status: %v", i, st)
+			if err := up(ctx, c); err != nil {
+				return fmt.Errorf("node%d up: %w", i, err)
+			}
+			t.Logf("node%d up!", i)
+			st, err = c.Status(ctx)
+			if err != nil {
+				return fmt.Errorf("node%d status: %w", i, err)
+			}
+			sts[i] = st
+
+			if st.BackendState != "Running" {
+				return fmt.Errorf("node%d state = %q", i, st.BackendState)
+			}
+			t.Logf("node%d up with %v", i, sts[i].Self.TailscaleIPs)
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		t.Fatalf("initial setup: %v", err)
+	}
+
+	defer nt.vnet.Close()
+
+	pingRes, err := ping(ctx, lc1, sts[1].Self.TailscaleIPs[0])
+	if err != nil {
+		t.Fatalf("ping failure: %v", err)
+	}
+	route := classifyPing(pingRes)
+	t.Logf("ping route: %v", route)
+	return route
+}
+
+func classifyPing(pr *ipnstate.PingResult) pingRoute {
+	if pr == nil {
+		return routeNil
+	}
+	if pr.Endpoint != "" {
+		ap, err := netip.ParseAddrPort(pr.Endpoint)
+		if err == nil {
+			if ap.Addr().IsPrivate() {
+				return routeLocal
+			}
+			return routeDirect
+		}
+	}
+	return routeDERP // presumably
+}
+
+type pingRoute string
+
+const (
+	routeDERP   pingRoute = "derp"
+	routeLocal  pingRoute = "local"
+	routeDirect pingRoute = "direct"
+	routeNil    pingRoute = "nil" // *ipnstate.PingResult is nil
+)
+
+func streamDaemonLogs(ctx context.Context, t testing.TB, c *vnet.NodeAgentClient, nodeID string) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	r, err := c.TailDaemonLogs(ctx)
+	if err != nil {
+		t.Errorf("tailDaemonLogs: %v", err)
+		return
+	}
+	logger := log.New(os.Stderr, nodeID+" ", log.Lmsgprefix)
+	dec := json.NewDecoder(r)
+	for {
+		// /{"logtail":{"client_time":"2024-08-08T17:42:31.95095956Z","proc_id":2024742977,"proc_seq":232},"text":"magicsock: derp-1 connected; connGen=1\n"}
+		var logEntry struct {
+			LogTail struct {
+				ClientTime time.Time `json:"client_time"`
+			}
+			Text string `json:"text"`
+		}
+		if err := dec.Decode(&logEntry); err != nil {
+			if err == io.EOF || errors.Is(err, context.Canceled) {
+				return
+			}
+			t.Errorf("log entry: %v", err)
+			return
+		}
+		logger.Printf("%s %s", logEntry.LogTail.ClientTime.Format("2006/01/02 15:04:05"), logEntry.Text)
+	}
+}
+
+func ping(ctx context.Context, c *vnet.NodeAgentClient, target netip.Addr) (*ipnstate.PingResult, error) {
+	n := 0
+	var res *ipnstate.PingResult
+	anyPong := false
+	for n < 10 {
+		n++
+		pr, err := c.PingWithOpts(ctx, target, tailcfg.PingDisco, tailscale.PingOpts{})
+		if err != nil {
+			if anyPong {
+				return res, nil
+			}
+			return nil, err
+		}
+		if pr.Err != "" {
+			return nil, errors.New(pr.Err)
+		}
+		if pr.DERPRegionID == 0 {
+			return pr, nil
+		}
+		res = pr
+		select {
+		case <-ctx.Done():
+		case <-time.After(time.Second):
+		}
+	}
+	if res == nil {
+		return nil, errors.New("no ping response")
+	}
+	return res, nil
+}
+
+func up(ctx context.Context, c *vnet.NodeAgentClient) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://unused/up", nil)
+	if err != nil {
+		return err
+	}
+	res, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	all, _ := io.ReadAll(res.Body)
+	if res.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code %v: %s", res.Status, all)
+	}
+	return nil
+}
+
+type nodeType struct {
+	name string
+	fn   addNodeFunc
+}
+
+var types = []nodeType{
+	{"easy", easy},
+	{"easyAF", easyAF},
+	{"hard", hard},
+	{"easyPMP", easyPMP},
+	{"hardPMP", hardPMP},
+	{"one2one", one2one},
+	{"sameLAN", sameLAN},
+}
+
+func TestEasyEasy(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, easy)
+}
+
+var pair = flag.String("pair", "", "comma-separated pair of types to test (easy, easyAF, hard, easyPMP, hardPMP, one2one, sameLAN)")
+
+func TestPair(t *testing.T) {
+	t1, t2, ok := strings.Cut(*pair, ",")
+	if !ok {
+		t.Skipf("skipping test without --pair=type1,type2 set")
+	}
+	find := func(name string) addNodeFunc {
+		for _, nt := range types {
+			if nt.name == name {
+				return nt.fn
+			}
+		}
+		t.Fatalf("unknown type %q", name)
+		return nil
+	}
+
+	nt := newNatTest(t)
+	nt.runTest(find(t1), find(t2))
+}
+
+var runGrid = flag.Bool("run-grid", false, "run grid test")
+
+func TestGrid(t *testing.T) {
+	if !*runGrid {
+		t.Skip("skipping grid test; set --run-grid to run")
+	}
+	t.Parallel()
+
+	sem := syncs.NewSemaphore(2)
+	var (
+		mu  sync.Mutex
+		res = make(map[string]pingRoute)
+	)
+	for _, a := range types {
+		for _, b := range types {
+			key := a.name + "-" + b.name
+			keyBack := b.name + "-" + a.name
+			t.Run(key, func(t *testing.T) {
+				t.Parallel()
+
+				sem.Acquire()
+				defer sem.Release()
+
+				filename := key + ".cache"
+				contents, _ := os.ReadFile(filename)
+				if len(contents) == 0 {
+					filename2 := keyBack + ".cache"
+					contents, _ = os.ReadFile(filename2)
+				}
+				route := pingRoute(strings.TrimSpace(string(contents)))
+
+				if route == "" {
+					nt := newNatTest(t)
+					route = nt.runTest(a.fn, b.fn)
+					if err := os.WriteFile(filename, []byte(string(route)), 0666); err != nil {
+						t.Fatalf("writeFile: %v", err)
+					}
+				}
+
+				mu.Lock()
+				defer mu.Unlock()
+				res[key] = route
+				t.Logf("results: %v", res)
+			})
+		}
+	}
+
+	t.Cleanup(func() {
+		mu.Lock()
+		defer mu.Unlock()
+		var hb bytes.Buffer
+		pf := func(format string, args ...any) {
+			fmt.Fprintf(&hb, format, args...)
+		}
+		rewrite := func(s string) string {
+			return strings.ReplaceAll(s, "PMP", "+pm")
+		}
+		pf("<html><table border=1 cellpadding=5>")
+		pf("<tr><td></td>")
+		for _, a := range types {
+			pf("<td><b>%s</b></td>", rewrite(a.name))
+		}
+		pf("</tr>\n")
+
+		for _, a := range types {
+			if a.name == "sameLAN" {
+				continue
+			}
+			pf("<tr><td><b>%s</b></td>", rewrite(a.name))
+			for _, b := range types {
+				key := a.name + "-" + b.name
+				key2 := b.name + "-" + a.name
+				v := cmp.Or(res[key], res[key2], "-")
+				if v == "derp" {
+					pf("<td><div style='color: red; font-weight: bold'>%s</div></td>", v)
+				} else if v == "local" {
+					pf("<td><div style='color: green; font-weight: bold'>%s</div></td>", v)
+				} else {
+					pf("<td>%s</td>", v)
+				}
+			}
+			pf("</tr>\n")
+		}
+		pf("</table>")
+		pf("<b>easy</b>: Endpoint-Independent Mapping, Address and Port-Dependent Filtering (e.g. Linux, Google Wifi, Unifi, eero)<br>")
+		pf("<b>easyAF</b>: Endpoint-Independent Mapping, Address-Dependent Filtering (James says telephony things or Zyxel type things)<br>")
+		pf("<b>hard</b>: Address and Port-Dependent Mapping, Address and Port-Dependent Filtering (FreeBSD, OPNSense, pfSense)<br>")
+		pf("<b>one2one</b>: One-to-One NAT (e.g. an EC2 instance with a public IPv4)<br>")
+		pf("<b>x+pm</b>: x, with port mapping (NAT-PMP, PCP, UPnP, etc)<br>")
+		pf("<b>sameLAN</b>: a second node in the same LAN as the first<br>")
+		pf("</html>")
+
+		if err := os.WriteFile("grid.html", hb.Bytes(), 0666); err != nil {
+			t.Fatalf("writeFile: %v", err)
+		}
+	})
+}

tstest/natlab/vnet/conf.go

@@ -0,0 +1,273 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"cmp"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"slices"
+
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcapgo"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/set"
+)
+
+// Note: the exported Node and Network are the configuration types;
+// the unexported node and network are the runtime types that are actually
+// used once the server is created.
+
+// Config is the requested state of the natlab virtual network.
+//
+// The zero value is a valid empty configuration. Call AddNode
+// and AddNetwork to methods on the returned Node and Network
+// values to modify the config before calling NewServer.
+// Once the NewServer is called, Config is no longer used.
+type Config struct {
+	nodes    []*Node
+	networks []*Network
+	pcapFile string
+}
+
+func (c *Config) SetPCAPFile(file string) {
+	c.pcapFile = file
+}
+
+func (c *Config) NumNodes() int {
+	return len(c.nodes)
+}
+
+func (c *Config) FirstNetwork() *Network {
+	if len(c.networks) == 0 {
+		return nil
+	}
+	return c.networks[0]
+}
+
+// AddNode creates a new node in the world.
+//
+// The opts may be of the following types:
+//   - *Network: zero, one, or more networks to add this node to
+//   - TODO: more
+//
+// On an error or unknown opt type, AddNode returns a
+// node with a carried error that gets returned later.
+func (c *Config) AddNode(opts ...any) *Node {
+	num := len(c.nodes)
+	n := &Node{
+		mac: MAC{0x52, 0xcc, 0xcc, 0xcc, 0xcc, byte(num)}, // 52=TS then 0xcc for ccclient
+	}
+	c.nodes = append(c.nodes, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case *Network:
+			if !slices.Contains(o.nodes, n) {
+				o.nodes = append(o.nodes, n)
+			}
+			n.nets = append(n.nets, o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNode option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// AddNetwork add a new network.
+//
+// The opts may be of the following types:
+//   - string IP address, for the network's WAN IP (if any)
+//   - string netip.Prefix, for the network's LAN IP (defaults to 192.168.0.0/24)
+//   - NAT, the type of NAT to use
+//   - NetworkService, a service to add to the network
+//
+// On an error or unknown opt type, AddNetwork returns a
+// network with a carried error that gets returned later.
+func (c *Config) AddNetwork(opts ...any) *Network {
+	num := len(c.networks)
+	n := &Network{
+		mac: MAC{0x52, 0xee, 0xee, 0xee, 0xee, byte(num)}, // 52=TS then 0xee for 'etwork
+	}
+	c.networks = append(c.networks, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case string:
+			if ip, err := netip.ParseAddr(o); err == nil {
+				n.wanIP = ip
+			} else if ip, err := netip.ParsePrefix(o); err == nil {
+				n.lanIP = ip
+			} else {
+				if n.err == nil {
+					n.err = fmt.Errorf("unknown string option %q", o)
+				}
+			}
+		case NAT:
+			n.natType = o
+		case NetworkService:
+			n.AddService(o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNetwork option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// Node is the configuration of a node in the virtual network.
+type Node struct {
+	err error
+	n   *node // nil until NewServer called
+
+	// TODO(bradfitz): this is halfway converted to supporting multiple NICs
+	// but not done. We need a MAC-per-Network.
+
+	mac  MAC
+	nets []*Network
+}
+
+// MAC returns the MAC address of the node.
+func (n *Node) MAC() MAC {
+	return n.mac
+}
+
+// Network returns the first network this node is connected to,
+// or nil if none.
+func (n *Node) Network() *Network {
+	if len(n.nets) == 0 {
+		return nil
+	}
+	return n.nets[0]
+}
+
+// Network is the configuration of a network in the virtual network.
+type Network struct {
+	mac     MAC // MAC address of the router/gateway
+	natType NAT
+
+	wanIP netip.Addr
+	lanIP netip.Prefix
+	nodes []*Node
+
+	svcs set.Set[NetworkService]
+
+	// ...
+	err error // carried error
+}
+
+func (n *Network) CanTakeMoreNodes() bool {
+	if n.natType == One2OneNAT {
+		return len(n.nodes) == 0
+	}
+	return len(n.nodes) < 150
+}
+
+// NetworkService is a service that can be added to a network.
+type NetworkService string
+
+const (
+	NATPMP NetworkService = "NAT-PMP"
+	PCP    NetworkService = "PCP"
+	UPnP   NetworkService = "UPnP"
+)
+
+// AddService adds a network service (such as port mapping protocols) to a
+// network.
+func (n *Network) AddService(s NetworkService) {
+	if n.svcs == nil {
+		n.svcs = set.Of(s)
+	} else {
+		n.svcs.Add(s)
+	}
+}
+
+// initFromConfig initializes the server from the previous calls
+// to NewNode and NewNetwork and returns an error if
+// there were any configuration issues.
+func (s *Server) initFromConfig(c *Config) error {
+	netOfConf := map[*Network]*network{}
+	if c.pcapFile != "" {
+		pcf, err := os.OpenFile(c.pcapFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		nw, err := pcapgo.NewNgWriter(pcf, layers.LinkTypeEthernet)
+		if err != nil {
+			return err
+		}
+		pw := &pcapWriter{
+			f: pcf,
+			w: nw,
+		}
+		s.pcapWriter = pw
+	}
+	for _, conf := range c.networks {
+		if conf.err != nil {
+			return conf.err
+		}
+		if !conf.lanIP.IsValid() {
+			conf.lanIP = netip.MustParsePrefix("192.168.0.0/24")
+		}
+		n := &network{
+			s:         s,
+			mac:       conf.mac,
+			portmap:   conf.svcs.Contains(NATPMP), // TODO: expand network.portmap
+			wanIP:     conf.wanIP,
+			lanIP:     conf.lanIP,
+			nodesByIP: map[netip.Addr]*node{},
+			logf:      logger.WithPrefix(log.Printf, fmt.Sprintf("[net-%v] ", conf.mac)),
+		}
+		netOfConf[conf] = n
+		s.networks.Add(n)
+		if _, ok := s.networkByWAN[conf.wanIP]; ok {
+			return fmt.Errorf("two networks have the same WAN IP %v; Anycast not (yet?) supported", conf.wanIP)
+		}
+		s.networkByWAN[conf.wanIP] = n
+	}
+	for i, conf := range c.nodes {
+		if conf.err != nil {
+			return conf.err
+		}
+		n := &node{
+			mac: conf.mac,
+			id:  i + 1,
+			net: netOfConf[conf.Network()],
+		}
+		if s.pcapWriter != nil {
+			s.pcapWriter.w.AddInterface(pcapgo.NgInterface{
+				Name:     fmt.Sprintf("node%d", n.id),
+				LinkType: layers.LinkTypeEthernet,
+			})
+		}
+		conf.n = n
+		if _, ok := s.nodeByMAC[n.mac]; ok {
+			return fmt.Errorf("two nodes have the same MAC %v", n.mac)
+		}
+		s.nodes = append(s.nodes, n)
+		s.nodeByMAC[n.mac] = n
+
+		// Allocate a lanIP for the node. Use the network's CIDR and use final
+		// octet 101 (for first node), 102, etc. The node number comes from the
+		// last octent of the MAC address (0-based)
+		ip4 := n.net.lanIP.Addr().As4()
+		ip4[3] = 101 + n.mac[5]
+		n.lanIP = netip.AddrFrom4(ip4)
+		n.net.nodesByIP[n.lanIP] = n
+	}
+
+	// Now that nodes are populated, set up NAT:
+	for _, conf := range c.networks {
+		n := netOfConf[conf]
+		natType := cmp.Or(conf.natType, EasyNAT)
+		if err := n.InitNAT(natType); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tstest/natlab/vnet/conf_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import "testing"
+
+func TestConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		setup   func(*Config)
+		wantErr string
+	}{
+		{
+			name: "simple",
+			setup: func(c *Config) {
+				c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", EasyNAT, NATPMP))
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", HardNAT))
+			},
+		},
+		{
+			name: "indirect",
+			setup: func(c *Config) {
+				n1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", HardNAT))
+				n1.Network().AddService(NATPMP)
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", NAT("hard")))
+			},
+		},
+		{
+			name: "multi-node-in-net",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+		},
+		{
+			name: "dup-wan-ip",
+			setup: func(c *Config) {
+				c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNetwork("2.1.1.1", "10.2.0.1/16")
+			},
+			wantErr: "two networks have the same WAN IP 2.1.1.1; Anycast not (yet?) supported",
+		},
+		{
+			name: "one-to-one-nat-with-multiple-nodes",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24", One2OneNAT)
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+			wantErr: "error creating NAT type \"one2one\" for network 2.1.1.1: can't use one2one NAT type on networks other than single-node networks",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var c Config
+			tt.setup(&c)
+			_, err := New(&c)
+			if err == nil {
+				if tt.wantErr == "" {
+					return
+				}
+				t.Fatalf("got success; wanted error %q", tt.wantErr)
+			}
+			if err.Error() != tt.wantErr {
+				t.Fatalf("got error %q; want %q", err, tt.wantErr)
+			}
+		})
+	}
+}

tstest/natlab/vnet/easyaf.go

@@ -0,0 +1,91 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"log"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+// easyAFNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux), but with only address filtering, not address+port
+// filtering.
+//
+// James says these are used by "anyone with “voip helpers” turned on"
+// "which is a lot of home modem routers" ... "probably like most of the zyxel
+// type things".
+type easyAFNAT struct {
+	pool    IPPool
+	wanIP   netip.Addr
+	out     map[netip.Addr]portMappingAndTime
+	in      map[uint16]lanAddrAndTime
+	lastOut map[srcAPDstAddrTuple]time.Time // (lan:port, wan:port) => last packet out time
+}
+
+type srcAPDstAddrTuple struct {
+	src netip.AddrPort
+	dst netip.Addr
+}
+
+func init() {
+	registerNATType(EasyAFNAT, func(p IPPool) (NATTable, error) {
+		return &easyAFNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyAFNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	_, ok := n.in[ap.Port()]
+	return ok
+}
+
+func (n *easyAFNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	mak.Set(&n.lastOut, srcAPDstAddrTuple{src, dst.Addr()}, at)
+	if pm, ok := n.out[src.Addr()]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+			if n.pool.IsPublicPortUsed(wanAddr) {
+				continue
+			}
+
+			// Found a free port.
+			mak.Set(&n.out, src.Addr(), portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyAFNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	lanDst = n.in[dst.Port()].lanAddr
+
+	// Stateful firewall: drop incoming packets that don't have traffic out.
+	// TODO(bradfitz): verify Linux does this in the router code, not in the NAT code.
+	if t, ok := n.lastOut[srcAPDstAddrTuple{lanDst, src.Addr()}]; !ok || at.Sub(t) > 300*time.Second {
+		log.Printf("Drop incoming packet from %v to %v; no recent outgoing packet", src, dst)
+		return netip.AddrPort{}
+	}
+
+	return lanDst
+}

tstest/natlab/vnet/nat.go

@@ -0,0 +1,293 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"errors"
+	"log"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+const (
+	One2OneNAT NAT = "one2one"
+	EasyNAT    NAT = "easy"   // address+port filtering
+	EasyAFNAT  NAT = "easyaf" // address filtering (not port)
+	HardNAT    NAT = "hard"
+)
+
+// IPPool is the interface that a NAT implementation uses to get information
+// about a network.
+//
+// Outside of tests, this is typically a *network.
+type IPPool interface {
+	// WANIP returns the primary WAN IP address.
+	//
+	// TODO: add another method for networks with multiple WAN IP addresses.
+	WANIP() netip.Addr
+
+	// SoleLanIP reports whether this network has a sole LAN client
+	// and if so, its IP address.
+	SoleLANIP() (_ netip.Addr, ok bool)
+
+	// IsPublicPortUsed reports whether the provided WAN IP+port is in use by
+	// anything. (In particular, the NAT-PMP/etc port mappers might have taken
+	// a port.) Implementations should check this before allocating a port,
+	// and then they should report IsPublicPortUsed themselves for that port.
+	IsPublicPortUsed(netip.AddrPort) bool
+}
+
+// newTableFunc is a constructor for a NAT table.
+// The provided IPPool is typically (outside of tests) a *network.
+type newTableFunc func(IPPool) (NATTable, error)
+
+// NAT is a type of NAT that's known to natlab.
+//
+// For example, "easy" for Linux-style NAT, "hard" for FreeBSD-style NAT, etc.
+type NAT string
+
+// natTypes are the known NAT types.
+var natTypes = map[NAT]newTableFunc{}
+
+// registerNATType registers a NAT type.
+func registerNATType(name NAT, f newTableFunc) {
+	if _, ok := natTypes[name]; ok {
+		panic("duplicate NAT type: " + name)
+	}
+	natTypes[name] = f
+}
+
+// NATTable is what a NAT implementation is expected to do.
+//
+// This project tests Tailscale as it faces various combinations various NAT
+// implementations (e.g. Linux easy style NAT vs FreeBSD hard/endpoint dependent
+// NAT vs Cloud 1:1 NAT, etc)
+//
+// Implementations of NATTable need not handle concurrency; the natlab serializes
+// all calls into a NATTable.
+//
+// The provided `at` value will typically be time.Now, except for tests.
+// Implementations should not use real time and should only compare
+// previously provided time values.
+type NATTable interface {
+	// PickOutgoingSrc returns the source address to use for an outgoing packet.
+	//
+	// The result should either be invalid (to drop the packet) or a WAN (not
+	// private) IP address.
+	//
+	// Typically, the src is a LAN source IP address, but it might also be a WAN
+	// IP address if the packet is being forwarded for a source machine that has
+	// a public IP address.
+	PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort)
+
+	// PickIncomingDst returns the destination address to use for an incoming
+	// packet. The incoming src address is always a public WAN IP.
+	//
+	// The result should either be invalid (to drop the packet) or the IP
+	// address of a machine on the local network address, usually a private
+	// LAN IP.
+	PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort)
+
+	// IsPublicPortUsed reports whether the provided WAN IP+port is in use by
+	// anything. The port mapper uses this to avoid grabbing an in-use port.
+	IsPublicPortUsed(netip.AddrPort) bool
+}
+
+// oneToOneNAT is a 1:1 NAT, like a typical EC2 VM.
+type oneToOneNAT struct {
+	lanIP netip.Addr
+	wanIP netip.Addr
+}
+
+func init() {
+	registerNATType(One2OneNAT, func(p IPPool) (NATTable, error) {
+		lanIP, ok := p.SoleLANIP()
+		if !ok {
+			return nil, errors.New("can't use one2one NAT type on networks other than single-node networks")
+		}
+		return &oneToOneNAT{lanIP: lanIP, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *oneToOneNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	return netip.AddrPortFrom(n.wanIP, src.Port())
+}
+
+func (n *oneToOneNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	return netip.AddrPortFrom(n.lanIP, dst.Port())
+}
+
+func (n *oneToOneNAT) IsPublicPortUsed(netip.AddrPort) bool {
+	return true // all ports are owned by the 1:1 NAT
+}
+
+type srcDstTuple struct {
+	src netip.AddrPort
+	dst netip.AddrPort
+}
+
+type hardKeyIn struct {
+	wanPort uint16
+	src     netip.AddrPort
+}
+
+type portMappingAndTime struct {
+	port uint16
+	at   time.Time
+}
+
+type lanAddrAndTime struct {
+	lanAddr netip.AddrPort
+	at      time.Time
+}
+
+// hardNAT is an "Endpoint Dependent" NAT, like FreeBSD/pfSense/OPNsense.
+// This is shown as "MappingVariesByDestIP: true" by netcheck, and what
+// Tailscale calls "Hard NAT".
+type hardNAT struct {
+	pool  IPPool
+	wanIP netip.Addr
+
+	out map[srcDstTuple]portMappingAndTime
+	in  map[hardKeyIn]lanAddrAndTime
+}
+
+func init() {
+	registerNATType(HardNAT, func(p IPPool) (NATTable, error) {
+		return &hardNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *hardNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	for k := range n.in {
+		if k.wanPort == ap.Port() {
+			return true
+		}
+	}
+	return false
+}
+
+func (n *hardNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	ko := srcDstTuple{src, dst}
+	if pm, ok := n.out[ko]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// No existing mapping exists. Create one.
+
+	// TODO: clean up old expired mappings
+
+	// Instead of proper data structures that would be efficient, we instead
+	// just loop a bunch and look for a free port. This project is only used
+	// by tests and doesn't care about performance, this is good enough.
+	for {
+		port := rand.N(uint16(32<<10)) + 32<<10 // pick some "ephemeral" port
+		if n.pool.IsPublicPortUsed(netip.AddrPortFrom(n.wanIP, port)) {
+			continue
+		}
+
+		ki := hardKeyIn{wanPort: port, src: dst}
+		if _, ok := n.in[ki]; ok {
+			// Port already in use.
+			continue
+		}
+		mak.Set(&n.in, ki, lanAddrAndTime{lanAddr: src, at: at})
+		mak.Set(&n.out, ko, portMappingAndTime{port: port, at: at})
+		return netip.AddrPortFrom(n.wanIP, port)
+	}
+}
+
+func (n *hardNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	ki := hardKeyIn{wanPort: dst.Port(), src: src}
+	if pm, ok := n.in[ki]; ok {
+		// Existing flow.
+		return pm.lanAddr
+	}
+	return netip.AddrPort{} // drop; no mapping
+}
+
+// easyNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux).
+//
+// This is shown as "MappingVariesByDestIP: false" by netcheck, and what
+// Tailscale calls "Easy NAT".
+//
+// Unlike Linux, this implementation is capped at 32k entries and doesn't resort
+// to other allocation strategies when all 32k WAN ports are taken.
+type easyNAT struct {
+	pool    IPPool
+	wanIP   netip.Addr
+	out     map[netip.AddrPort]portMappingAndTime
+	in      map[uint16]lanAddrAndTime
+	lastOut map[srcDstTuple]time.Time // (lan:port, wan:port) => last packet out time
+}
+
+func init() {
+	registerNATType(EasyNAT, func(p IPPool) (NATTable, error) {
+		return &easyNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	_, ok := n.in[ap.Port()]
+	return ok
+}
+
+func (n *easyNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	mak.Set(&n.lastOut, srcDstTuple{src, dst}, at)
+	if pm, ok := n.out[src]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+			if n.pool.IsPublicPortUsed(wanAddr) {
+				continue
+			}
+
+			// Found a free port.
+			mak.Set(&n.out, src, portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	lanDst = n.in[dst.Port()].lanAddr
+
+	// Stateful firewall: drop incoming packets that don't have traffic out.
+	// TODO(bradfitz): verify Linux does this in the router code, not in the NAT code.
+	if t, ok := n.lastOut[srcDstTuple{lanDst, src}]; !ok || at.Sub(t) > 300*time.Second {
+		log.Printf("Drop incoming packet from %v to %v; no recent outgoing packet", src, dst)
+		return netip.AddrPort{}
+	}
+
+	return lanDst
+}

tstest/natlab/vnet/pcap.go

@@ -0,0 +1,39 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/pcapgo"
+)
+
+type pcapWriter struct {
+	f *os.File
+
+	mu sync.Mutex
+	w  *pcapgo.NgWriter
+}
+
+func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.w == nil {
+		return io.ErrClosedPipe
+	}
+	return p.w.WritePacket(ci, data)
+}
+
+func (p *pcapWriter) Close() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.w != nil {
+		p.w.Flush()
+		p.w = nil
+	}
+	return p.f.Close()
+}