metrics: use constants for metric types

NewMultiLabelMap does not return an error so it's easy for someone to
pass an incorrect/unsupported metric type and serve invalid metrics to
Prometheus.

This would read as `metrics.Counter` and `metrics.Gauge` when used,
which seems readable enough; but if you think bare strings are better,
an alternative to this approach might be to verify string value in
`NewMultiLabelMap` and panic?

Updates tailscale/corp#22075

Signed-off-by: Anton Tolchanov <anton@tailscale.com>

client/tailscale/acl.go

@@ -286,6 +286,9 @@ type UserRuleMatch struct {
 	Users      []string `json:"users"`
 	Ports      []string `json:"ports"`
 	LineNumber int      `json:"lineNumber"`
+	// Via is the list of targets through which Users can access Ports.
+	// See https://tailscale.com/kb/1378/via for more information.
+	Via []string `json:"via,omitempty"`
 
 	// Postures is a list of posture policies that are
 	// associated with this match. The rules can be looked

client/tailscale/localclient.go

@@ -69,6 +69,14 @@ type LocalClient struct {
 	// connecting to the GUI client variants.
 	UseSocketOnly bool
 
+	// OmitAuth, if true, omits sending the local Tailscale daemon any
+	// authentication token that might be required by the platform.
+	//
+	// As of 2024-08-12, only macOS uses an authentication token. OmitAuth is
+	// meant for when Dial is set and the LocalAPI is being proxied to a
+	// different operating system, such as in integration tests.
+	OmitAuth bool
+
 	// tsClient does HTTP requests to the local Tailscale daemon.
 	// It's lazily initialized on first use.
 	tsClient     *http.Client
@@ -124,8 +132,10 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 			},
 		}
 	})
-	if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
-		req.SetBasicAuth("", token)
+	if !lc.OmitAuth {
+		if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
+			req.SetBasicAuth("", token)
+		}
 	}
 	return lc.tsClient.Do(req)
 }

cmd/derper/depaware.txt

@@ -10,7 +10,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -146,9 +146,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -159,6 +161,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
@@ -180,6 +184,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/k8s-operator/depaware.txt

@@ -96,7 +96,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -804,6 +804,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+

cmd/stunstamp/stunstamp.go

@@ -1,13 +1,14 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-// The stunstamp binary measures STUN round-trip latency with DERPs.
+// The stunstamp binary measures round-trip latency with DERPs.
 package main
 
 import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -22,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"runtime"
 	"slices"
 	"strconv"
 	"strings"
@@ -31,8 +33,10 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/prompb"
+	"github.com/tcnksm/go-httpstat"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/stun"
+	"tailscale.com/net/tcpinfo"
 	"tailscale.com/tailcfg"
 )
 
@@ -42,7 +46,10 @@ var (
 	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
 	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
 	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
-	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+	flagSTUNDstPorts   = flag.String("stun-dst-ports", "", "comma-separated list of STUN destination ports to monitor")
+	flagHTTPSDstPorts  = flag.String("https-dst-ports", "", "comma-separated list of HTTPS destination ports to monitor")
+	flagTCPDstPorts    = flag.String("tcp-dst-ports", "", "comma-separated list of TCP destination ports to monitor")
+	flagICMP           = flag.Bool("icmp", false, "probe ICMP")
 )
 
 const (
@@ -89,12 +96,22 @@ func (t timestampSource) String() string {
 	}
 }
 
+type protocol string
+
+const (
+	protocolSTUN  protocol = "stun"
+	protocolICMP  protocol = "icmp"
+	protocolHTTPS protocol = "https"
+	protocolTCP   protocol = "tcp"
+)
+
 // resultKey contains the stable dimensions and their values for a given
 // timeseries, i.e. not time and not rtt/timeout.
 type resultKey struct {
 	meta            nodeMeta
 	timestampSource timestampSource
 	connStability   connStability
+	protocol        protocol
 	dstPort         int
 }
 
@@ -104,7 +121,203 @@ type result struct {
 	rtt *time.Duration // nil signifies failure, e.g. timeout
 }
 
-func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+type lportsPool struct {
+	sync.Mutex
+	ports []int
+}
+
+func (l *lportsPool) get() int {
+	l.Lock()
+	defer l.Unlock()
+	ret := l.ports[0]
+	l.ports = append(l.ports[:0], l.ports[1:]...)
+	return ret
+}
+
+func (l *lportsPool) put(i int) {
+	l.Lock()
+	defer l.Unlock()
+	l.ports = append(l.ports, int(i))
+}
+
+var (
+	lports *lportsPool
+)
+
+const (
+	lportPoolSize = 16000
+	lportBase     = 2048
+)
+
+func init() {
+	lports = &lportsPool{
+		ports: make([]int, 0, lportPoolSize),
+	}
+	for i := lportBase; i < lportBase+lportPoolSize; i++ {
+		lports.ports = append(lports.ports, i)
+	}
+}
+
+// lportForTCPConn satisfies io.ReadWriteCloser, but is really just used to pass
+// around a persistent laddr for stableConn purposes. The underlying TCP
+// connection is not created until measurement time as in some cases we need to
+// measure dial time.
+type lportForTCPConn int
+
+func (l *lportForTCPConn) Close() error {
+	if *l == 0 {
+		return nil
+	}
+	lports.put(int(*l))
+	return nil
+}
+
+func (l *lportForTCPConn) Write([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func (l *lportForTCPConn) Read([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func addrInUse(err error, lport *lportForTCPConn) bool {
+	if errors.Is(err, syscall.EADDRINUSE) {
+		old := int(*lport)
+		// abandon port, don't return it to pool
+		*lport = lportForTCPConn(lports.get()) // get a new port
+		log.Printf("EADDRINUSE: %v old: %d new: %d", err, old, *lport)
+		return true
+	}
+	return false
+}
+
+func tcpDial(ctx context.Context, lport *lportForTCPConn, dst netip.AddrPort) (net.Conn, error) {
+	for {
+		var opErr error
+		dialer := &net.Dialer{
+			LocalAddr: &net.TCPAddr{
+				Port: int(*lport),
+			},
+			Control: func(network, address string, c syscall.RawConn) error {
+				return c.Control(func(fd uintptr) {
+					// we may restart faster than TIME_WAIT can clear
+					opErr = setSOReuseAddr(fd)
+				})
+			},
+		}
+		if opErr != nil {
+			panic(opErr)
+		}
+		tcpConn, err := dialer.DialContext(ctx, "tcp", dst.String())
+		if err != nil {
+			if addrInUse(err, lport) {
+				continue
+			}
+			return nil, err
+		}
+		return tcpConn, nil
+	}
+}
+
+type tempError struct {
+	error
+}
+
+func (t tempError) Temporary() bool {
+	return true
+}
+
+func measureTCPRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	// Set a dial timeout < 1s (TCP_TIMEOUT_INIT on Linux) as a means to avoid
+	// SYN retries, which can contribute to tcpi->rtt below. This simply limits
+	// retries from the initiator, but SYN+ACK on the reverse path can also
+	// time out and be retransmitted.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*750)
+	defer cancel()
+	tcpConn, err := tcpDial(ctx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	// This is an unreliable method to measure TCP RTT. The Linux kernel
+	// describes it as such in tcp_rtt_estimator(). We take some care in how we
+	// hold tcp_info->rtt here, e.g. clamping dial timeout, but if we are to
+	// actually use this elsewhere as an input to some decision it warrants a
+	// deeper study and consideration for alternative methods. Its usefulness
+	// here is as a point of comparison against the other methods.
+	rtt, err = tcpinfo.RTT(tcpConn)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	return rtt, nil
+}
+
+func measureHTTPSRTT(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	var httpResult httpstat.Result
+	// 5s mirrors net/netcheck.overallProbeTimeout used in net/netcheck.Client.measureHTTPSLatency.
+	reqCtx, cancel := context.WithTimeout(httpstat.WithHTTPStat(context.Background(), &httpResult), time.Second*5)
+	defer cancel()
+	reqURL := "https://" + dst.String() + "/derp/latency-check"
+	req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil)
+	if err != nil {
+		return 0, err
+	}
+	client := &http.Client{}
+	// 1.5s mirrors derp/derphttp.dialnodeTimeout used in derp/derphttp.DialNode().
+	dialCtx, dialCancel := context.WithTimeout(reqCtx, time.Millisecond*1500)
+	defer dialCancel()
+	tcpConn, err := tcpDial(dialCtx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	tlsConn := tls.Client(tcpConn, &tls.Config{
+		ServerName: hostname,
+	})
+	// Mirror client/netcheck behavior, which handshakes before handing the
+	// tlsConn over to the http.Client via http.Transport
+	err = tlsConn.Handshake()
+	if err != nil {
+		return 0, tempError{err}
+	}
+	tlsConnCh := make(chan net.Conn, 1)
+	tlsConnCh <- tlsConn
+	tr := &http.Transport{
+		DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
+			select {
+			case tlsConn := <-tlsConnCh:
+				return tlsConn, nil
+			default:
+				return nil, errors.New("unexpected second call of DialTLSContext")
+			}
+		},
+	}
+	client.Transport = tr
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	if resp.StatusCode/100 != 2 {
+		return 0, tempError{fmt.Errorf("unexpected status code: %d", resp.StatusCode)}
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(io.Discard, io.LimitReader(resp.Body, 8<<10))
+	if err != nil {
+		return 0, tempError{err}
+	}
+	httpResult.End(time.Now())
+	return httpResult.ServerProcessing, nil
+}
+
+func measureSTUNRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	uconn, ok := conn.(*net.UDPConn)
 	if !ok {
 		return 0, fmt.Errorf("unexpected conn type: %T", conn)
@@ -116,7 +329,10 @@ func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, e
 	txID := stun.NewTxID()
 	req := stun.Request(txID)
 	txAt := time.Now()
-	_, err = uconn.WriteToUDP(req, dst)
+	_, err = uconn.WriteToUDP(req, &net.UDPAddr{
+		IP:   dst.Addr().AsSlice(),
+		Port: int(dst.Port()),
+	})
 	if err != nil {
 		return 0, fmt.Errorf("error writing to udp socket: %w", err)
 	}
@@ -153,20 +369,19 @@ type nodeMeta struct {
 	addr       netip.Addr
 }
 
-type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
+type measureFn func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error)
 
-// probe measures STUN round trip time for the node described by meta over
-// conn against dstPort. It may return a nil duration and nil error if the
-// STUN request timed out. A non-nil error indicates an unrecoverable or
-// non-temporary error.
-func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
+// probe measures round trip time for the node described by meta over cf against
+// dstPort. It may return a nil duration and nil error in the event of a
+// timeout. A non-nil error indicates an unrecoverable or non-temporary error.
+func probe(meta nodeMeta, cf *connAndMeasureFn, dstPort int) (*time.Duration, error) {
 	ua := &net.UDPAddr{
 		IP:   net.IP(meta.addr.AsSlice()),
 		Port: dstPort,
 	}
 
 	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
-	rtt, err := fn(conn, ua)
+	rtt, err := cf.fn(cf.conn, meta.hostname, netip.AddrPortFrom(meta.addr, uint16(dstPort)))
 	if err != nil {
 		if isTemporaryOrTimeoutErr(err) {
 			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
@@ -237,43 +452,138 @@ func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]node
 	return stale, nil
 }
 
-func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
-	conns := [2]io.ReadWriteCloser{}
-	byDstPort, ok := stableConns[addr]
-	if ok {
-		conns, ok = byDstPort[dstPort]
-		if ok {
-			return conns, nil
-		}
-	}
-	if supportsKernelTS() {
-		kconn, err := getConnKernelTimestamp()
-		if err != nil {
-			return conns, err
-		}
-		conns[timestampSourceKernel] = kconn
-	}
-	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
-	if err != nil {
-		if supportsKernelTS() {
-			conns[timestampSourceKernel].Close()
-		}
-		return conns, err
-	}
-	conns[timestampSourceUserspace] = uconn
-	if byDstPort == nil {
-		byDstPort = make(map[int][2]io.ReadWriteCloser)
-	}
-	byDstPort[dstPort] = conns
-	stableConns[addr] = byDstPort
-	return conns, nil
+type connAndMeasureFn struct {
+	conn io.ReadWriteCloser
+	fn   measureFn
 }
 
-// probeNodes measures the round-trip time for STUN binding requests against the
-// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
-// UDP sockets that should be recycled across runs. It returns the results or
-// an error if one occurs.
-func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+// newConnAndMeasureFn returns a connAndMeasureFn or an error. It may return
+// nil for both if some combination of the supplied timestampSource, protocol,
+// or connStability is unsupported.
+func newConnAndMeasureFn(source timestampSource, protocol protocol, stable connStability) (*connAndMeasureFn, error) {
+	info := getProtocolSupportInfo(protocol)
+	if !info.stableConn && bool(stable) {
+		return nil, nil
+	}
+	if !info.userspaceTS && source == timestampSourceUserspace {
+		return nil, nil
+	}
+	if !info.kernelTS && source == timestampSourceKernel {
+		return nil, nil
+	}
+	switch protocol {
+	case protocolSTUN:
+		if source == timestampSourceKernel {
+			conn, err := getUDPConnKernelTimestamp()
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTTKernel,
+			}, nil
+		} else {
+			conn, err := net.ListenUDP("udp", &net.UDPAddr{})
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTT,
+			}, nil
+		}
+	case protocolICMP:
+		// TODO(jwhited): implement
+		return nil, nil
+	case protocolHTTPS:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureHTTPSRTT,
+		}, nil
+	case protocolTCP:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureTCPRTT,
+		}, nil
+	}
+	return nil, errors.New("unknown protocol")
+}
+
+type stableConnKey struct {
+	node     netip.Addr
+	protocol protocol
+	port     int
+}
+
+type protocolSupportInfo struct {
+	kernelTS    bool
+	userspaceTS bool
+	stableConn  bool
+}
+
+func getConns(
+	stableConns map[stableConnKey][2]*connAndMeasureFn,
+	addr netip.Addr,
+	protocol protocol,
+	dstPort int,
+) (stable, unstable [2]*connAndMeasureFn, err error) {
+	key := stableConnKey{addr, protocol, dstPort}
+	defer func() {
+		if err != nil {
+			for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+				c := stable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+				c = unstable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+			}
+		}
+	}()
+
+	var ok bool
+	stable, ok = stableConns[key]
+	if !ok {
+		for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+			var cf *connAndMeasureFn
+			cf, err = newConnAndMeasureFn(source, protocol, stableConn)
+			if err != nil {
+				return
+			}
+			stable[source] = cf
+		}
+		stableConns[key] = stable
+	}
+
+	for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+		var cf *connAndMeasureFn
+		cf, err = newConnAndMeasureFn(source, protocol, unstableConn)
+		if err != nil {
+			return
+		}
+		unstable[source] = cf
+	}
+	return stable, unstable, nil
+}
+
+// probeNodes measures the round-trip time for the protocols and ports described
+// by portsByProtocol against the DERP nodes described by nodeMetaByAddr.
+// stableConns are used to recycle connections across calls to probeNodes.
+// probeNodes is also responsible for trimming stableConns based on node
+// lifetime in nodeMetaByAddr. It returns the results or an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[stableConnKey][2]*connAndMeasureFn, portsByProtocol map[protocol][]int) ([]result, error) {
 	wg := sync.WaitGroup{}
 	results := make([]result, 0)
 	resultsCh := make(chan result)
@@ -283,40 +593,19 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 	at := time.Now()
 	addrsToProbe := make(map[netip.Addr]bool)
 
-	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+	doProbe := func(cf *connAndMeasureFn, meta nodeMeta, source timestampSource, stable connStability, protocol protocol, dstPort int) {
 		defer wg.Done()
 		r := result{
 			key: resultKey{
 				meta:            meta,
 				timestampSource: source,
+				connStability:   stable,
 				dstPort:         dstPort,
+				protocol:        protocol,
 			},
 			at: at,
 		}
-		if conn == nil {
-			var err error
-			if source == timestampSourceKernel {
-				conn, err = getConnKernelTimestamp()
-			} else {
-				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
-			}
-			if err != nil {
-				select {
-				case <-doneCh:
-					return
-				case errCh <- err:
-					return
-				}
-			}
-			defer conn.Close()
-		} else {
-			r.key.connStability = stableConn
-		}
-		fn := measureRTT
-		if source == timestampSourceKernel {
-			fn = measureRTTKernel
-		}
-		rtt, err := probe(meta, conn, fn, dstPort)
+		rtt, err := probe(meta, cf, dstPort)
 		if err != nil {
 			select {
 			case <-doneCh:
@@ -334,37 +623,42 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 
 	for _, meta := range nodeMetaByAddr {
 		addrsToProbe[meta.addr] = true
-		for _, port := range dstPorts {
-			stable, err := getStableConns(stableConns, meta.addr, port)
-			if err != nil {
-				close(doneCh)
-				wg.Wait()
-				return nil, err
-			}
+		for p, ports := range portsByProtocol {
+			for _, port := range ports {
+				stable, unstable, err := getConns(stableConns, meta.addr, p, port)
+				if err != nil {
+					close(doneCh)
+					wg.Wait()
+					return nil, err
+				}
 
-			wg.Add(2)
-			numProbes += 2
-			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
-			go doProbe(nil, meta, timestampSourceUserspace, port)
-			if supportsKernelTS() {
-				wg.Add(2)
-				numProbes += 2
-				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
-				go doProbe(nil, meta, timestampSourceKernel, port)
+				for i, cf := range stable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), stableConn, p, port)
+					}
+				}
+
+				for i, cf := range unstable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), unstableConn, p, port)
+					}
+				}
 			}
 		}
 	}
 
 	// cleanup conns we no longer need
-	for k, byDstPort := range stableConns {
-		if !addrsToProbe[k] {
-			for _, conns := range byDstPort {
-				if conns[timestampSourceKernel] != nil {
-					conns[timestampSourceKernel].Close()
-				}
-				conns[timestampSourceUserspace].Close()
-				delete(stableConns, k)
+	for k, cf := range stableConns {
+		if !addrsToProbe[k.node] {
+			if cf[timestampSourceKernel] != nil {
+				cf[timestampSourceKernel].conn.Close()
 			}
+			cf[timestampSourceUserspace].conn.Close()
+			delete(stableConns, k)
 		}
 	}
 
@@ -391,11 +685,11 @@ const (
 )
 
 const (
-	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
-	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+	rttMetricName      = "stunstamp_derp_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_timeouts_total"
 )
 
-func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, protocol protocol, dstPort int) []prompb.Label {
 	addressFamily := "ipv4"
 	if meta.addr.Is6() {
 		addressFamily = "ipv6"
@@ -425,6 +719,10 @@ func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source
 		Name:  "hostname",
 		Value: meta.hostname,
 	})
+	labels = append(labels, prompb.Label{
+		Name:  "protocol",
+		Value: string(protocol),
+	})
 	labels = append(labels, prompb.Label{
 		Name:  "dst_port",
 		Value: strconv.Itoa(dstPort),
@@ -453,53 +751,35 @@ const (
 	staleNaN uint64 = 0x7ff0000000000002
 )
 
-func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, portsByProtocol map[protocol][]int) []prompb.TimeSeries {
 	staleMarkers := make([]prompb.TimeSeries, 0)
 	now := time.Now()
-	for _, s := range stale {
-		for _, dstPort := range dstPorts {
-			samples := []prompb.Sample{
-				{
-					Timestamp: now.UnixMilli(),
-					Value:     math.Float64frombits(staleNaN),
-				},
-			}
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			if supportsKernelTS() {
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
+
+	for p, ports := range portsByProtocol {
+		for _, port := range ports {
+			for _, s := range stale {
+				samples := []prompb.Sample{
+					{
+						Timestamp: now.UnixMilli(),
+						Value:     math.Float64frombits(staleNaN),
+					},
+				}
+				// We send stale markers for all combinations in the interest
+				// of simplicity.
+				for _, name := range []string{rttMetricName, timeoutsMetricName} {
+					for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+						for _, stable := range []connStability{unstableConn, stableConn} {
+							staleMarkers = append(staleMarkers, prompb.TimeSeries{
+								Labels:  timeSeriesLabels(name, s, instance, source, stable, p, port),
+								Samples: samples,
+							})
+						}
+					}
+				}
 			}
 		}
 	}
+
 	return staleMarkers
 }
 
@@ -513,7 +793,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 	for _, r := range results {
 		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
 		seenKeys[r.key] = true
-		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		rttSamples := make([]prompb.Sample, 1)
 		rttSamples[0].Timestamp = r.at.UnixMilli()
 		if r.rtt != nil {
@@ -528,7 +808,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 		}
 		all = append(all, rttTS)
 		timeouts[r.key] = timeoutsCount
-		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		timeoutsSamples := make([]prompb.Sample, 1)
 		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
 		timeoutsSamples[0].Value = float64(timeoutsCount)
@@ -620,22 +900,66 @@ func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSer
 	}
 }
 
-func main() {
-	flag.Parse()
-	if len(*flagDstPorts) == 0 {
-		log.Fatal("dst-ports flag is unset")
+func getPortsFromFlag(f string) ([]int, error) {
+	if len(f) == 0 {
+		return nil, nil
 	}
-	dstPortsSplit := strings.Split(*flagDstPorts, ",")
-	slices.Sort(dstPortsSplit)
-	dstPortsSplit = slices.Compact(dstPortsSplit)
-	dstPorts := make([]int, 0, len(dstPortsSplit))
-	for _, d := range dstPortsSplit {
-		i, err := strconv.ParseUint(d, 10, 16)
+	split := strings.Split(f, ",")
+	slices.Sort(split)
+	split = slices.Compact(split)
+	ports := make([]int, 0)
+	for _, portStr := range split {
+		port, err := strconv.ParseUint(portStr, 10, 16)
 		if err != nil {
-			log.Fatal("invalid dst-ports")
+			return nil, err
 		}
-		dstPorts = append(dstPorts, int(i))
+		ports = append(ports, int(port))
 	}
+	return ports, nil
+}
+
+func main() {
+	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
+		log.Fatal("unsupported platform")
+	}
+	flag.Parse()
+
+	portsByProtocol := make(map[protocol][]int)
+	stunPorts, err := getPortsFromFlag(*flagSTUNDstPorts)
+	if err != nil {
+		log.Fatalf("invalid stun-dst-ports flag value: %v", err)
+	}
+	if len(stunPorts) > 0 {
+		portsByProtocol[protocolSTUN] = stunPorts
+	}
+	httpsPorts, err := getPortsFromFlag(*flagHTTPSDstPorts)
+	if err != nil {
+		log.Fatalf("invalid https-dst-ports flag value: %v", err)
+	}
+	if len(httpsPorts) > 0 {
+		portsByProtocol[protocolHTTPS] = httpsPorts
+	}
+	tcpPorts, err := getPortsFromFlag(*flagTCPDstPorts)
+	if err != nil {
+		log.Fatalf("invalid tcp-dst-ports flag value: %v", err)
+	}
+	if len(tcpPorts) > 0 {
+		portsByProtocol[protocolTCP] = tcpPorts
+	}
+	if *flagICMP {
+		portsByProtocol[protocolICMP] = []int{0}
+	}
+	if len(portsByProtocol) == 0 {
+		log.Fatal("nothing to probe")
+	}
+
+	// TODO(jwhited): remove protocol restriction
+	for k := range portsByProtocol {
+		if k != protocolSTUN && k != protocolHTTPS && k != protocolTCP {
+			log.Fatal("ICMP is not yet supported")
+		}
+	}
+
 	if len(*flagDERPMap) < 1 {
 		log.Fatal("derp-map flag is unset")
 	}
@@ -645,7 +969,7 @@ func main() {
 	if len(*flagRemoteWriteURL) < 1 {
 		log.Fatal("rw-url flag is unset")
 	}
-	_, err := url.Parse(*flagRemoteWriteURL)
+	_, err = url.Parse(*flagRemoteWriteURL)
 	if err != nil {
 		log.Fatalf("invalid rw-url flag value: %v", err)
 	}
@@ -707,7 +1031,7 @@ func main() {
 		for _, v := range nodeMetaByAddr {
 			staleMeta = append(staleMeta, v)
 		}
-		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 		if len(staleMarkers) > 0 {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 			rwc.write(ctx, staleMarkers)
@@ -723,8 +1047,8 @@ func main() {
 	// in a higher probability of the packets traversing the same underlay path.
 	// Comparison of stable and unstable 5-tuple results can shed light on
 	// differences between paths where hashing (multipathing/load balancing)
-	// comes into play.
-	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+	// comes into play. The inner 2 element array index is timestampSource.
+	stableConns := make(map[stableConnKey][2]*connAndMeasureFn)
 
 	// timeouts holds counts of timeout events. Values are persisted for the
 	// lifetime of the related node in the DERP map.
@@ -738,7 +1062,7 @@ func main() {
 	for {
 		select {
 		case <-probeTicker.C:
-			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			results, err := probeNodes(nodeMetaByAddr, stableConns, portsByProtocol)
 			if err != nil {
 				log.Printf("unrecoverable error while probing: %v", err)
 				shutdown()
@@ -761,7 +1085,7 @@ func main() {
 				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
 				continue
 			}
-			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 			if len(staleMarkers) < 1 {
 				continue
 			}
@@ -780,7 +1104,7 @@ func main() {
 				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 				defer cancel()
 				updatedDM, err := getDERPMap(ctx, *flagDERPMap)
-				if err != nil {
+				if err == nil {
 					dmCh <- updatedDM
 				}
 			}()

cmd/stunstamp/stunstamp_default.go

@@ -8,18 +8,42 @@ package main
 import (
 	"errors"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unimplemented")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	return 0, errors.New("unimplemented")
 }
 
-func supportsKernelTS() bool {
-	return false
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	return nil
 }

cmd/stunstamp/stunstamp_linux.go

@@ -10,7 +10,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
+	"net/netip"
+	"syscall"
 	"time"
 
 	"github.com/mdlayher/socket"
@@ -24,7 +25,7 @@ const (
 		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
 	if err != nil {
 		return nil, err
@@ -56,24 +57,23 @@ func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
 	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	sconn, ok := conn.(*socket.Conn)
 	if !ok {
 		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
 	}
 
 	var to unix.Sockaddr
-	to4 := dst.IP.To4()
-	if to4 != nil {
+	if dst.Addr().Is4() {
 		to = &unix.SockaddrInet4{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
 	} else {
 		to = &unix.SockaddrInet6{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
 	}
 
 	txID := stun.NewTxID()
@@ -138,6 +138,32 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 
 }
 
-func supportsKernelTS() bool {
-	return true
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+		// TODO(jwhited): add ICMP
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	// we may restart faster than TIME_WAIT can clear
+	return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
 }

cmd/tailscale/depaware.txt

@@ -9,7 +9,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -152,9 +152,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dnscache+
         tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy+
         tailscale.com/util/ctxkey                                    from tailscale.com/types/logger
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/groupmember                               from tailscale.com/client/web
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -167,6 +169,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
@@ -191,7 +195,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
    W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
-        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli
+        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+

cmd/tailscaled/depaware.txt

@@ -90,7 +90,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 github.com/djherbis/times                                    from tailscale.com/drive/driveimpl
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/tstun+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -396,6 +396,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/cmd/tailscaled+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+

cmd/tailscaled/tailscaled.go

@@ -35,6 +35,7 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive/driveimpl"
 	"tailscale.com/envknob"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/conffile"
 	"tailscale.com/ipn/ipnlocal"
@@ -154,9 +155,11 @@ var beCLI func() // non-nil if CLI is linked in
 func main() {
 	envknob.PanicIfAnyEnvCheckedInInit()
 	envknob.ApplyDiskConfig()
+	applyIntegrationTestEnvKnob()
 
+	defaultVerbosity := envknob.RegisterInt("TS_LOG_VERBOSITY")
 	printVersion := false
-	flag.IntVar(&args.verbose, "verbose", 0, "log verbosity level; 0 is default, 1 or higher are increasingly verbose")
+	flag.IntVar(&args.verbose, "verbose", defaultVerbosity(), "log verbosity level; 0 is default, 1 or higher are increasingly verbose")
 	flag.BoolVar(&args.cleanUp, "cleanup", false, "clean up system state and exit")
 	flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server")
 	flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`)
@@ -895,3 +898,24 @@ func dieOnPipeReadErrorOfFD(fd int) {
 	f.Read(make([]byte, 1))
 	os.Exit(1)
 }
+
+// applyIntegrationTestEnvKnob applies the tailscaled.env=... environment
+// variables specified on the Linux kernel command line, if the VM is being
+// run in NATLab integration tests.
+//
+// They're specified as: tailscaled.env=FOO=bar tailscaled.env=BAR=baz
+func applyIntegrationTestEnvKnob() {
+	if runtime.GOOS != "linux" || !hostinfo.IsNATLabGuestVM() {
+		return
+	}
+	cmdLine, _ := os.ReadFile("/proc/cmdline")
+	for _, s := range strings.Fields(string(cmdLine)) {
+		suf, ok := strings.CutPrefix(s, "tailscaled.env=")
+		if !ok {
+			continue
+		}
+		if k, v, ok := strings.Cut(suf, "="); ok {
+			envknob.Setenv(k, v)
+		}
+	}
+}

cmd/tsidp/tsidp.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/rsa"
@@ -16,6 +17,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -25,6 +27,7 @@ import (
 	"net/netip"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +38,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -44,13 +48,22 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 	"tailscale.com/util/rands"
+	"tailscale.com/version"
 )
 
+// ctxConn is a key to look up a net.Conn stored in an HTTP request's context.
+type ctxConn struct{}
+
+// funnelClientsFile is the file where client IDs and secrets for OIDC clients
+// accessing the IDP over Funnel are persisted.
+const funnelClientsFile = "oidc-funnel-clients.json"
+
 var (
 	flagVerbose            = flag.Bool("verbose", false, "be verbose")
 	flagPort               = flag.Int("port", 443, "port to listen on")
 	flagLocalPort          = flag.Int("local-port", -1, "allow requests from localhost")
 	flagUseLocalTailscaled = flag.Bool("use-local-tailscaled", false, "use local tailscaled instead of tsnet")
+	flagFunnel             = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet")
 )
 
 func main() {
@@ -61,9 +74,11 @@ func main() {
 	}
 
 	var (
-		lc  *tailscale.LocalClient
-		st  *ipnstate.Status
-		err error
+		lc          *tailscale.LocalClient
+		st          *ipnstate.Status
+		err         error
+		watcherChan chan error
+		cleanup     func()
 
 		lns []net.Listener
 	)
@@ -90,6 +105,18 @@ func main() {
 		if !anySuccess {
 			log.Fatalf("failed to listen on any of %v", st.TailscaleIPs)
 		}
+
+		// tailscaled needs to be setting an HTTP header for funneled requests
+		// that older versions don't provide.
+		// TODO(naman): is this the correct check?
+		if *flagFunnel && !version.AtLeast(st.Version, "1.71.0") {
+			log.Fatalf("Local tailscaled not new enough to support -funnel. Update Tailscale or use tsnet mode.")
+		}
+		cleanup, watcherChan, err = serveOnLocalTailscaled(ctx, lc, st, uint16(*flagPort), *flagFunnel)
+		if err != nil {
+			log.Fatalf("could not serve on local tailscaled: %v", err)
+		}
+		defer cleanup()
 	} else {
 		ts := &tsnet.Server{
 			Hostname: "idp",
@@ -105,7 +132,15 @@ func main() {
 		if err != nil {
 			log.Fatalf("getting local client: %v", err)
 		}
-		ln, err := ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		var ln net.Listener
+		if *flagFunnel {
+			if err := ipn.CheckFunnelAccess(uint16(*flagPort), st.Self); err != nil {
+				log.Fatalf("%v", err)
+			}
+			ln, err = ts.ListenFunnel("tcp", fmt.Sprintf(":%d", *flagPort))
+		} else {
+			ln, err = ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		}
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -113,13 +148,26 @@ func main() {
 	}
 
 	srv := &idpServer{
-		lc: lc,
+		lc:          lc,
+		funnel:      *flagFunnel,
+		localTSMode: *flagUseLocalTailscaled,
 	}
 	if *flagPort != 443 {
 		srv.serverURL = fmt.Sprintf("https://%s:%d", strings.TrimSuffix(st.Self.DNSName, "."), *flagPort)
 	} else {
 		srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, "."))
 	}
+	if *flagFunnel {
+		f, err := os.Open(funnelClientsFile)
+		if err == nil {
+			srv.funnelClients = make(map[string]*funnelClient)
+			if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil {
+				log.Fatalf("could not parse %s: %v", funnelClientsFile, err)
+			}
+		} else if !errors.Is(err, os.ErrNotExist) {
+			log.Fatalf("could not open %s: %v", funnelClientsFile, err)
+		}
+	}
 
 	log.Printf("Running tsidp at %s ...", srv.serverURL)
 
@@ -134,35 +182,129 @@ func main() {
 	}
 
 	for _, ln := range lns {
-		go http.Serve(ln, srv)
+		server := http.Server{
+			Handler: srv,
+			ConnContext: func(ctx context.Context, c net.Conn) context.Context {
+				return context.WithValue(ctx, ctxConn{}, c)
+			},
+		}
+		go server.Serve(ln)
 	}
-	select {}
+	// need to catch os.Interrupt, otherwise deferred cleanup code doesn't run
+	exitChan := make(chan os.Signal, 1)
+	signal.Notify(exitChan, os.Interrupt)
+	select {
+	case <-exitChan:
+		log.Printf("interrupt, exiting")
+		return
+	case <-watcherChan:
+		if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
+			log.Printf("watcher closed, exiting")
+			return
+		}
+		log.Fatalf("watcher error: %v", err)
+		return
+	}
+}
+
+// serveOnLocalTailscaled starts a serve session using an already-running
+// tailscaled instead of starting a fresh tsnet server, making something
+// listening on clientDNSName:dstPort accessible over serve/funnel.
+func serveOnLocalTailscaled(ctx context.Context, lc *tailscale.LocalClient, st *ipnstate.Status, dstPort uint16, shouldFunnel bool) (cleanup func(), watcherChan chan error, err error) {
+	// In order to support funneling out in local tailscaled mode, we need
+	// to add a serve config to forward the listeners we bound above and
+	// allow those forwarders to be funneled out.
+	sc, err := lc.GetServeConfig(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get serve config: %v", err)
+	}
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+
+	// We watch the IPN bus just to get a session ID. The session expires
+	// when we stop watching the bus, and that auto-deletes the foreground
+	// serve/funnel configs we are creating below.
+	watcher, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not set up ipn bus watcher: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+	n, err := watcher.Next()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get initial state from ipn bus watcher: %v", err)
+	}
+	if n.SessionID == "" {
+		err = fmt.Errorf("missing sessionID in ipn.Notify")
+		return nil, nil, err
+	}
+	watcherChan = make(chan error)
+	go func() {
+		for {
+			_, err = watcher.Next()
+			if err != nil {
+				watcherChan <- err
+				return
+			}
+		}
+	}()
+
+	// Create a foreground serve config that gets cleaned up when tsidp
+	// exits and the session ID associated with this config is invalidated.
+	foregroundSc := new(ipn.ServeConfig)
+	mak.Set(&sc.Foreground, n.SessionID, foregroundSc)
+	serverURL := strings.TrimSuffix(st.Self.DNSName, ".")
+	fmt.Printf("setting funnel for %s:%v\n", serverURL, dstPort)
+
+	foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel)
+	foregroundSc.SetWebHandler(&ipn.HTTPHandler{
+		Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))),
+	}, serverURL, uint16(*flagPort), "/", true)
+	err = lc.SetServeConfig(ctx, sc)
+	if err != nil {
+		return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err)
+	}
+
+	return func() { watcher.Close() }, watcherChan, nil
 }
 
 type idpServer struct {
 	lc          *tailscale.LocalClient
 	loopbackURL string
 	serverURL   string // "https://foo.bar.ts.net"
+	funnel      bool
+	localTSMode bool
 
 	lazyMux        lazy.SyncValue[*http.ServeMux]
 	lazySigningKey lazy.SyncValue[*signingKey]
 	lazySigner     lazy.SyncValue[jose.Signer]
 
-	mu          sync.Mutex              // guards the fields below
-	code        map[string]*authRequest // keyed by random hex
-	accessToken map[string]*authRequest // keyed by random hex
+	mu            sync.Mutex               // guards the fields below
+	code          map[string]*authRequest  // keyed by random hex
+	accessToken   map[string]*authRequest  // keyed by random hex
+	funnelClients map[string]*funnelClient // keyed by client ID
 }
 
 type authRequest struct {
 	// localRP is true if the request is from a relying party running on the
-	// same machine as the idp server. It is mutually exclusive with rpNodeID.
+	// same machine as the idp server. It is mutually exclusive with rpNodeID
+	// and funnelRP.
 	localRP bool
 
 	// rpNodeID is the NodeID of the relying party (who requested the auth, such
 	// as Proxmox or Synology), not the user node who is being authenticated. It
-	// is mutually exclusive with localRP.
+	// is mutually exclusive with localRP and funnelRP.
 	rpNodeID tailcfg.NodeID
 
+	// funnelRP is non-nil if the request is from a relying party outside the
+	// tailnet, via Tailscale Funnel. It is mutually exclusive with rpNodeID
+	// and localRP.
+	funnelRP *funnelClient
+
 	// clientID is the "client_id" sent in the authorized request.
 	clientID string
 
@@ -181,9 +323,12 @@ type authRequest struct {
 	validTill time.Time
 }
 
-func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string, lc *tailscale.LocalClient) error {
+// allowRelyingParty validates that a relying party identified either by a
+// known remoteAddr or a valid client ID/secret pair is allowed to proceed
+// with the authorization flow associated with this authRequest.
+func (ar *authRequest) allowRelyingParty(r *http.Request, lc *tailscale.LocalClient) error {
 	if ar.localRP {
-		ra, err := netip.ParseAddrPort(remoteAddr)
+		ra, err := netip.ParseAddrPort(r.RemoteAddr)
 		if err != nil {
 			return err
 		}
@@ -192,7 +337,18 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 		}
 		return nil
 	}
-	who, err := lc.WhoIs(ctx, remoteAddr)
+	if ar.funnelRP != nil {
+		clientID, clientSecret, ok := r.BasicAuth()
+		if !ok {
+			clientID = r.FormValue("client_id")
+			clientSecret = r.FormValue("client_secret")
+		}
+		if ar.funnelRP.ID != clientID || ar.funnelRP.Secret != clientSecret {
+			return fmt.Errorf("tsidp: invalid client credentials")
+		}
+		return nil
+	}
+	who, err := lc.WhoIs(r.Context(), r.RemoteAddr)
 	if err != nil {
 		return fmt.Errorf("tsidp: error getting WhoIs: %w", err)
 	}
@@ -203,24 +359,60 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 }
 
 func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
-	who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr)
+	// This URL is visited by the user who is being authenticated. If they are
+	// visiting the URL over Funnel, that means they are not part of the
+	// tailnet that they are trying to be authenticated for.
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	uq := r.URL.Query()
+
+	redirectURI := uq.Get("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must specify redirect_uri", http.StatusBadRequest)
+		return
+	}
+
+	var remoteAddr string
+	if s.localTSMode {
+		// in local tailscaled mode, the local tailscaled is forwarding us
+		// HTTP requests, so reading r.RemoteAddr will just get us our own
+		// address.
+		remoteAddr = r.Header.Get("X-Forwarded-For")
+	} else {
+		remoteAddr = r.RemoteAddr
+	}
+	who, err := s.lc.WhoIs(r.Context(), remoteAddr)
 	if err != nil {
 		log.Printf("Error getting WhoIs: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	uq := r.URL.Query()
-
 	code := rands.HexString(32)
 	ar := &authRequest{
 		nonce:       uq.Get("nonce"),
 		remoteUser:  who,
-		redirectURI: uq.Get("redirect_uri"),
+		redirectURI: redirectURI,
 		clientID:    uq.Get("client_id"),
 	}
 
-	if r.URL.Path == "/authorize/localhost" {
+	if r.URL.Path == "/authorize/funnel" {
+		s.mu.Lock()
+		c, ok := s.funnelClients[ar.clientID]
+		s.mu.Unlock()
+		if !ok {
+			http.Error(w, "tsidp: invalid client ID", http.StatusBadRequest)
+			return
+		}
+		if ar.redirectURI != c.RedirectURI {
+			http.Error(w, "tsidp: redirect_uri mismatch", http.StatusBadRequest)
+			return
+		}
+		ar.funnelRP = c
+	} else if r.URL.Path == "/authorize/localhost" {
 		ar.localRP = true
 	} else {
 		var ok bool
@@ -237,8 +429,10 @@ func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
 
 	q := make(url.Values)
 	q.Set("code", code)
-	q.Set("state", uq.Get("state"))
-	u := uq.Get("redirect_uri") + "?" + q.Encode()
+	if state := uq.Get("state"); state != "" {
+		q.Set("state", state)
+	}
+	u := redirectURI + "?" + q.Encode()
 	log.Printf("Redirecting to %q", u)
 
 	http.Redirect(w, r, u, http.StatusFound)
@@ -251,6 +445,7 @@ func (s *idpServer) newMux() *http.ServeMux {
 	mux.HandleFunc("/authorize/", s.authorize)
 	mux.HandleFunc("/userinfo", s.serveUserInfo)
 	mux.HandleFunc("/token", s.serveToken)
+	mux.HandleFunc("/clients/", s.serveClients)
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
 			io.WriteString(w, "<html><body><h1>Tailscale OIDC IdP</h1>")
@@ -284,11 +479,6 @@ func (s *idpServer) serveUserInfo(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: invalid token", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
-		log.Printf("Error allowing relying party: %v", err)
-		http.Error(w, err.Error(), http.StatusForbidden)
-		return
-	}
 
 	if ar.validTill.Before(time.Now()) {
 		http.Error(w, "tsidp: token expired", http.StatusBadRequest)
@@ -348,7 +538,7 @@ func (s *idpServer) serveToken(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: code not found", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
+	if err := ar.allowRelyingParty(r, s.lc); err != nil {
 		log.Printf("Error allowing relying party: %v", err)
 		http.Error(w, err.Error(), http.StatusForbidden)
 		return
@@ -581,7 +771,9 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 	var authorizeEndpoint string
 	rpEndpoint := s.serverURL
-	if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
+	if isFunnelRequest(r) {
+		authorizeEndpoint = fmt.Sprintf("%s/authorize/funnel", s.serverURL)
+	} else if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
 		authorizeEndpoint = fmt.Sprintf("%s/authorize/%d", s.serverURL, who.Node.ID)
 	} else if ap.Addr().IsLoopback() {
 		rpEndpoint = s.loopbackURL
@@ -611,6 +803,148 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// funnelClient represents an OIDC client/relying party that is accessing the
+// IDP over Funnel.
+type funnelClient struct {
+	ID          string `json:"client_id"`
+	Secret      string `json:"client_secret,omitempty"`
+	Name        string `json:"name,omitempty"`
+	RedirectURI string `json:"redirect_uri"`
+}
+
+// /clients is a privileged endpoint that allows the visitor to create new
+// Funnel-capable OIDC clients, so it is only accessible over the tailnet.
+func (s *idpServer) serveClients(w http.ResponseWriter, r *http.Request) {
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	path := strings.TrimPrefix(r.URL.Path, "/clients/")
+
+	if path == "new" {
+		s.serveNewClient(w, r)
+		return
+	}
+
+	if path == "" {
+		s.serveGetClientsList(w, r)
+		return
+	}
+
+	s.mu.Lock()
+	c, ok := s.funnelClients[path]
+	s.mu.Unlock()
+	if !ok {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	switch r.Method {
+	case "DELETE":
+		s.serveDeleteClient(w, r, path)
+	case "GET":
+		json.NewEncoder(w).Encode(&funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	default:
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *idpServer) serveNewClient(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	redirectURI := r.FormValue("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must provide redirect_uri", http.StatusBadRequest)
+		return
+	}
+	clientID := rands.HexString(32)
+	clientSecret := rands.HexString(64)
+	newClient := funnelClient{
+		ID:          clientID,
+		Secret:      clientSecret,
+		Name:        r.FormValue("name"),
+		RedirectURI: redirectURI,
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.funnelClients, clientID, &newClient)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// delete the new client to avoid inconsistent state between memory
+		// and disk
+		delete(s.funnelClients, clientID)
+		return
+	}
+	json.NewEncoder(w).Encode(newClient)
+}
+
+func (s *idpServer) serveGetClientsList(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	redactedClients := make([]funnelClient, 0, len(s.funnelClients))
+	for _, c := range s.funnelClients {
+		redactedClients = append(redactedClients, funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	}
+	s.mu.Unlock()
+	json.NewEncoder(w).Encode(redactedClients)
+}
+
+func (s *idpServer) serveDeleteClient(w http.ResponseWriter, r *http.Request, clientID string) {
+	if r.Method != "DELETE" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.funnelClients == nil {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	if _, ok := s.funnelClients[clientID]; !ok {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	deleted := s.funnelClients[clientID]
+	delete(s.funnelClients, clientID)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// restore the deleted value to avoid inconsistent state between memory
+		// and disk
+		s.funnelClients[clientID] = deleted
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// storeFunnelClientsLocked writes the current mapping of OIDC client ID/secret
+// pairs for RPs that access the IDP over funnel. s.mu must be held while
+// calling this.
+func (s *idpServer) storeFunnelClientsLocked() error {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(s.funnelClients); err != nil {
+		return err
+	}
+	return os.WriteFile(funnelClientsFile, buf.Bytes(), 0600)
+}
+
 const (
 	minimumRSAKeySize = 2048
 )
@@ -700,3 +1034,24 @@ func parseID[T ~int64](input string) (_ T, ok bool) {
 	}
 	return T(i), true
 }
+
+// isFunnelRequest checks if an HTTP request is coming over Tailscale Funnel.
+func isFunnelRequest(r *http.Request) bool {
+	// If we're funneling through the local tailscaled, it will set this HTTP
+	// header.
+	if r.Header.Get("Tailscale-Funnel-Request") != "" {
+		return true
+	}
+
+	// If the funneled connection is from tsnet, then the net.Conn will be of
+	// type ipn.FunnelConn.
+	netConn := r.Context().Value(ctxConn{})
+	// if the conn is wrapped inside TLS, unwrap it
+	if tlsConn, ok := netConn.(*tls.Conn); ok {
+		netConn = tlsConn.NetConn()
+	}
+	if _, ok := netConn.(*ipn.FunnelConn); ok {
+		return true
+	}
+	return false
+}

cmd/tta/fw_linux.go

@@ -0,0 +1,128 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"encoding/binary"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/expr"
+	"tailscale.com/types/ptr"
+)
+
+func init() {
+	addFirewall = addFirewallLinux
+}
+
+func addFirewallLinux() error {
+	c, err := nftables.New()
+	if err != nil {
+		return err
+	}
+
+	// Create a new table
+	table := &nftables.Table{
+		Family: nftables.TableFamilyIPv4, // TableFamilyINet doesn't work (why?. oh well.)
+		Name:   "filter",
+	}
+	c.AddTable(table)
+
+	// Create a new chain for incoming traffic
+	inputChain := &nftables.Chain{
+		Name:     "input",
+		Table:    table,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookInput,
+		Priority: nftables.ChainPriorityFilter,
+		Policy:   ptr.To(nftables.ChainPolicyDrop),
+	}
+	c.AddChain(inputChain)
+
+	// Allow traffic from the loopback interface
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte("lo"),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	// Accept established and related connections
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Ct{
+				Register: 1,
+				Key:      expr.CtKeySTATE,
+			},
+			&expr.Bitwise{
+				SourceRegister: 1,
+				DestRegister:   1,
+				Len:            4,
+				Mask:           binary.NativeEndian.AppendUint32(nil, 0x06), // CT_STATE_BIT_ESTABLISHED | CT_STATE_BIT_RELATED
+				Xor:            binary.NativeEndian.AppendUint32(nil, 0),
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     binary.NativeEndian.AppendUint32(nil, 0x00),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	// Allow TCP packets in that don't have the SYN bit set, even if they're not
+	// ESTABLISHED or RELATED. This is because the test suite gets TCP
+	// connections up & idle (for HTTP) before it conditionally installs these
+	// firewall rules. But because conntrack wasn't previously active, existing
+	// TCP flows aren't ESTABLISHED and get dropped. So this rule allows
+	// previously established TCP connections that predates the firewall rules
+	// to continue working, as they don't have conntrack state.
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte{0x06}, // TCP
+			},
+			&expr.Payload{ // get TCP flags
+				DestRegister: 1,
+				Base:         2,
+				Offset:       13, // flags
+				Len:          1,
+			},
+			&expr.Bitwise{
+				SourceRegister: 1,
+				DestRegister:   1,
+				Len:            1,
+				Mask:           []byte{2}, // TCP_SYN
+				Xor:            []byte{0},
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     []byte{2}, // TCP_SYN
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	return c.Flush()
+}

cmd/tta/tta.go

@@ -0,0 +1,238 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The tta server is the Tailscale Test Agent.
+//
+// It runs on each Tailscale node being integration tested and permits the test
+// harness to control the node. It connects out to the test drver (rather than
+// accepting any TCP connections inbound, which might be blocked depending on
+// the scenario being tested) and then the test driver turns the TCP connection
+// around and sends request back.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/hostinfo"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+	"tailscale.com/version/distro"
+)
+
+var (
+	driverAddr = flag.String("driver", "test-driver.tailscale:8008", "address of the test driver; by default we use the DNS name test-driver.tailscale which is special cased in the emulated network's DNS server")
+)
+
+func absify(cmd string) string {
+	if distro.Get() == distro.Gokrazy && !strings.Contains(cmd, "/") {
+		return "/user/" + cmd
+	}
+	return cmd
+}
+
+func serveCmd(w http.ResponseWriter, cmd string, args ...string) {
+	log.Printf("Got serveCmd for %q %v", cmd, args)
+	out, err := exec.Command(absify(cmd), args...).CombinedOutput()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if err != nil {
+		w.Header().Set("Exec-Err", err.Error())
+		w.WriteHeader(500)
+		log.Printf("Err on serveCmd for %q %v, %d bytes of output: %v", cmd, args, len(out), err)
+	} else {
+		log.Printf("Did serveCmd for %q %v, %d bytes of output", cmd, args, len(out))
+	}
+	w.Write(out)
+}
+
+type localClientRoundTripper struct {
+	lc tailscale.LocalClient
+}
+
+func (rt *localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.RequestURI = ""
+	return rt.lc.DoLocalRequest(req)
+}
+
+func main() {
+	if distro.Get() == distro.Gokrazy {
+		if !hostinfo.IsNATLabGuestVM() {
+			// "Exiting immediately with status code 0 when the
+			// GOKRAZY_FIRST_START=1 environment variable is set means “don’t
+			// start the program on boot”"
+			return
+		}
+	}
+	flag.Parse()
+
+	if distro.Get() == distro.Gokrazy {
+		nsRx := regexp.MustCompile(`(?m)^nameserver (.*)`)
+		for t := time.Now(); time.Since(t) < 10*time.Second; time.Sleep(10 * time.Millisecond) {
+			all, _ := os.ReadFile("/etc/resolv.conf")
+			if nsRx.Match(all) {
+				break
+			}
+		}
+	}
+
+	logc, err := net.Dial("tcp", "9.9.9.9:124")
+	if err == nil {
+		log.SetOutput(logc)
+	}
+
+	log.Printf("Tailscale Test Agent running.")
+
+	gokRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+	gokRP.Transport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if network != "tcp" {
+				return nil, errors.New("unexpected network")
+			}
+			if addr != "gokrazy:80" {
+				return nil, errors.New("unexpected addr")
+			}
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", "/run/gokrazy-http.sock")
+		},
+	}
+
+	var ttaMux http.ServeMux // agent mux
+	var serveMux http.ServeMux
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("X-TTA-GoKrazy") == "1" {
+			gokRP.ServeHTTP(w, r)
+			return
+		}
+		ttaMux.ServeHTTP(w, r)
+	})
+	var hs http.Server
+	hs.Handler = &serveMux
+	var (
+		stMu   sync.Mutex
+		newSet = set.Set[net.Conn]{} // conns in StateNew
+	)
+	needConnCh := make(chan bool, 1)
+	hs.ConnState = func(c net.Conn, s http.ConnState) {
+		stMu.Lock()
+		defer stMu.Unlock()
+		oldLen := len(newSet)
+		switch s {
+		case http.StateNew:
+			newSet.Add(c)
+		default:
+			newSet.Delete(c)
+		}
+		if oldLen != 0 && len(newSet) == 0 {
+			select {
+			case needConnCh <- true:
+			default:
+			}
+		}
+	}
+	conns := make(chan net.Conn, 1)
+
+	lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock")))
+	lcRP.Transport = new(localClientRoundTripper)
+	ttaMux.HandleFunc("/localapi/", func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("Got localapi request: %v", r.URL)
+		t0 := time.Now()
+		lcRP.ServeHTTP(w, r)
+		log.Printf("Did localapi request in %v: %v", time.Since(t0).Round(time.Millisecond), r.URL)
+	})
+
+	ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "TTA\n")
+		return
+	})
+	ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale")
+	})
+	ttaMux.HandleFunc("/fw", addFirewallHandler)
+
+	go hs.Serve(chanListener(conns))
+
+	// For doing agent operations locally from gokrazy:
+	// (e.g. with "wget -O - localhost:8123/fw")
+	go func() {
+		err := http.ListenAndServe("127.0.0.1:8123", &ttaMux)
+		if err != nil {
+			log.Fatalf("ListenAndServe: %v", err)
+		}
+	}()
+
+	var lastErr string
+	needConnCh <- true
+	for {
+		<-needConnCh
+		c, err := connect()
+		if err != nil {
+			s := err.Error()
+			if s != lastErr {
+				log.Printf("Connect failure: %v", s)
+			}
+			lastErr = s
+			time.Sleep(time.Second)
+			continue
+		}
+		conns <- c
+	}
+}
+
+func connect() (net.Conn, error) {
+	c, err := net.Dial("tcp", *driverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+type chanListener <-chan net.Conn
+
+func (cl chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl
+	if !ok {
+		return nil, errors.New("closed")
+	}
+	return c, nil
+}
+
+func (cl chanListener) Close() error {
+	return nil
+}
+
+func (cl chanListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("52.0.0.34"), // TS..DR(iver)
+		Port: 123,
+	}
+}
+
+func addFirewallHandler(w http.ResponseWriter, r *http.Request) {
+	if addFirewall == nil {
+		http.Error(w, "firewall not supported", 500)
+		return
+	}
+	err := addFirewall()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	io.WriteString(w, "OK\n")
+}
+
+var addFirewall func() error // set by fw_linux.go

cmd/vnet/run-krazy.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "Type 'C-a c' to enter monitor; q to quit."
+
+set -eux
+qemu-system-x86_64 -M microvm,isa-serial=off \
+    -m 1G \
+    -nodefaults -no-user-config -nographic \
+    -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
+    -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1 tailscaled.env=TS_DEBUG_RAW_DISCO=1" \
+    -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/tsapp.img,format=raw \
+    -device virtio-blk-device,drive=blk0 \
+    -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \
+    -device virtio-serial-device \
+    -device virtio-net-device,netdev=net0,mac=52:cc:cc:cc:cc:00 \
+    -chardev stdio,id=virtiocon0,mux=on \
+    -device virtconsole,chardev=virtiocon0 \
+    -mon chardev=virtiocon0,mode=readline \
+    -audio none
+

cmd/vnet/vnet-main.go

@@ -0,0 +1,118 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The vnet binary runs a virtual network stack in userspace for qemu instances
+// to connect to and simulate various network conditions.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+)
+
+var (
+	listen  = flag.String("listen", "/tmp/qemu.sock", "path to listen on")
+	nat     = flag.String("nat", "easy", "type of NAT to use")
+	nat2    = flag.String("nat2", "hard", "type of NAT to use for second network")
+	portmap = flag.Bool("portmap", false, "enable portmapping")
+	dgram   = flag.Bool("dgram", false, "enable datagram mode; for use with macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment")
+)
+
+func main() {
+	flag.Parse()
+
+	if _, err := os.Stat(*listen); err == nil {
+		os.Remove(*listen)
+	}
+
+	var srv net.Listener
+	var err error
+	var conn *net.UnixConn
+	if *dgram {
+		addr, err := net.ResolveUnixAddr("unixgram", *listen)
+		if err != nil {
+			log.Fatalf("ResolveUnixAddr: %v", err)
+		}
+		conn, err = net.ListenUnixgram("unixgram", addr)
+		if err != nil {
+			log.Fatalf("ListenUnixgram: %v", err)
+		}
+		defer conn.Close()
+	} else {
+		srv, err = net.Listen("unix", *listen)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var c vnet.Config
+	node1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.NAT(*nat)))
+	c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", vnet.NAT(*nat2)))
+	if *portmap {
+		node1.Network().AddService(vnet.NATPMP)
+	}
+
+	s, err := vnet.New(&c)
+	if err != nil {
+		log.Fatalf("newServer: %v", err)
+	}
+
+	if err := s.PopulateDERPMapIPs(); err != nil {
+		log.Printf("warning: ignoring failure to populate DERP map: %v", err)
+	}
+
+	s.WriteStartingBanner(os.Stdout)
+	nc := s.NodeAgentClient(node1)
+	go func() {
+		rp := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+		d := rp.Director
+		rp.Director = func(r *http.Request) {
+			d(r)
+			r.Header.Set("X-TTA-GoKrazy", "1")
+		}
+		rp.Transport = nc.HTTPClient.Transport
+		http.ListenAndServe(":8080", rp)
+	}()
+	go func() {
+		getStatus := func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			st, err := nc.Status(ctx)
+			if err != nil {
+				log.Printf("NodeStatus: %v", err)
+				return
+			}
+			log.Printf("NodeStatus: %v", logger.AsJSON(st))
+		}
+		for {
+			time.Sleep(5 * time.Second)
+			//continue
+			getStatus()
+		}
+	}()
+
+	if conn != nil {
+		s.ServeUnixConn(conn, vnet.ProtocolUnixDGRAM)
+		return
+	}
+
+	for {
+		c, err := srv.Accept()
+		if err != nil {
+			log.Printf("Accept: %v", err)
+			continue
+		}
+		go s.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+	}
+}

derp/derphttp/derphttp_server.go

@@ -18,6 +18,7 @@ import (
 // following its HTTP request.
 const fastStartHeader = "Derp-Fast-Start"
 
+// Handler returns an http.Handler to be mounted at /derp, serving s.
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// These are installed both here and in cmd/derper. The check here
@@ -79,3 +80,29 @@ func ProbeHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
 	}
 }
+
+// ServeNoContent generates the /generate_204 response used by Tailscale's
+// captive portal detection.
+func ServeNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(NoContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(NoContentResponseHeader, "response "+challenge)
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
+const (
+	NoContentChallengeHeader = "X-Tailscale-Challenge"
+	NoContentResponseHeader  = "X-Tailscale-Response"
+)

flake.nix

@@ -120,4 +120,4 @@
   in
     flake-utils.lib.eachDefaultSystem (system: flakeForSystem nixpkgs system);
 }
-# nix-direnv cache busting line: sha256-1hekcJr1jEJFu4ZnapNkbAAv+8phTQuMloULIZ0f018=
+# nix-direnv cache busting line: sha256-AjiaBRP527d0gxb+fwh7hAgE1Jq6uHn/WDpQLfjK+eg=

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/golangci/golangci-lint v1.52.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.18.0
+	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
 	github.com/goreleaser/nfpm/v2 v2.33.1

go.mod.sri

@@ -1 +1 @@
-sha256-1hekcJr1jEJFu4ZnapNkbAAv+8phTQuMloULIZ0f018=
+sha256-AjiaBRP527d0gxb+fwh7hAgE1Jq6uHn/WDpQLfjK+eg=

go.sum

@@ -477,6 +477,8 @@ github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=

go.toolchain.rev

@@ -1 +1 @@
-2f152a4eff5875655a9a84fce8f8d329f8d9a321
+22ef9eb38e9a2d21b4a45f7adc75addb05f3efb8

gokrazy/Makefile

@@ -6,3 +6,6 @@ image:
 
 qemu: image
 	qemu-system-x86_64 -m 1G -drive file=tsapp.img,format=raw -boot d -netdev user,id=user.0 -device virtio-net-pci,netdev=user.0 -serial mon:stdio -audio none
+
+qcow2: image
+	qemu-img convert -O qcow2 tsapp.img tsapp.qcow2

gokrazy/go.mod

@@ -2,12 +2,12 @@ module tailscale.com/gokrazy
 
 go 1.22
 
-require github.com/gokrazy/tools v0.0.0-20240510170341-34b02e215bc2
+require github.com/gokrazy/tools v0.0.0-20240730192548-9f81add3a91e
 
 require (
 	github.com/breml/rootcerts v0.2.10 // indirect
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 // indirect
-	github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
 	github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 // indirect
 	github.com/google/renameio/v2 v2.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -20,6 +20,4 @@ require (
 
 replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
 
-replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf
-
-replace github.com/gokrazy/internal => github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd
+replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e

gokrazy/go.sum

@@ -3,6 +3,8 @@ github.com/breml/rootcerts v0.2.10/go.mod h1:24FDtzYMpqIeYC7QzaE8VPRQaFZU5TIUDly
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 h1:kBY5R1tSf+EYZ+QaSrofLaVJtBqYsVNVBWkdMq3Smcg=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2/go.mod h1:PYOvzGOL4nlBmuxu7IyKQTFLaxr61+WPRNRzVtuYOHw=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
@@ -17,10 +19,8 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd h1:ZJplHHhYSzxYmrXuDPCNChGRZbLkPqRkYqRBM7KNyng=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf h1:lmAGqLbIVoMK1TYWqJvxKFsu+Tb1OecgvXTmypZGAZY=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf/go.mod h1:+PSix9a8BHqAz6RV/9+tiE3C1ou0GA1ViR8pqAZVfwI=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e h1:3/xIc1QCvnKL7BCLng9od98HEvxCadjvqiI/bN+Twso=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e/go.mod h1:eTZ0QsugEPFU5UAQ/87bKMkPxQuTNa7+iFAIahOFwRg=
 golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.mod

@@ -2,6 +2,14 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect
+require (
+	github.com/gokrazy/gokrazy v0.0.0-20240802144848-676865a4e84f // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/kenshaw/evdev v0.1.0 // indirect
+	github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
 
-replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
+replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.sum

@@ -2,6 +2,8 @@ github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZ
 github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a h1:FKeN678rNpKTpWRdFbAhYL9mWzPu57R5XPXCR3WmXdI=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/kenshaw/evdev v0.1.0 h1:wmtceEOFfilChgdNT+c/djPJ2JineVsQ0N14kGzFRUo=
@@ -12,5 +14,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a h1:7dnA8x14JihQmKbPr++Y5CCN/XSyDmOB6cXUxcIj6VQ=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f h1:ZSAGWpgs+6dK2oIz5OR+HUul3oJbnhFn8YNgcZ3d9SQ=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+golang.org/x/sys v0.0.0-20201005065044-765f4ea38db3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod

@@ -2,4 +2,4 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 // indirect
+require github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e // indirect

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.sum

@@ -1,2 +1,4 @@
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 h1:xzf+cMvBJBcA/Av7OTWBa0Tjrbfcy00TeatJeJt6zrY=
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e h1:tyUUgeRPGHjCZWycRnhdx8Lx9DRkjl3WsVUxYMrVBOw=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=

gokrazy/tsapp/builddir/tailscale.com/go.sum

@@ -122,8 +122,12 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1 h1:ycpNCSYwzZ7x4G4ioPNtKQmIY0G/3o4pVf8wCZq6blY=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98 h1:RNpJrXfI5u6e+uzyIzvmnXbhmhdRkVf//90sMBH3lso=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -170,6 +174,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3 h1:/8/t5pz/mgdRXhYOIeqqYhFAQLE4DDGegc0Y4ZjyFJM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=

gokrazy/tsapp/config.json

@@ -1,16 +1,21 @@
 {
     "Hostname": "tsapp",
-    "Update": { "NoPassword": true },
+    "Update": {
+        "NoPassword": true
+    },
     "SerialConsole": "ttyS0,115200",
     "Packages": [
         "github.com/gokrazy/serial-busybox",
         "github.com/gokrazy/breakglass",
         "tailscale.com/cmd/tailscale",
-        "tailscale.com/cmd/tailscaled"
+        "tailscale.com/cmd/tailscaled",
+        "tailscale.com/cmd/tta"
     ],
     "PackageConfig": {
         "github.com/gokrazy/breakglass": {
-            "CommandLineFlags": [ "-authorized_keys=ec2" ]
+            "CommandLineFlags": [
+                "-authorized_keys=ec2"
+            ]
         },
         "tailscale.com/cmd/tailscale": {
             "ExtraFilePaths": {
@@ -21,4 +26,4 @@
     "KernelPackage": "github.com/tailscale/gokrazy-kernel",
     "FirmwarePackage": "github.com/tailscale/gokrazy-kernel",
     "InternalCompatibilityFlags": {}
-}
+}

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/lineread"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 )
 
 var started = time.Now()
@@ -462,3 +463,15 @@ func IsSELinuxEnforcing() bool {
 	out, _ := exec.Command("getenforce").Output()
 	return string(bytes.TrimSpace(out)) == "Enforcing"
 }
+
+// IsNATLabGuestVM reports whether the current host is a NAT Lab guest VM.
+func IsNATLabGuestVM() bool {
+	if runtime.GOOS == "linux" && distro.Get() == distro.Gokrazy {
+		cmdLine, _ := os.ReadFile("/proc/cmdline")
+		return bytes.Contains(cmdLine, []byte("tailscale-tta=1"))
+	}
+	return false
+}
+
+// NAT Lab VMs have a unique MAC address prefix.
+// See

ipn/ipnlocal/local.go

@@ -3781,7 +3781,7 @@ func (b *LocalBackend) TCPHandlerForDst(src, dst netip.AddrPort) (handler func(c
 			return nil
 		}, opts
 	}
-	if handler := b.tcpHandlerForServe(dst.Port(), src); handler != nil {
+	if handler := b.tcpHandlerForServe(dst.Port(), src, nil); handler != nil {
 		return handler, opts
 	}
 	return nil, nil

ipn/ipnlocal/serve.go

@@ -56,6 +56,16 @@ var serveHTTPContextKey ctxkey.Key[*serveHTTPContext]
 type serveHTTPContext struct {
 	SrcAddr  netip.AddrPort
 	DestPort uint16
+
+	// provides funnel-specific context, nil if not funneled
+	Funnel *funnelFlow
+}
+
+// funnelFlow represents a funneled connection initiated via IngressPeer
+// to Host.
+type funnelFlow struct {
+	Host        string
+	IngressPeer tailcfg.NodeView
 }
 
 // localListener is the state of host-level net.Listen for a specific (Tailscale IP, port)
@@ -91,7 +101,7 @@ func (b *LocalBackend) newServeListener(ctx context.Context, ap netip.AddrPort,
 
 		handler: func(conn net.Conn) error {
 			srcAddr := conn.RemoteAddr().(*net.TCPAddr).AddrPort()
-			handler := b.tcpHandlerForServe(ap.Port(), srcAddr)
+			handler := b.tcpHandlerForServe(ap.Port(), srcAddr, nil)
 			if handler == nil {
 				b.logf("[unexpected] local-serve: no handler for %v to port %v", srcAddr, ap.Port())
 				conn.Close()
@@ -382,7 +392,7 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 		return
 	}
 
-	_, port, err := net.SplitHostPort(string(target))
+	host, port, err := net.SplitHostPort(string(target))
 	if err != nil {
 		logf("got ingress conn for bad target %q; rejecting", target)
 		sendRST()
@@ -407,9 +417,10 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 			return
 		}
 	}
-	// TODO(bradfitz): pass ingressPeer etc in context to tcpHandlerForServe,
-	// extend serveHTTPContext or similar.
-	handler := b.tcpHandlerForServe(dport, srcAddr)
+	handler := b.tcpHandlerForServe(dport, srcAddr, &funnelFlow{
+		Host:        host,
+		IngressPeer: ingressPeer,
+	})
 	if handler == nil {
 		logf("[unexpected] no matching ingress serve handler for %v to port %v", srcAddr, dport)
 		sendRST()
@@ -424,8 +435,9 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 }
 
 // tcpHandlerForServe returns a handler for a TCP connection to be served via
-// the ipn.ServeConfig.
-func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort) (handler func(net.Conn) error) {
+// the ipn.ServeConfig. The funnelFlow can be nil if this is not a funneled
+// connection.
+func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort, f *funnelFlow) (handler func(net.Conn) error) {
 	b.mu.Lock()
 	sc := b.serveConfig
 	b.mu.Unlock()
@@ -444,6 +456,7 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort)
 			Handler: http.HandlerFunc(b.serveWebHandler),
 			BaseContext: func(_ net.Listener) context.Context {
 				return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{
+					Funnel:   f,
 					SrcAddr:  srcAddr,
 					DestPort: dport,
 				})
@@ -712,15 +725,20 @@ func (b *LocalBackend) addTailscaleIdentityHeaders(r *httputil.ProxyRequest) {
 	r.Out.Header.Del("Tailscale-User-Login")
 	r.Out.Header.Del("Tailscale-User-Name")
 	r.Out.Header.Del("Tailscale-User-Profile-Pic")
+	r.Out.Header.Del("Tailscale-Funnel-Request")
 	r.Out.Header.Del("Tailscale-Headers-Info")
 
 	c, ok := serveHTTPContextKey.ValueOk(r.Out.Context())
 	if !ok {
 		return
 	}
+	if c.Funnel != nil {
+		r.Out.Header.Set("Tailscale-Funnel-Request", "?1")
+		return
+	}
 	node, user, ok := b.WhoIs("tcp", c.SrcAddr)
 	if !ok {
-		return // traffic from outside of Tailnet (funneled)
+		return // traffic from outside of Tailnet (funneled or local machine)
 	}
 	if node.IsTagged() {
 		// 2023-06-14: Not setting identity headers for tagged nodes.

logpolicy/logpolicy.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/envknob"
 	"tailscale.com/health"
+	"tailscale.com/hostinfo"
 	"tailscale.com/log/filelogger"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
@@ -463,6 +464,11 @@ func New(collection string, netMon *netmon.Monitor, health *health.Tracker, logf
 // The netMon parameter is optional. It should be specified in environments where
 // Tailscaled is manipulating the routing table.
 func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, health *health.Tracker, logf logger.Logf) *Policy {
+	if hostinfo.IsNATLabGuestVM() {
+		// In NATLab Gokrazy instances, tailscaled comes up concurently with
+		// DHCP and the doesn't have DNS for a while. Wait for DHCP first.
+		awaitGokrazyNetwork()
+	}
 	var lflags int
 	if term.IsTerminal(2) || runtime.GOOS == "windows" {
 		lflags = 0
@@ -816,3 +822,25 @@ func (noopPretendSuccessTransport) RoundTrip(req *http.Request) (*http.Response,
 		Status:     "200 OK",
 	}, nil
 }
+
+func awaitGokrazyNetwork() {
+	if runtime.GOOS != "linux" || distro.Get() != distro.Gokrazy {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	for {
+		// Before DHCP finishes, the /etc/resolv.conf file has just "#MANUAL".
+		all, _ := os.ReadFile("/etc/resolv.conf")
+		if bytes.Contains(all, []byte("nameserver ")) {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(500 * time.Millisecond):
+		}
+	}
+}

metrics/multilabelmap.go

@@ -13,6 +13,13 @@ import (
 	"sync"
 )
 
+// MetricType is a Prometheus metric type. At the moment we only support
+// counter and gauge.
+type MetricType string
+
+const Counter MetricType = "counter"
+const Gauge MetricType = "gauge"
+
 // MultiLabelMap is a struct-value-to-Var map variable that satisfies the
 // [expvar.Var] interface but also allows for multiple Prometheus labels to be
 // associated with each value.
@@ -22,7 +29,7 @@ import (
 // The struct fields must all be strings, and the string values must be valid
 // Prometheus label values without requiring quoting.
 type MultiLabelMap[T comparable] struct {
-	Type string // optional Prometheus type ("counter", "gauge")
+	Type MetricType
 	Help string // optional Prometheus help string
 
 	m sync.Map // map[T]expvar.Var
@@ -33,7 +40,7 @@ type MultiLabelMap[T comparable] struct {
 
 // NewMultiLabelMap creates and publishes (via expvar.Publish) a new
 // MultiLabelMap[T] variable with the given name and returns it.
-func NewMultiLabelMap[T comparable](name string, promType, helpText string) *MultiLabelMap[T] {
+func NewMultiLabelMap[T comparable](name string, promType MetricType, helpText string) *MultiLabelMap[T] {
 	m := &MultiLabelMap[T]{
 		Type: promType,
 		Help: helpText,
@@ -105,7 +112,7 @@ func (v *MultiLabelMap[T]) WritePrometheus(w io.Writer, name string) {
 		io.WriteString(w, "# TYPE ")
 		io.WriteString(w, name)
 		io.WriteString(w, " ")
-		io.WriteString(w, v.Type)
+		io.WriteString(w, string(v.Type))
 		io.WriteString(w, "\n")
 	}
 	if v.Help != "" {

metrics/multilabelmap_test.go

@@ -82,7 +82,7 @@ func TestMultiLabelMapTypes(t *testing.T) {
 	}
 
 	m := new(MultiLabelMap[LabelTypes])
-	m.Type = "counter"
+	m.Type = Counter
 	m.Help = "some good stuff"
 	m.Add(LabelTypes{"a", true, -1, 2}, 3)
 	var buf bytes.Buffer

net/dns/resolver/forwarder.go

@@ -6,6 +6,8 @@ package resolver
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -498,9 +500,10 @@ var (
 func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDelay) (ret []byte, err error) {
 	if verboseDNSForward() {
 		id := forwarderCount.Add(1)
-		f.logf("forwarder.send(%q) [%d] ...", rr.name.Addr, id)
+		domain, typ, _ := nameFromQuery(fq.packet)
+		f.logf("forwarder.send(%q, %d, %v, %d) [%d] ...", rr.name.Addr, fq.txid, typ, len(domain), id)
 		defer func() {
-			f.logf("forwarder.send(%q) [%d] = %v, %v", rr.name.Addr, id, len(ret), err)
+			f.logf("forwarder.send(%q, %d, %v, %d) [%d] = %v, %v", rr.name.Addr, fq.txid, typ, len(domain), id, len(ret), err)
 		}()
 	}
 	if strings.HasPrefix(rr.name.Addr, "http://") {
@@ -866,7 +869,7 @@ type forwardQuery struct {
 // node DNS proxy queries), otherwise f.resolvers is used.
 func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, responseChan chan<- packet, resolvers ...resolverAndDelay) error {
 	metricDNSFwd.Add(1)
-	domain, err := nameFromQuery(query.bs)
+	domain, typ, err := nameFromQuery(query.bs)
 	if err != nil {
 		metricDNSFwdErrorName.Add(1)
 		return err
@@ -943,6 +946,12 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 	}
 	defer fq.closeOnCtxDone.Close()
 
+	if verboseDNSForward() {
+		domainSha256 := sha256.Sum256([]byte(domain))
+		domainSig := base64.RawStdEncoding.EncodeToString(domainSha256[:3])
+		f.logf("request(%d, %v, %d, %s) %d...", fq.txid, typ, len(domain), domainSig, len(fq.packet))
+	}
+
 	resc := make(chan []byte, 1) // it's fine buffered or not
 	errc := make(chan error, 1)  // it's fine buffered or not too
 	for i := range resolvers {
@@ -982,6 +991,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 				metricDNSFwdErrorContext.Add(1)
 				return fmt.Errorf("waiting to send response: %w", ctx.Err())
 			case responseChan <- packet{v, query.family, query.addr}:
+				if verboseDNSForward() {
+					f.logf("response(%d, %v, %d) = %d, nil", fq.txid, typ, len(domain), len(v))
+				}
 				metricDNSFwdSuccess.Add(1)
 				f.health.SetHealthy(dnsForwarderFailing)
 				return nil
@@ -1009,6 +1021,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 						}
 						f.health.SetUnhealthy(dnsForwarderFailing, health.Args{health.ArgDNSServers: strings.Join(resolverAddrs, ",")})
 					case responseChan <- res:
+						if verboseDNSForward() {
+							f.logf("forwarder response(%d, %v, %d) = %d, %v", fq.txid, typ, len(domain), len(res.bs), firstErr)
+						}
 					}
 				}
 				return firstErr
@@ -1037,24 +1052,28 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 var initListenConfig func(_ *net.ListenConfig, _ *netmon.Monitor, tunName string) error
 
 // nameFromQuery extracts the normalized query name from bs.
-func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
+func nameFromQuery(bs []byte) (dnsname.FQDN, dns.Type, error) {
 	var parser dns.Parser
 
 	hdr, err := parser.Start(bs)
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
 	if hdr.Response {
-		return "", errNotQuery
+		return "", 0, errNotQuery
 	}
 
 	q, err := parser.Question()
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
 
 	n := q.Name.Data[:q.Name.Length]
-	return dnsname.ToFQDN(rawNameToLower(n))
+	fqdn, err := dnsname.ToFQDN(rawNameToLower(n))
+	if err != nil {
+		return "", 0, err
+	}
+	return fqdn, q.Type, nil
 }
 
 // nxDomainResponse returns an NXDomain DNS reply for the provided request.

net/dns/resolver/forwarder_test.go

@@ -201,7 +201,7 @@ func BenchmarkNameFromQuery(b *testing.B) {
 	b.ResetTimer()
 	b.ReportAllocs()
 	for range b.N {
-		_, err := nameFromQuery(msg)
+		_, _, err := nameFromQuery(msg)
 		if err != nil {
 			b.Fatal(err)
 		}

net/tlsdial/tlsdial.go

@@ -26,6 +26,7 @@ import (
 
 	"tailscale.com/envknob"
 	"tailscale.com/health"
+	"tailscale.com/hostinfo"
 )
 
 var counterFallbackOK int32 // atomic
@@ -77,6 +78,12 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config {
 	// (with the baked-in fallback root) in the VerifyConnection hook.
 	conf.InsecureSkipVerify = true
 	conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) {
+		if host == "log.tailscale.io" && hostinfo.IsNATLabGuestVM() {
+			// Allow log.tailscale.io TLS MITM for integration tests when
+			// the client's running within a NATLab VM.
+			return nil
+		}
+
 		// Perform some health checks on this certificate before we do
 		// any verification.
 		var selfSignedIssuer string

prober/prober.go

@@ -496,7 +496,8 @@ func (p *Prober) RunHandler(w http.ResponseWriter, r *http.Request) error {
 		return nil
 	}
 
-	stats := fmt.Sprintf("Previous runs: success rate %d%%, median latency %v",
+	stats := fmt.Sprintf("Last %d probes: success rate %d%%, median latency %v\n",
+		len(prevInfo.RecentResults),
 		int(prevInfo.RecentSuccessRatio()*100), prevInfo.RecentMedianLatency())
 	if err != nil {
 		return tsweb.Error(respStatus, fmt.Sprintf("Probe failed: %s\n%s", err.Error(), stats), err)

prober/status.go

@@ -86,7 +86,7 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc
 			}
 			s := probeStatus{ProbeInfo: info}
 			if !info.End.IsZero() {
-				s.TimeSinceLast = time.Since(info.End)
+				s.TimeSinceLast = time.Since(info.End).Truncate(time.Second)
 			}
 			for textTpl, urlTpl := range params.probeLinks {
 				text, err := renderTemplate(textTpl, info)

prober/status.html

@@ -71,12 +71,12 @@
     <table class="sortable">
         <thead><tr>
             <th>Name</th>
-            <th>Class & Labels</th>
+            <th>Probe Class & Labels</th>
             <th>Interval</th>
-            <th>Result</th>
+            <th>Last Attempt</th>
             <th>Success</th>
             <th>Latency</th>
-            <th>Error</th>
+            <th>Last Error</th>
         </tr></thead>
         <tbody>
         {{range $name, $probeInfo := .Probes}}
@@ -100,8 +100,8 @@
             <td>{{$probeInfo.Interval}}</td>
             <td data-sort="{{$probeInfo.TimeSinceLast.Milliseconds}}">
                 {{if $probeInfo.TimeSinceLast}}
-                    {{$probeInfo.TimeSinceLast.String}}<br/>
-                    <span class="small">{{$probeInfo.End}}</span>
+                    {{$probeInfo.TimeSinceLast.String}} ago<br/>
+                    <span class="small">{{$probeInfo.End.Format "2006-01-02T15:04:05Z07:00"}}</span>
                 {{else}}
                     Never
                 {{end}}

shell.nix

@@ -16,4 +16,4 @@
 ) {
   src =  ./.;
 }).shellNix
-# nix-direnv cache busting line: sha256-1hekcJr1jEJFu4ZnapNkbAAv+8phTQuMloULIZ0f018=
+# nix-direnv cache busting line: sha256-AjiaBRP527d0gxb+fwh7hAgE1Jq6uHn/WDpQLfjK+eg=

tstest/integration/integration.go

@@ -190,6 +190,7 @@ func RunDERPAndSTUN(t testing.TB, logf logger.Logf, ipAddress string) (derpMap *
 	}
 
 	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Listener.Close()
 	httpsrv.Listener = ln
 	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
 	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))

tstest/integration/nat/nat_test.go

@@ -0,0 +1,620 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package nat
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/mod/modfile"
+	"golang.org/x/sync/errgroup"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/syncs"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+var (
+	logTailscaled = flag.Bool("log-tailscaled", false, "log tailscaled output")
+	pcapFile      = flag.String("pcap", "", "write pcap to file")
+)
+
+type natTest struct {
+	tb      testing.TB
+	base    string // base image
+	tempDir string // for qcow2 images
+	vnet    *vnet.Server
+	kernel  string // linux kernel path
+
+	gotRoute pingRoute
+}
+
+func newNatTest(tb testing.TB) *natTest {
+	root, err := os.Getwd()
+	if err != nil {
+		tb.Fatal(err)
+	}
+	modRoot := filepath.Join(root, "../../..")
+
+	nt := &natTest{
+		tb:      tb,
+		tempDir: tb.TempDir(),
+		base:    filepath.Join(modRoot, "gokrazy/tsapp.qcow2"),
+	}
+
+	if _, err := os.Stat(nt.base); err != nil {
+		tb.Skipf("skipping test; base image %q not found", nt.base)
+	}
+
+	nt.kernel, err = findKernelPath(filepath.Join(modRoot, "gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod"))
+	if err != nil {
+		tb.Skipf("skipping test; kernel not found: %v", err)
+	}
+	tb.Logf("found kernel: %v", nt.kernel)
+
+	return nt
+}
+
+func findKernelPath(goMod string) (string, error) {
+	b, err := os.ReadFile(goMod)
+	if err != nil {
+		return "", err
+	}
+	mf, err := modfile.Parse("go.mod", b, nil)
+	if err != nil {
+		return "", err
+	}
+	goModB, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	for _, r := range mf.Require {
+		if r.Mod.Path == "github.com/tailscale/gokrazy-kernel" {
+			return strings.TrimSpace(string(goModB)) + "/" + r.Mod.String() + "/vmlinuz", nil
+		}
+	}
+	return "", fmt.Errorf("failed to find kernel in %v", goMod)
+}
+
+type addNodeFunc func(c *vnet.Config) *vnet.Node // returns nil to omit test
+
+func easy(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT))
+}
+
+// easy + host firewall
+func easyFW(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(vnet.HostFirewall, c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT))
+}
+
+func easyAF(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyAFNAT))
+}
+
+func sameLAN(c *vnet.Config) *vnet.Node {
+	nw := c.FirstNetwork()
+	if nw == nil {
+		return nil
+	}
+	if !nw.CanTakeMoreNodes() {
+		return nil
+	}
+	return c.AddNode(nw)
+}
+
+func one2one(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("172.16.%d.1/24", n), vnet.One2OneNAT))
+}
+
+func easyPMP(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
+}
+
+// easy + port mapping + host firewall
+func easyPMPFW(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(vnet.HostFirewall,
+		c.AddNetwork(
+			fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+			fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
+}
+
+// easy + port mapping + host firewall - BPF
+func easyPMPFWNoBPF(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(
+		vnet.HostFirewall,
+		vnet.TailscaledEnv{
+			Key:   "TS_DEBUG_DISABLE_RAW_DISCO",
+			Value: "1",
+		},
+		c.AddNetwork(
+			fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+			fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
+}
+
+func hard(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("10.0.%d.1/24", n), vnet.HardNAT))
+}
+
+func hardPMP(c *vnet.Config) *vnet.Node {
+	n := c.NumNodes() + 1
+	return c.AddNode(c.AddNetwork(
+		fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+		fmt.Sprintf("10.7.%d.1/24", n), vnet.HardNAT, vnet.NATPMP))
+}
+
+func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute {
+	t := nt.tb
+
+	var c vnet.Config
+	c.SetPCAPFile(*pcapFile)
+	nodes := []*vnet.Node{
+		node1(&c),
+		node2(&c),
+	}
+	if nodes[0] == nil || nodes[1] == nil {
+		t.Skip("skipping test; not applicable combination")
+	}
+
+	var err error
+	nt.vnet, err = vnet.New(&c)
+	if err != nil {
+		t.Fatalf("newServer: %v", err)
+	}
+	nt.tb.Cleanup(func() {
+		nt.vnet.Close()
+	})
+
+	var wg sync.WaitGroup // waiting for srv.Accept goroutine
+	defer wg.Wait()
+
+	sockAddr := filepath.Join(nt.tempDir, "qemu.sock")
+	srv, err := net.Listen("unix", sockAddr)
+	if err != nil {
+		t.Fatalf("Listen: %v", err)
+	}
+	defer srv.Close()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			c, err := srv.Accept()
+			if err != nil {
+				return
+			}
+			go nt.vnet.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+		}
+	}()
+
+	for i, node := range nodes {
+		disk := fmt.Sprintf("%s/node-%d.qcow2", nt.tempDir, i)
+		out, err := exec.Command("qemu-img", "create",
+			"-f", "qcow2",
+			"-F", "qcow2",
+			"-b", nt.base,
+			disk).CombinedOutput()
+		if err != nil {
+			t.Fatalf("qemu-img create: %v, %s", err, out)
+		}
+
+		var envBuf bytes.Buffer
+		for _, e := range node.Env() {
+			fmt.Fprintf(&envBuf, " tailscaled.env=%s=%s", e.Key, e.Value)
+		}
+		envStr := envBuf.String()
+
+		cmd := exec.Command("qemu-system-x86_64",
+			"-M", "microvm,isa-serial=off",
+			"-m", "384M",
+			"-nodefaults", "-no-user-config", "-nographic",
+			"-kernel", nt.kernel,
+			"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1"+envStr,
+			"-drive", "id=blk0,file="+disk+",format=qcow2",
+			"-device", "virtio-blk-device,drive=blk0",
+			"-netdev", "stream,id=net0,addr.type=unix,addr.path="+sockAddr,
+			"-device", "virtio-serial-device",
+			"-device", "virtio-net-device,netdev=net0,mac="+node.MAC().String(),
+			"-chardev", "stdio,id=virtiocon0,mux=on",
+			"-device", "virtconsole,chardev=virtiocon0",
+			"-mon", "chardev=virtiocon0,mode=readline",
+			"-audio", "none",
+		)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Start(); err != nil {
+			t.Fatalf("qemu: %v", err)
+		}
+		nt.tb.Cleanup(func() {
+			cmd.Process.Kill()
+			cmd.Wait()
+		})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	lc1 := nt.vnet.NodeAgentClient(nodes[0])
+	lc2 := nt.vnet.NodeAgentClient(nodes[1])
+	clients := []*vnet.NodeAgentClient{lc1, lc2}
+
+	var eg errgroup.Group
+	var sts [2]*ipnstate.Status
+	for i, c := range clients {
+		i, c := i, c
+		eg.Go(func() error {
+			if *logTailscaled {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					streamDaemonLogs(ctx, t, c, fmt.Sprintf("node%d:", i))
+				}()
+			}
+			st, err := c.Status(ctx)
+			if err != nil {
+				return fmt.Errorf("node%d status: %w", i, err)
+			}
+			t.Logf("node%d status: %v", i, st)
+
+			node := nodes[i]
+			if node.HostFirewall() {
+				if err := c.EnableHostFirewall(ctx); err != nil {
+					return fmt.Errorf("node%d firewall: %w", i, err)
+				}
+				t.Logf("node%d firewalled", i)
+			}
+
+			if err := up(ctx, c); err != nil {
+				return fmt.Errorf("node%d up: %w", i, err)
+			}
+			t.Logf("node%d up!", i)
+
+			st, err = c.Status(ctx)
+			if err != nil {
+				return fmt.Errorf("node%d status: %w", i, err)
+			}
+			sts[i] = st
+
+			if st.BackendState != "Running" {
+				return fmt.Errorf("node%d state = %q", i, st.BackendState)
+			}
+			t.Logf("node%d up with %v", i, sts[i].Self.TailscaleIPs)
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		t.Fatalf("initial setup: %v", err)
+	}
+
+	defer nt.vnet.Close()
+
+	pingRes, err := ping(ctx, lc1, sts[1].Self.TailscaleIPs[0])
+	if err != nil {
+		t.Fatalf("ping failure: %v", err)
+	}
+	nt.gotRoute = classifyPing(pingRes)
+	t.Logf("ping route: %v", nt.gotRoute)
+
+	return nt.gotRoute
+}
+
+func classifyPing(pr *ipnstate.PingResult) pingRoute {
+	if pr == nil {
+		return routeNil
+	}
+	if pr.Endpoint != "" {
+		ap, err := netip.ParseAddrPort(pr.Endpoint)
+		if err == nil {
+			if ap.Addr().IsPrivate() {
+				return routeLocal
+			}
+			return routeDirect
+		}
+	}
+	return routeDERP // presumably
+}
+
+type pingRoute string
+
+const (
+	routeDERP   pingRoute = "derp"
+	routeLocal  pingRoute = "local"
+	routeDirect pingRoute = "direct"
+	routeNil    pingRoute = "nil" // *ipnstate.PingResult is nil
+)
+
+func streamDaemonLogs(ctx context.Context, t testing.TB, c *vnet.NodeAgentClient, nodeID string) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	r, err := c.TailDaemonLogs(ctx)
+	if err != nil {
+		t.Errorf("tailDaemonLogs: %v", err)
+		return
+	}
+	logger := log.New(os.Stderr, nodeID+" ", log.Lmsgprefix)
+	dec := json.NewDecoder(r)
+	for {
+		// /{"logtail":{"client_time":"2024-08-08T17:42:31.95095956Z","proc_id":2024742977,"proc_seq":232},"text":"magicsock: derp-1 connected; connGen=1\n"}
+		var logEntry struct {
+			LogTail struct {
+				ClientTime time.Time `json:"client_time"`
+			}
+			Text string `json:"text"`
+		}
+		if err := dec.Decode(&logEntry); err != nil {
+			if err == io.EOF || errors.Is(err, context.Canceled) {
+				return
+			}
+			t.Errorf("log entry: %v", err)
+			return
+		}
+		logger.Printf("%s %s", logEntry.LogTail.ClientTime.Format("2006/01/02 15:04:05"), logEntry.Text)
+	}
+}
+
+func ping(ctx context.Context, c *vnet.NodeAgentClient, target netip.Addr) (*ipnstate.PingResult, error) {
+	n := 0
+	var res *ipnstate.PingResult
+	anyPong := false
+	for n < 10 {
+		n++
+		pr, err := c.PingWithOpts(ctx, target, tailcfg.PingDisco, tailscale.PingOpts{})
+		if err != nil {
+			if anyPong {
+				return res, nil
+			}
+			return nil, err
+		}
+		if pr.Err != "" {
+			return nil, errors.New(pr.Err)
+		}
+		if pr.DERPRegionID == 0 {
+			return pr, nil
+		}
+		res = pr
+		select {
+		case <-ctx.Done():
+		case <-time.After(time.Second):
+		}
+	}
+	if res == nil {
+		return nil, errors.New("no ping response")
+	}
+	return res, nil
+}
+
+func up(ctx context.Context, c *vnet.NodeAgentClient) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://unused/up", nil)
+	if err != nil {
+		return err
+	}
+	res, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	all, _ := io.ReadAll(res.Body)
+	if res.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code %v: %s", res.Status, all)
+	}
+	return nil
+}
+
+type nodeType struct {
+	name string
+	fn   addNodeFunc
+}
+
+var types = []nodeType{
+	{"easy", easy},
+	{"easyAF", easyAF},
+	{"hard", hard},
+	{"easyPMP", easyPMP},
+	{"hardPMP", hardPMP},
+	{"one2one", one2one},
+	{"sameLAN", sameLAN},
+}
+
+// want sets the expected ping route for the test.
+func (nt *natTest) want(r pingRoute) {
+	if nt.gotRoute != r {
+		nt.tb.Errorf("ping route = %v; want %v", nt.gotRoute, r)
+	}
+}
+
+func TestEasyEasy(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, easy)
+	nt.want(routeDirect)
+}
+
+func TestSameLAN(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, sameLAN)
+	nt.want(routeLocal)
+}
+
+// TestBPFDisco tests https://github.com/tailscale/tailscale/issues/3824 ...
+// * server behind a Hard NAT
+// * client behind a NAT with UPnP support
+// * client machine has a stateful host firewall (e.g. ufw)
+func TestBPFDisco(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easyPMPFW, hard)
+	nt.want(routeDirect)
+}
+
+func TestHostFWNoBPF(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easyPMPFWNoBPF, hard)
+	nt.want(routeDERP)
+}
+
+func TestHostFWPair(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easyFW, easyFW)
+	nt.want(routeDirect)
+}
+
+func TestOneHostFW(t *testing.T) {
+	nt := newNatTest(t)
+	nt.runTest(easy, easyFW)
+	nt.want(routeDirect)
+}
+
+var pair = flag.String("pair", "", "comma-separated pair of types to test (easy, easyAF, hard, easyPMP, hardPMP, one2one, sameLAN)")
+
+func TestPair(t *testing.T) {
+	t1, t2, ok := strings.Cut(*pair, ",")
+	if !ok {
+		t.Skipf("skipping test without --pair=type1,type2 set")
+	}
+	find := func(name string) addNodeFunc {
+		for _, nt := range types {
+			if nt.name == name {
+				return nt.fn
+			}
+		}
+		t.Fatalf("unknown type %q", name)
+		return nil
+	}
+
+	nt := newNatTest(t)
+	nt.runTest(find(t1), find(t2))
+}
+
+var runGrid = flag.Bool("run-grid", false, "run grid test")
+
+func TestGrid(t *testing.T) {
+	if !*runGrid {
+		t.Skip("skipping grid test; set --run-grid to run")
+	}
+	t.Parallel()
+
+	sem := syncs.NewSemaphore(2)
+	var (
+		mu  sync.Mutex
+		res = make(map[string]pingRoute)
+	)
+	for _, a := range types {
+		for _, b := range types {
+			key := a.name + "-" + b.name
+			keyBack := b.name + "-" + a.name
+			t.Run(key, func(t *testing.T) {
+				t.Parallel()
+
+				sem.Acquire()
+				defer sem.Release()
+
+				filename := key + ".cache"
+				contents, _ := os.ReadFile(filename)
+				if len(contents) == 0 {
+					filename2 := keyBack + ".cache"
+					contents, _ = os.ReadFile(filename2)
+				}
+				route := pingRoute(strings.TrimSpace(string(contents)))
+
+				if route == "" {
+					nt := newNatTest(t)
+					route = nt.runTest(a.fn, b.fn)
+					if err := os.WriteFile(filename, []byte(string(route)), 0666); err != nil {
+						t.Fatalf("writeFile: %v", err)
+					}
+				}
+
+				mu.Lock()
+				defer mu.Unlock()
+				res[key] = route
+				t.Logf("results: %v", res)
+			})
+		}
+	}
+
+	t.Cleanup(func() {
+		mu.Lock()
+		defer mu.Unlock()
+		var hb bytes.Buffer
+		pf := func(format string, args ...any) {
+			fmt.Fprintf(&hb, format, args...)
+		}
+		rewrite := func(s string) string {
+			return strings.ReplaceAll(s, "PMP", "+pm")
+		}
+		pf("<html><table border=1 cellpadding=5>")
+		pf("<tr><td></td>")
+		for _, a := range types {
+			pf("<td><b>%s</b></td>", rewrite(a.name))
+		}
+		pf("</tr>\n")
+
+		for _, a := range types {
+			if a.name == "sameLAN" {
+				continue
+			}
+			pf("<tr><td><b>%s</b></td>", rewrite(a.name))
+			for _, b := range types {
+				key := a.name + "-" + b.name
+				key2 := b.name + "-" + a.name
+				v := cmp.Or(res[key], res[key2], "-")
+				if v == "derp" {
+					pf("<td><div style='color: red; font-weight: bold'>%s</div></td>", v)
+				} else if v == "local" {
+					pf("<td><div style='color: green; font-weight: bold'>%s</div></td>", v)
+				} else {
+					pf("<td>%s</td>", v)
+				}
+			}
+			pf("</tr>\n")
+		}
+		pf("</table>")
+		pf("<b>easy</b>: Endpoint-Independent Mapping, Address and Port-Dependent Filtering (e.g. Linux, Google Wifi, Unifi, eero)<br>")
+		pf("<b>easyAF</b>: Endpoint-Independent Mapping, Address-Dependent Filtering (James says telephony things or Zyxel type things)<br>")
+		pf("<b>hard</b>: Address and Port-Dependent Mapping, Address and Port-Dependent Filtering (FreeBSD, OPNSense, pfSense)<br>")
+		pf("<b>one2one</b>: One-to-One NAT (e.g. an EC2 instance with a public IPv4)<br>")
+		pf("<b>x+pm</b>: x, with port mapping (NAT-PMP, PCP, UPnP, etc)<br>")
+		pf("<b>sameLAN</b>: a second node in the same LAN as the first<br>")
+		pf("</html>")
+
+		if err := os.WriteFile("grid.html", hb.Bytes(), 0666); err != nil {
+			t.Fatalf("writeFile: %v", err)
+		}
+	})
+}

tstest/integration/tailscaled_deps_test_darwin.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/health"
+	_ "tailscale.com/hostinfo"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/conffile"
 	_ "tailscale.com/ipn/ipnlocal"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/health"
+	_ "tailscale.com/hostinfo"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/conffile"
 	_ "tailscale.com/ipn/ipnlocal"

tstest/integration/tailscaled_deps_test_linux.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/health"
+	_ "tailscale.com/hostinfo"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/conffile"
 	_ "tailscale.com/ipn/ipnlocal"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/health"
+	_ "tailscale.com/hostinfo"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/conffile"
 	_ "tailscale.com/ipn/ipnlocal"

tstest/integration/tailscaled_deps_test_windows.go

@@ -25,6 +25,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/health"
+	_ "tailscale.com/hostinfo"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/conffile"
 	_ "tailscale.com/ipn/ipnlocal"

tstest/natlab/vnet/conf.go

@@ -0,0 +1,313 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"cmp"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"slices"
+
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcapgo"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+)
+
+// Note: the exported Node and Network are the configuration types;
+// the unexported node and network are the runtime types that are actually
+// used once the server is created.
+
+// Config is the requested state of the natlab virtual network.
+//
+// The zero value is a valid empty configuration. Call AddNode
+// and AddNetwork to methods on the returned Node and Network
+// values to modify the config before calling NewServer.
+// Once the NewServer is called, Config is no longer used.
+type Config struct {
+	nodes    []*Node
+	networks []*Network
+	pcapFile string
+}
+
+func (c *Config) SetPCAPFile(file string) {
+	c.pcapFile = file
+}
+
+func (c *Config) NumNodes() int {
+	return len(c.nodes)
+}
+
+func (c *Config) FirstNetwork() *Network {
+	if len(c.networks) == 0 {
+		return nil
+	}
+	return c.networks[0]
+}
+
+// AddNode creates a new node in the world.
+//
+// The opts may be of the following types:
+//   - *Network: zero, one, or more networks to add this node to
+//   - TODO: more
+//
+// On an error or unknown opt type, AddNode returns a
+// node with a carried error that gets returned later.
+func (c *Config) AddNode(opts ...any) *Node {
+	num := len(c.nodes)
+	n := &Node{
+		mac: MAC{0x52, 0xcc, 0xcc, 0xcc, 0xcc, byte(num)}, // 52=TS then 0xcc for ccclient
+	}
+	c.nodes = append(c.nodes, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case *Network:
+			if !slices.Contains(o.nodes, n) {
+				o.nodes = append(o.nodes, n)
+			}
+			n.nets = append(n.nets, o)
+		case TailscaledEnv:
+			n.env = append(n.env, o)
+		case NodeOption:
+			if o == HostFirewall {
+				n.hostFW = true
+			} else {
+				if n.err == nil {
+					n.err = fmt.Errorf("unknown NodeOption %q", o)
+				}
+			}
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNode option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// NodeOption is an option that can be passed to Config.AddNode.
+type NodeOption string
+
+const (
+	HostFirewall NodeOption = "HostFirewall"
+)
+
+// TailscaledEnv is а option that can be passed to Config.AddNode
+// to set an environment variable for tailscaled.
+type TailscaledEnv struct {
+	Key, Value string
+}
+
+// AddNetwork add a new network.
+//
+// The opts may be of the following types:
+//   - string IP address, for the network's WAN IP (if any)
+//   - string netip.Prefix, for the network's LAN IP (defaults to 192.168.0.0/24)
+//   - NAT, the type of NAT to use
+//   - NetworkService, a service to add to the network
+//
+// On an error or unknown opt type, AddNetwork returns a
+// network with a carried error that gets returned later.
+func (c *Config) AddNetwork(opts ...any) *Network {
+	num := len(c.networks)
+	n := &Network{
+		mac: MAC{0x52, 0xee, 0xee, 0xee, 0xee, byte(num)}, // 52=TS then 0xee for 'etwork
+	}
+	c.networks = append(c.networks, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case string:
+			if ip, err := netip.ParseAddr(o); err == nil {
+				n.wanIP = ip
+			} else if ip, err := netip.ParsePrefix(o); err == nil {
+				n.lanIP = ip
+			} else {
+				if n.err == nil {
+					n.err = fmt.Errorf("unknown string option %q", o)
+				}
+			}
+		case NAT:
+			n.natType = o
+		case NetworkService:
+			n.AddService(o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNetwork option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// Node is the configuration of a node in the virtual network.
+type Node struct {
+	err error
+	n   *node // nil until NewServer called
+
+	env    []TailscaledEnv
+	hostFW bool
+
+	// TODO(bradfitz): this is halfway converted to supporting multiple NICs
+	// but not done. We need a MAC-per-Network.
+
+	mac  MAC
+	nets []*Network
+}
+
+// MAC returns the MAC address of the node.
+func (n *Node) MAC() MAC {
+	return n.mac
+}
+
+func (n *Node) Env() []TailscaledEnv {
+	return n.env
+}
+
+func (n *Node) HostFirewall() bool {
+	return n.hostFW
+}
+
+// Network returns the first network this node is connected to,
+// or nil if none.
+func (n *Node) Network() *Network {
+	if len(n.nets) == 0 {
+		return nil
+	}
+	return n.nets[0]
+}
+
+// Network is the configuration of a network in the virtual network.
+type Network struct {
+	mac     MAC // MAC address of the router/gateway
+	natType NAT
+
+	wanIP netip.Addr
+	lanIP netip.Prefix
+	nodes []*Node
+
+	svcs set.Set[NetworkService]
+
+	// ...
+	err error // carried error
+}
+
+func (n *Network) CanTakeMoreNodes() bool {
+	if n.natType == One2OneNAT {
+		return len(n.nodes) == 0
+	}
+	return len(n.nodes) < 150
+}
+
+// NetworkService is a service that can be added to a network.
+type NetworkService string
+
+const (
+	NATPMP NetworkService = "NAT-PMP"
+	PCP    NetworkService = "PCP"
+	UPnP   NetworkService = "UPnP"
+)
+
+// AddService adds a network service (such as port mapping protocols) to a
+// network.
+func (n *Network) AddService(s NetworkService) {
+	if n.svcs == nil {
+		n.svcs = set.Of(s)
+	} else {
+		n.svcs.Add(s)
+	}
+}
+
+// initFromConfig initializes the server from the previous calls
+// to NewNode and NewNetwork and returns an error if
+// there were any configuration issues.
+func (s *Server) initFromConfig(c *Config) error {
+	netOfConf := map[*Network]*network{}
+	if c.pcapFile != "" {
+		pcf, err := os.OpenFile(c.pcapFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		nw, err := pcapgo.NewNgWriter(pcf, layers.LinkTypeEthernet)
+		if err != nil {
+			return err
+		}
+		pw := &pcapWriter{
+			f: pcf,
+			w: nw,
+		}
+		s.pcapWriter = pw
+	}
+	for i, conf := range c.networks {
+		if conf.err != nil {
+			return conf.err
+		}
+		if !conf.lanIP.IsValid() {
+			conf.lanIP = netip.MustParsePrefix("192.168.0.0/24")
+		}
+		n := &network{
+			s:         s,
+			mac:       conf.mac,
+			portmap:   conf.svcs.Contains(NATPMP), // TODO: expand network.portmap
+			wanIP:     conf.wanIP,
+			lanIP:     conf.lanIP,
+			nodesByIP: map[netip.Addr]*node{},
+			logf:      logger.WithPrefix(log.Printf, fmt.Sprintf("[net-%v] ", conf.mac)),
+		}
+		netOfConf[conf] = n
+		s.networks.Add(n)
+		if _, ok := s.networkByWAN[conf.wanIP]; ok {
+			return fmt.Errorf("two networks have the same WAN IP %v; Anycast not (yet?) supported", conf.wanIP)
+		}
+		s.networkByWAN[conf.wanIP] = n
+		n.lanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("network%d-lan", i+1),
+			LinkType: layers.LinkTypeIPv4,
+		}))
+		n.wanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("network%d-wan", i+1),
+			LinkType: layers.LinkTypeIPv4,
+		}))
+	}
+	for i, conf := range c.nodes {
+		if conf.err != nil {
+			return conf.err
+		}
+		n := &node{
+			mac: conf.mac,
+			net: netOfConf[conf.Network()],
+		}
+		n.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("node%d", i+1),
+			LinkType: layers.LinkTypeEthernet,
+		}))
+		conf.n = n
+		if _, ok := s.nodeByMAC[n.mac]; ok {
+			return fmt.Errorf("two nodes have the same MAC %v", n.mac)
+		}
+		s.nodes = append(s.nodes, n)
+		s.nodeByMAC[n.mac] = n
+
+		// Allocate a lanIP for the node. Use the network's CIDR and use final
+		// octet 101 (for first node), 102, etc. The node number comes from the
+		// last octent of the MAC address (0-based)
+		ip4 := n.net.lanIP.Addr().As4()
+		ip4[3] = 101 + n.mac[5]
+		n.lanIP = netip.AddrFrom4(ip4)
+		n.net.nodesByIP[n.lanIP] = n
+	}
+
+	// Now that nodes are populated, set up NAT:
+	for _, conf := range c.networks {
+		n := netOfConf[conf]
+		natType := cmp.Or(conf.natType, EasyNAT)
+		if err := n.InitNAT(natType); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tstest/natlab/vnet/conf_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import "testing"
+
+func TestConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		setup   func(*Config)
+		wantErr string
+	}{
+		{
+			name: "simple",
+			setup: func(c *Config) {
+				c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", EasyNAT, NATPMP))
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", HardNAT))
+			},
+		},
+		{
+			name: "indirect",
+			setup: func(c *Config) {
+				n1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", HardNAT))
+				n1.Network().AddService(NATPMP)
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", NAT("hard")))
+			},
+		},
+		{
+			name: "multi-node-in-net",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+		},
+		{
+			name: "dup-wan-ip",
+			setup: func(c *Config) {
+				c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNetwork("2.1.1.1", "10.2.0.1/16")
+			},
+			wantErr: "two networks have the same WAN IP 2.1.1.1; Anycast not (yet?) supported",
+		},
+		{
+			name: "one-to-one-nat-with-multiple-nodes",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24", One2OneNAT)
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+			wantErr: "error creating NAT type \"one2one\" for network 2.1.1.1: can't use one2one NAT type on networks other than single-node networks",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var c Config
+			tt.setup(&c)
+			_, err := New(&c)
+			if err == nil {
+				if tt.wantErr == "" {
+					return
+				}
+				t.Fatalf("got success; wanted error %q", tt.wantErr)
+			}
+			if err.Error() != tt.wantErr {
+				t.Fatalf("got error %q; want %q", err, tt.wantErr)
+			}
+		})
+	}
+}

tstest/natlab/vnet/easyaf.go

@@ -0,0 +1,91 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"log"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+// easyAFNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux), but with only address filtering, not address+port
+// filtering.
+//
+// James says these are used by "anyone with “voip helpers” turned on"
+// "which is a lot of home modem routers" ... "probably like most of the zyxel
+// type things".
+type easyAFNAT struct {
+	pool    IPPool
+	wanIP   netip.Addr
+	out     map[netip.Addr]portMappingAndTime
+	in      map[uint16]lanAddrAndTime
+	lastOut map[srcAPDstAddrTuple]time.Time // (lan:port, wan:port) => last packet out time
+}
+
+type srcAPDstAddrTuple struct {
+	src netip.AddrPort
+	dst netip.Addr
+}
+
+func init() {
+	registerNATType(EasyAFNAT, func(p IPPool) (NATTable, error) {
+		return &easyAFNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyAFNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	_, ok := n.in[ap.Port()]
+	return ok
+}
+
+func (n *easyAFNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	mak.Set(&n.lastOut, srcAPDstAddrTuple{src, dst.Addr()}, at)
+	if pm, ok := n.out[src.Addr()]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+			if n.pool.IsPublicPortUsed(wanAddr) {
+				continue
+			}
+
+			// Found a free port.
+			mak.Set(&n.out, src.Addr(), portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyAFNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	lanDst = n.in[dst.Port()].lanAddr
+
+	// Stateful firewall: drop incoming packets that don't have traffic out.
+	// TODO(bradfitz): verify Linux does this in the router code, not in the NAT code.
+	if t, ok := n.lastOut[srcAPDstAddrTuple{lanDst, src.Addr()}]; !ok || at.Sub(t) > 300*time.Second {
+		log.Printf("Drop incoming packet from %v to %v; no recent outgoing packet", src, dst)
+		return netip.AddrPort{}
+	}
+
+	return lanDst
+}

tstest/natlab/vnet/nat.go

@@ -0,0 +1,293 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"errors"
+	"log"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+const (
+	One2OneNAT NAT = "one2one"
+	EasyNAT    NAT = "easy"   // address+port filtering
+	EasyAFNAT  NAT = "easyaf" // address filtering (not port)
+	HardNAT    NAT = "hard"
+)
+
+// IPPool is the interface that a NAT implementation uses to get information
+// about a network.
+//
+// Outside of tests, this is typically a *network.
+type IPPool interface {
+	// WANIP returns the primary WAN IP address.
+	//
+	// TODO: add another method for networks with multiple WAN IP addresses.
+	WANIP() netip.Addr
+
+	// SoleLanIP reports whether this network has a sole LAN client
+	// and if so, its IP address.
+	SoleLANIP() (_ netip.Addr, ok bool)
+
+	// IsPublicPortUsed reports whether the provided WAN IP+port is in use by
+	// anything. (In particular, the NAT-PMP/etc port mappers might have taken
+	// a port.) Implementations should check this before allocating a port,
+	// and then they should report IsPublicPortUsed themselves for that port.
+	IsPublicPortUsed(netip.AddrPort) bool
+}
+
+// newTableFunc is a constructor for a NAT table.
+// The provided IPPool is typically (outside of tests) a *network.
+type newTableFunc func(IPPool) (NATTable, error)
+
+// NAT is a type of NAT that's known to natlab.
+//
+// For example, "easy" for Linux-style NAT, "hard" for FreeBSD-style NAT, etc.
+type NAT string
+
+// natTypes are the known NAT types.
+var natTypes = map[NAT]newTableFunc{}
+
+// registerNATType registers a NAT type.
+func registerNATType(name NAT, f newTableFunc) {
+	if _, ok := natTypes[name]; ok {
+		panic("duplicate NAT type: " + name)
+	}
+	natTypes[name] = f
+}
+
+// NATTable is what a NAT implementation is expected to do.
+//
+// This project tests Tailscale as it faces various combinations various NAT
+// implementations (e.g. Linux easy style NAT vs FreeBSD hard/endpoint dependent
+// NAT vs Cloud 1:1 NAT, etc)
+//
+// Implementations of NATTable need not handle concurrency; the natlab serializes
+// all calls into a NATTable.
+//
+// The provided `at` value will typically be time.Now, except for tests.
+// Implementations should not use real time and should only compare
+// previously provided time values.
+type NATTable interface {
+	// PickOutgoingSrc returns the source address to use for an outgoing packet.
+	//
+	// The result should either be invalid (to drop the packet) or a WAN (not
+	// private) IP address.
+	//
+	// Typically, the src is a LAN source IP address, but it might also be a WAN
+	// IP address if the packet is being forwarded for a source machine that has
+	// a public IP address.
+	PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort)
+
+	// PickIncomingDst returns the destination address to use for an incoming
+	// packet. The incoming src address is always a public WAN IP.
+	//
+	// The result should either be invalid (to drop the packet) or the IP
+	// address of a machine on the local network address, usually a private
+	// LAN IP.
+	PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort)
+
+	// IsPublicPortUsed reports whether the provided WAN IP+port is in use by
+	// anything. The port mapper uses this to avoid grabbing an in-use port.
+	IsPublicPortUsed(netip.AddrPort) bool
+}
+
+// oneToOneNAT is a 1:1 NAT, like a typical EC2 VM.
+type oneToOneNAT struct {
+	lanIP netip.Addr
+	wanIP netip.Addr
+}
+
+func init() {
+	registerNATType(One2OneNAT, func(p IPPool) (NATTable, error) {
+		lanIP, ok := p.SoleLANIP()
+		if !ok {
+			return nil, errors.New("can't use one2one NAT type on networks other than single-node networks")
+		}
+		return &oneToOneNAT{lanIP: lanIP, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *oneToOneNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	return netip.AddrPortFrom(n.wanIP, src.Port())
+}
+
+func (n *oneToOneNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	return netip.AddrPortFrom(n.lanIP, dst.Port())
+}
+
+func (n *oneToOneNAT) IsPublicPortUsed(netip.AddrPort) bool {
+	return true // all ports are owned by the 1:1 NAT
+}
+
+type srcDstTuple struct {
+	src netip.AddrPort
+	dst netip.AddrPort
+}
+
+type hardKeyIn struct {
+	wanPort uint16
+	src     netip.AddrPort
+}
+
+type portMappingAndTime struct {
+	port uint16
+	at   time.Time
+}
+
+type lanAddrAndTime struct {
+	lanAddr netip.AddrPort
+	at      time.Time
+}
+
+// hardNAT is an "Endpoint Dependent" NAT, like FreeBSD/pfSense/OPNsense.
+// This is shown as "MappingVariesByDestIP: true" by netcheck, and what
+// Tailscale calls "Hard NAT".
+type hardNAT struct {
+	pool  IPPool
+	wanIP netip.Addr
+
+	out map[srcDstTuple]portMappingAndTime
+	in  map[hardKeyIn]lanAddrAndTime
+}
+
+func init() {
+	registerNATType(HardNAT, func(p IPPool) (NATTable, error) {
+		return &hardNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *hardNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	for k := range n.in {
+		if k.wanPort == ap.Port() {
+			return true
+		}
+	}
+	return false
+}
+
+func (n *hardNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	ko := srcDstTuple{src, dst}
+	if pm, ok := n.out[ko]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// No existing mapping exists. Create one.
+
+	// TODO: clean up old expired mappings
+
+	// Instead of proper data structures that would be efficient, we instead
+	// just loop a bunch and look for a free port. This project is only used
+	// by tests and doesn't care about performance, this is good enough.
+	for {
+		port := rand.N(uint16(32<<10)) + 32<<10 // pick some "ephemeral" port
+		if n.pool.IsPublicPortUsed(netip.AddrPortFrom(n.wanIP, port)) {
+			continue
+		}
+
+		ki := hardKeyIn{wanPort: port, src: dst}
+		if _, ok := n.in[ki]; ok {
+			// Port already in use.
+			continue
+		}
+		mak.Set(&n.in, ki, lanAddrAndTime{lanAddr: src, at: at})
+		mak.Set(&n.out, ko, portMappingAndTime{port: port, at: at})
+		return netip.AddrPortFrom(n.wanIP, port)
+	}
+}
+
+func (n *hardNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	ki := hardKeyIn{wanPort: dst.Port(), src: src}
+	if pm, ok := n.in[ki]; ok {
+		// Existing flow.
+		return pm.lanAddr
+	}
+	return netip.AddrPort{} // drop; no mapping
+}
+
+// easyNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux).
+//
+// This is shown as "MappingVariesByDestIP: false" by netcheck, and what
+// Tailscale calls "Easy NAT".
+//
+// Unlike Linux, this implementation is capped at 32k entries and doesn't resort
+// to other allocation strategies when all 32k WAN ports are taken.
+type easyNAT struct {
+	pool    IPPool
+	wanIP   netip.Addr
+	out     map[netip.AddrPort]portMappingAndTime
+	in      map[uint16]lanAddrAndTime
+	lastOut map[srcDstTuple]time.Time // (lan:port, wan:port) => last packet out time
+}
+
+func init() {
+	registerNATType(EasyNAT, func(p IPPool) (NATTable, error) {
+		return &easyNAT{pool: p, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyNAT) IsPublicPortUsed(ap netip.AddrPort) bool {
+	if ap.Addr() != n.wanIP {
+		return false
+	}
+	_, ok := n.in[ap.Port()]
+	return ok
+}
+
+func (n *easyNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	mak.Set(&n.lastOut, srcDstTuple{src, dst}, at)
+	if pm, ok := n.out[src]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+			if n.pool.IsPublicPortUsed(wanAddr) {
+				continue
+			}
+
+			// Found a free port.
+			mak.Set(&n.out, src, portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	lanDst = n.in[dst.Port()].lanAddr
+
+	// Stateful firewall: drop incoming packets that don't have traffic out.
+	// TODO(bradfitz): verify Linux does this in the router code, not in the NAT code.
+	if t, ok := n.lastOut[srcDstTuple{lanDst, src}]; !ok || at.Sub(t) > 300*time.Second {
+		log.Printf("Drop incoming packet from %v to %v; no recent outgoing packet", src, dst)
+		return netip.AddrPort{}
+	}
+
+	return lanDst
+}

tstest/natlab/vnet/pcap.go

@@ -0,0 +1,56 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/pcapgo"
+)
+
+// pcapWriter is a pcapgo.NgWriter that writes to a file.
+// It is safe for concurrent use. The nil value is a no-op.
+type pcapWriter struct {
+	f *os.File
+
+	mu sync.Mutex
+	w  *pcapgo.NgWriter
+}
+
+func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if p == nil {
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.w == nil {
+		return io.ErrClosedPipe
+	}
+	return p.w.WritePacket(ci, data)
+}
+
+func (p *pcapWriter) AddInterface(i pcapgo.NgInterface) (int, error) {
+	if p == nil {
+		return 0, nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.w.AddInterface(i)
+}
+
+func (p *pcapWriter) Close() error {
+	if p == nil {
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.w != nil {
+		p.w.Flush()
+		p.w = nil
+	}
+	return p.f.Close()
+}

tsweb/tsweb_test.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"tailscale.com/cmd/testwrapper/flakytest"
 	"tailscale.com/metrics"
 	"tailscale.com/tstest"
 	"tailscale.com/util/httpm"
@@ -865,14 +864,14 @@ func TestStdHandler_CanceledAfterHeader(t *testing.T) {
 }
 
 func TestStdHandler_ConnectionClosedDuringBody(t *testing.T) {
-	flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/13017")
 	now := time.Now()
 
-	// Start a HTTP server that returns 1MB of data.
+	// Start a HTTP server that writes back zeros until the request is abandoned.
 	// We next put a reverse-proxy in front of this server.
 	rs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		for range 1024 {
-			w.Write(make([]byte, 1024))
+		zeroes := make([]byte, 1024)
+		for r.Context().Err() == nil {
+			w.Write(zeroes)
 		}
 	}))
 	defer rs.Close()
@@ -880,8 +879,9 @@ func TestStdHandler_ConnectionClosedDuringBody(t *testing.T) {
 	r := make(chan AccessLogRecord)
 	var e *HTTPError
 	responseStarted := make(chan struct{})
+	requestCanceled := make(chan struct{})
 
-	// Create another server which proxies our 1MB server.
+	// Create another server which proxies our zeroes server.
 	// The [httputil.ReverseProxy] will panic with [http.ErrAbortHandler] when
 	// it fails to copy the response to the client.
 	h := StdHandler(
@@ -890,10 +890,6 @@ func TestStdHandler_ConnectionClosedDuringBody(t *testing.T) {
 				Director: func(r *http.Request) {
 					r.URL = must.Get(url.Parse(rs.URL))
 				},
-				ModifyResponse: func(r *http.Response) error {
-					close(responseStarted)
-					return nil
-				},
 			}).ServeHTTP(w, r)
 			return nil
 		}),
@@ -908,7 +904,11 @@ func TestStdHandler_ConnectionClosedDuringBody(t *testing.T) {
 			},
 		},
 	)
-	s := httptest.NewServer(h)
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		close(responseStarted)
+		<-requestCanceled
+		h.ServeHTTP(w, r.WithContext(context.WithoutCancel(r.Context())))
+	}))
 	t.Cleanup(s.Close)
 
 	// Create a context which gets canceled after the handler starts processing
@@ -925,6 +925,7 @@ func TestStdHandler_ConnectionClosedDuringBody(t *testing.T) {
 		t.Fatalf("making request: %s", err)
 	}
 	res, err := http.DefaultClient.Do(req)
+	close(requestCanceled)
 	if !errors.Is(err, context.Canceled) {
 		t.Errorf("got error %v, want context.Canceled", err)
 	}

util/syspolicy/internal/internal.go

@@ -0,0 +1,63 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package internal contains miscellaneous functions and types
+// that are internal to the syspolicy packages.
+package internal
+
+import (
+	"bytes"
+
+	"github.com/go-json-experiment/json/jsontext"
+	"tailscale.com/types/lazy"
+	"tailscale.com/version"
+)
+
+// OSForTesting is the operating system override used for testing.
+// It follows the same naming convention as [version.OS].
+var OSForTesting lazy.SyncValue[string]
+
+// OS is like [version.OS], but supports a test hook.
+func OS() string {
+	return OSForTesting.Get(version.OS)
+}
+
+// TB is a subset of testing.TB that we use to set up test helpers.
+// It's defined here to avoid pulling in the testing package.
+type TB interface {
+	Helper()
+	Cleanup(func())
+	Logf(format string, args ...any)
+	Error(args ...any)
+	Errorf(format string, args ...any)
+	Fatal(args ...any)
+	Fatalf(format string, args ...any)
+}
+
+// EqualJSONForTest compares the JSON in j1 and j2 for semantic equality.
+// It returns "", "", true if j1 and j2 are equal. Otherwise, it returns
+// indented versions of j1 and j2 and false.
+func EqualJSONForTest(tb TB, j1, j2 jsontext.Value) (s1, s2 string, equal bool) {
+	tb.Helper()
+	j1 = j1.Clone()
+	j2 = j2.Clone()
+	// Canonicalize JSON values for comparison.
+	if err := j1.Canonicalize(); err != nil {
+		tb.Error(err)
+	}
+	if err := j2.Canonicalize(); err != nil {
+		tb.Error(err)
+	}
+	// Check and return true if the two values are structurally equal.
+	if bytes.Equal(j1, j2) {
+		return "", "", true
+	}
+	// Otherwise, format the values for display and return false.
+	if err := j1.Indent("", "\t"); err != nil {
+		tb.Fatal(err)
+	}
+	if err := j2.Indent("", "\t"); err != nil {
+		tb.Fatal(err)
+	}
+	return j1.String(), j2.String(), false
+}

util/syspolicy/policy_keys.go

@@ -3,7 +3,9 @@
 
 package syspolicy
 
-type Key string
+import "tailscale.com/util/syspolicy/setting"
+
+type Key = setting.Key
 
 const (
 	// Keys with a string value

util/syspolicy/setting/errors.go

@@ -0,0 +1,71 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"errors"
+
+	"tailscale.com/types/ptr"
+)
+
+var (
+	// ErrNotConfigured is returned when the requested policy setting is not configured.
+	ErrNotConfigured = errors.New("not configured")
+	// ErrTypeMismatch is returned when there's a type mismatch between the actual type
+	// of the setting value and the expected type.
+	ErrTypeMismatch = errors.New("type mismatch")
+	// ErrNoSuchKey is returned by [DefinitionOf] when no policy setting
+	// has been registered with the specified key.
+	//
+	// Until 2024-08-02, this error was also returned by a [Handler] when the specified
+	// key did not have a value set. While the package maintains compatibility with this
+	// usage of ErrNoSuchKey, it is recommended to return [ErrNotConfigured] from newer
+	// [source.Store] implementations.
+	ErrNoSuchKey = errors.New("no such key")
+)
+
+// ErrorText represents an error that occurs when reading or parsing a policy setting.
+// This includes errors due to permissions issues, value type and format mismatches,
+// and other platform- or source-specific errors. It does not include
+// [ErrNotConfigured] and [ErrNoSuchKey], as those correspond to unconfigured
+// policy settings rather than settings that cannot be read or parsed
+// due to an error.
+//
+// ErrorText is used to marshal errors when a policy setting is sent over the wire,
+// allowing the error to be logged or displayed. It does not preserve the
+// type information of the underlying error.
+type ErrorText string
+
+// NewErrorText returns a [ErrorText] with the specified error message.
+func NewErrorText(text string) *ErrorText {
+	return ptr.To(ErrorText(text))
+}
+
+// NewErrorTextFromError returns an [ErrorText] with the text of the specified error,
+// or nil if err is nil, [ErrNotConfigured], or [ErrNoSuchKey].
+func NewErrorTextFromError(err error) *ErrorText {
+	if err == nil || errors.Is(err, ErrNotConfigured) || errors.Is(err, ErrNoSuchKey) {
+		return nil
+	}
+	if err, ok := err.(*ErrorText); ok {
+		return err
+	}
+	return ptr.To(ErrorText(err.Error()))
+}
+
+// Error implements error.
+func (e ErrorText) Error() string {
+	return string(e)
+}
+
+// MarshalText implements [encoding.TextMarshaler].
+func (e ErrorText) MarshalText() (text []byte, err error) {
+	return []byte(e.Error()), nil
+}
+
+// UnmarshalText implements [encoding.TextUnmarshaler].
+func (e *ErrorText) UnmarshalText(text []byte) error {
+	*e = ErrorText(text)
+	return nil
+}

util/syspolicy/setting/key.go

@@ -0,0 +1,13 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+// Key is a string that uniquely identifies a policy and must remain unchanged
+// once established and documented for a given policy setting. It may contain
+// alphanumeric characters and zero or more [KeyPathSeparator]s to group
+// individual policy settings into categories.
+type Key string
+
+// KeyPathSeparator allows logical grouping of policy settings into categories.
+const KeyPathSeparator = "/"

util/syspolicy/setting/origin.go

@@ -0,0 +1,71 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"fmt"
+
+	jsonv2 "github.com/go-json-experiment/json"
+	"github.com/go-json-experiment/json/jsontext"
+)
+
+// Origin describes where a policy or a policy setting is configured.
+type Origin struct {
+	data settingOrigin
+}
+
+// settingOrigin is the marshallable data of an [Origin].
+type settingOrigin struct {
+	Name  string `json:",omitzero"`
+	Scope PolicyScope
+}
+
+// NewOrigin returns a new [Origin] with the specified scope.
+func NewOrigin(scope PolicyScope) *Origin {
+	return NewNamedOrigin("", scope)
+}
+
+// NewNamedOrigin returns a new [Origin] with the specified scope and name.
+func NewNamedOrigin(name string, scope PolicyScope) *Origin {
+	return &Origin{settingOrigin{name, scope}}
+}
+
+// Scope reports the policy [PolicyScope] where the setting is configured.
+func (s Origin) Scope() PolicyScope {
+	return s.data.Scope
+}
+
+// Name returns the name of the policy source where the setting is configured,
+// or "" if not available.
+func (s Origin) Name() string {
+	return s.data.Name
+}
+
+// String implements [fmt.Stringer].
+func (s Origin) String() string {
+	if s.Name() != "" {
+		return fmt.Sprintf("%s (%v)", s.Name(), s.Scope())
+	}
+	return s.Scope().String()
+}
+
+// MarshalJSONV2 implements [jsonv2.MarshalerV2].
+func (s Origin) MarshalJSONV2(out *jsontext.Encoder, opts jsonv2.Options) error {
+	return jsonv2.MarshalEncode(out, &s.data, opts)
+}
+
+// UnmarshalJSONV2 implements [jsonv2.UnmarshalerV2].
+func (s *Origin) UnmarshalJSONV2(in *jsontext.Decoder, opts jsonv2.Options) error {
+	return jsonv2.UnmarshalDecode(in, &s.data, opts)
+}
+
+// MarshalJSON implements [json.Marshaler].
+func (s Origin) MarshalJSON() ([]byte, error) {
+	return jsonv2.Marshal(s) // uses MarshalJSONV2
+}
+
+// UnmarshalJSON implements [json.Unmarshaler].
+func (s *Origin) UnmarshalJSON(b []byte) error {
+	return jsonv2.Unmarshal(b, s) // uses UnmarshalJSONV2
+}

util/syspolicy/setting/policy_scope.go

@@ -0,0 +1,189 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"fmt"
+	"strings"
+
+	"tailscale.com/types/lazy"
+)
+
+var (
+	lazyDefaultScope lazy.SyncValue[PolicyScope]
+
+	// DeviceScope indicates a scope containing device-global policies.
+	DeviceScope = PolicyScope{kind: DeviceSetting}
+	// CurrentProfileScope indicates a scope containing policies that apply to the
+	// currently active Tailscale profile.
+	CurrentProfileScope = PolicyScope{kind: ProfileSetting}
+	// CurrentUserScope indicates a scope containing policies that apply to the
+	// current user, for whatever that means on the current platform and
+	// in the current application context.
+	CurrentUserScope = PolicyScope{kind: UserSetting}
+)
+
+// PolicyScope is a management scope.
+type PolicyScope struct {
+	kind      Scope
+	userID    string
+	profileID string
+}
+
+// DefaultScope returns the default [PolicyScope] to be used by a program
+// when querying policy settings.
+// It returns [DeviceScope], unless explicitly changed with [SetDefaultScope].
+func DefaultScope() PolicyScope {
+	return lazyDefaultScope.Get(func() PolicyScope { return DeviceScope })
+}
+
+// SetDefaultScope attempts to set the specified scope as the default scope
+// to be used by a program when querying policy settings.
+// It fails and returns false if called more than once, or if the [DefaultScope]
+// has already been used.
+func SetDefaultScope(scope PolicyScope) bool {
+	return lazyDefaultScope.Set(scope)
+}
+
+// UserScopeOf returns a policy [PolicyScope] of the user with the specified id.
+func UserScopeOf(uid string) PolicyScope {
+	return PolicyScope{kind: UserSetting, userID: uid}
+}
+
+// Kind reports the scope kind of s.
+func (s PolicyScope) Kind() Scope {
+	return s.kind
+}
+
+// IsApplicableSetting reports whether the specified setting applies to
+// and can be retrieved for this scope. Policy settings are applicable
+// to their own scopes as well as more specific scopes. For example,
+// device settings are applicable to device, profile and user scopes,
+// but user settings are only applicable to user scopes.
+// For instance, a menu visibility setting is inherently a user setting
+// and only makes sense in the context of a specific user.
+func (s PolicyScope) IsApplicableSetting(setting *Definition) bool {
+	return setting != nil && setting.Scope() <= s.Kind()
+}
+
+// IsConfigurableSetting reports whether the specified setting can be configured
+// by a policy at this scope. Policy settings are configurable at their own scopes
+// as well as broader scopes. For example, [UserSetting]s are configurable in
+// user, profile, and device scopes, but [DeviceSetting]s are only configurable
+// in the [DeviceScope]. For instance, the InstallUpdates policy setting
+// can only be configured in the device scope, as it controls whether updates
+// will be installed automatically on the device, rather than for specific users.
+func (s PolicyScope) IsConfigurableSetting(setting *Definition) bool {
+	return setting != nil && setting.Scope() >= s.Kind()
+}
+
+// Contains reports whether policy settings that apply to s also apply to s2.
+// For example, policy settings that apply to the [DeviceScope] also apply to
+// the [CurrentUserScope].
+func (s PolicyScope) Contains(s2 PolicyScope) bool {
+	if s.Kind() > s2.Kind() {
+		return false
+	}
+	switch s.Kind() {
+	case DeviceSetting:
+		return true
+	case ProfileSetting:
+		return s.profileID == s2.profileID
+	case UserSetting:
+		return s.userID == s2.userID
+	default:
+		panic("unreachable")
+	}
+}
+
+// StrictlyContains is like [PolicyScope.Contains], but returns false
+// when s and s2 is the same scope.
+func (s PolicyScope) StrictlyContains(s2 PolicyScope) bool {
+	return s != s2 && s.Contains(s2)
+}
+
+// String implements [fmt.Stringer].
+func (s PolicyScope) String() string {
+	if s.profileID == "" && s.userID == "" {
+		return s.kind.String()
+	}
+	return s.stringSlow()
+}
+
+// MarshalText implements [encoding.TextMarshaler].
+func (s PolicyScope) MarshalText() ([]byte, error) {
+	return []byte(s.String()), nil
+}
+
+// MarshalText implements [encoding.TextUnmarshaler].
+func (s *PolicyScope) UnmarshalText(b []byte) error {
+	*s = PolicyScope{}
+	parts := strings.SplitN(string(b), "/", 2)
+	for i, part := range parts {
+		kind, id, err := parseScopeAndID(part)
+		if err != nil {
+			return err
+		}
+		if i > 0 && kind <= s.kind {
+			return fmt.Errorf("invalid scope hierarchy: %s", b)
+		}
+		s.kind = kind
+		switch kind {
+		case DeviceSetting:
+			if id != "" {
+				return fmt.Errorf("the device scope must not have an ID: %s", b)
+			}
+		case ProfileSetting:
+			s.profileID = id
+		case UserSetting:
+			s.userID = id
+		}
+	}
+	return nil
+}
+
+func (s PolicyScope) stringSlow() string {
+	var sb strings.Builder
+	writeScopeWithID := func(s Scope, id string) {
+		sb.WriteString(s.String())
+		if id != "" {
+			sb.WriteRune('(')
+			sb.WriteString(id)
+			sb.WriteRune(')')
+		}
+	}
+	if s.kind == ProfileSetting || s.profileID != "" {
+		writeScopeWithID(ProfileSetting, s.profileID)
+		if s.kind != ProfileSetting {
+			sb.WriteRune('/')
+		}
+	}
+	if s.kind == UserSetting {
+		writeScopeWithID(UserSetting, s.userID)
+	}
+	return sb.String()
+}
+
+func parseScopeAndID(s string) (scope Scope, id string, err error) {
+	name, params, ok := extractScopeAndParams(s)
+	if !ok {
+		return 0, "", fmt.Errorf("%q is not a valid scope string", s)
+	}
+	if err := scope.UnmarshalText([]byte(name)); err != nil {
+		return 0, "", err
+	}
+	return scope, params, nil
+}
+
+func extractScopeAndParams(s string) (name, params string, ok bool) {
+	paramsStart := strings.Index(s, "(")
+	if paramsStart == -1 {
+		return s, "", true
+	}
+	paramsEnd := strings.LastIndex(s, ")")
+	if paramsEnd < paramsStart {
+		return "", "", false
+	}
+	return s[0:paramsStart], s[paramsStart+1 : paramsEnd], true
+}

util/syspolicy/setting/policy_scope_test.go

@@ -0,0 +1,565 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"reflect"
+	"testing"
+
+	jsonv2 "github.com/go-json-experiment/json"
+)
+
+func TestPolicyScopeIsApplicableSetting(t *testing.T) {
+	tests := []struct {
+		name           string
+		scope          PolicyScope
+		setting        *Definition
+		wantApplicable bool
+	}{
+		{
+			name:           "DeviceScope/DeviceSetting",
+			scope:          DeviceScope,
+			setting:        NewDefinition("TestSetting", DeviceSetting, IntegerValue),
+			wantApplicable: true,
+		},
+		{
+			name:           "DeviceScope/ProfileSetting",
+			scope:          DeviceScope,
+			setting:        NewDefinition("TestSetting", ProfileSetting, IntegerValue),
+			wantApplicable: false,
+		},
+		{
+			name:           "DeviceScope/UserSetting",
+			scope:          DeviceScope,
+			setting:        NewDefinition("TestSetting", UserSetting, IntegerValue),
+			wantApplicable: false,
+		},
+		{
+			name:           "ProfileScope/DeviceSetting",
+			scope:          CurrentProfileScope,
+			setting:        NewDefinition("TestSetting", DeviceSetting, IntegerValue),
+			wantApplicable: true,
+		},
+		{
+			name:           "ProfileScope/ProfileSetting",
+			scope:          CurrentProfileScope,
+			setting:        NewDefinition("TestSetting", ProfileSetting, IntegerValue),
+			wantApplicable: true,
+		},
+		{
+			name:           "ProfileScope/UserSetting",
+			scope:          CurrentProfileScope,
+			setting:        NewDefinition("TestSetting", UserSetting, IntegerValue),
+			wantApplicable: false,
+		},
+		{
+			name:           "UserScope/DeviceSetting",
+			scope:          CurrentUserScope,
+			setting:        NewDefinition("TestSetting", DeviceSetting, IntegerValue),
+			wantApplicable: true,
+		},
+		{
+			name:           "UserScope/ProfileSetting",
+			scope:          CurrentUserScope,
+			setting:        NewDefinition("TestSetting", ProfileSetting, IntegerValue),
+			wantApplicable: true,
+		},
+		{
+			name:           "UserScope/UserSetting",
+			scope:          CurrentUserScope,
+			setting:        NewDefinition("TestSetting", UserSetting, IntegerValue),
+			wantApplicable: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotApplicable := tt.scope.IsApplicableSetting(tt.setting)
+			if gotApplicable != tt.wantApplicable {
+				t.Fatalf("got %v, want %v", gotApplicable, tt.wantApplicable)
+			}
+		})
+	}
+}
+
+func TestPolicyScopeIsConfigurableSetting(t *testing.T) {
+	tests := []struct {
+		name             string
+		scope            PolicyScope
+		setting          *Definition
+		wantConfigurable bool
+	}{
+		{
+			name:             "DeviceScope/DeviceSetting",
+			scope:            DeviceScope,
+			setting:          NewDefinition("TestSetting", DeviceSetting, IntegerValue),
+			wantConfigurable: true,
+		},
+		{
+			name:             "DeviceScope/ProfileSetting",
+			scope:            DeviceScope,
+			setting:          NewDefinition("TestSetting", ProfileSetting, IntegerValue),
+			wantConfigurable: true,
+		},
+		{
+			name:             "DeviceScope/UserSetting",
+			scope:            DeviceScope,
+			setting:          NewDefinition("TestSetting", UserSetting, IntegerValue),
+			wantConfigurable: true,
+		},
+		{
+			name:             "ProfileScope/DeviceSetting",
+			scope:            CurrentProfileScope,
+			setting:          NewDefinition("TestSetting", DeviceSetting, IntegerValue),
+			wantConfigurable: false,
+		},
+		{
+			name:             "ProfileScope/ProfileSetting",
+			scope:            CurrentProfileScope,
+			setting:          NewDefinition("TestSetting", ProfileSetting, IntegerValue),
+			wantConfigurable: true,
+		},
+		{
+			name:             "ProfileScope/UserSetting",
+			scope:            CurrentProfileScope,
+			setting:          NewDefinition("TestSetting", UserSetting, IntegerValue),
+			wantConfigurable: true,
+		},
+		{
+			name:             "UserScope/DeviceSetting",
+			scope:            CurrentUserScope,
+			setting:          NewDefinition("TestSetting", DeviceSetting, IntegerValue),
+			wantConfigurable: false,
+		},
+		{
+			name:             "UserScope/ProfileSetting",
+			scope:            CurrentUserScope,
+			setting:          NewDefinition("TestSetting", ProfileSetting, IntegerValue),
+			wantConfigurable: false,
+		},
+		{
+			name:             "UserScope/UserSetting",
+			scope:            CurrentUserScope,
+			setting:          NewDefinition("TestSetting", UserSetting, IntegerValue),
+			wantConfigurable: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotConfigurable := tt.scope.IsConfigurableSetting(tt.setting)
+			if gotConfigurable != tt.wantConfigurable {
+				t.Fatalf("got %v, want %v", gotConfigurable, tt.wantConfigurable)
+			}
+		})
+	}
+}
+
+func TestPolicyScopeContains(t *testing.T) {
+	tests := []struct {
+		name                   string
+		scopeA                 PolicyScope
+		scopeB                 PolicyScope
+		wantAContainsB         bool
+		wantAStrictlyContainsB bool
+	}{
+		{
+			name:                   "DeviceScope/DeviceScope",
+			scopeA:                 DeviceScope,
+			scopeB:                 DeviceScope,
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "DeviceScope/CurrentProfileScope",
+			scopeA:                 DeviceScope,
+			scopeB:                 CurrentProfileScope,
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: true,
+		},
+		{
+			name:                   "DeviceScope/UserScope",
+			scopeA:                 DeviceScope,
+			scopeB:                 CurrentUserScope,
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: true,
+		},
+		{
+			name:                   "ProfileScope/DeviceScope",
+			scopeA:                 CurrentProfileScope,
+			scopeB:                 DeviceScope,
+			wantAContainsB:         false,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "ProfileScope/ProfileScope",
+			scopeA:                 CurrentProfileScope,
+			scopeB:                 CurrentProfileScope,
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "ProfileScope/UserScope",
+			scopeA:                 CurrentProfileScope,
+			scopeB:                 CurrentUserScope,
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: true,
+		},
+		{
+			name:                   "UserScope/DeviceScope",
+			scopeA:                 CurrentUserScope,
+			scopeB:                 DeviceScope,
+			wantAContainsB:         false,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "UserScope/ProfileScope",
+			scopeA:                 CurrentUserScope,
+			scopeB:                 CurrentProfileScope,
+			wantAContainsB:         false,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "UserScope/UserScope",
+			scopeA:                 CurrentUserScope,
+			scopeB:                 CurrentUserScope,
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "UserScope(1234)/UserScope(1234)",
+			scopeA:                 UserScopeOf("1234"),
+			scopeB:                 UserScopeOf("1234"),
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "UserScope(1234)/UserScope(5678)",
+			scopeA:                 UserScopeOf("1234"),
+			scopeB:                 UserScopeOf("5678"),
+			wantAContainsB:         false,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "ProfileScope(A)/UserScope(A/1234)",
+			scopeA:                 PolicyScope{kind: ProfileSetting, profileID: "A"},
+			scopeB:                 PolicyScope{kind: UserSetting, userID: "1234", profileID: "A"},
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: true,
+		},
+		{
+			name:                   "ProfileScope(A)/UserScope(B/1234)",
+			scopeA:                 PolicyScope{kind: ProfileSetting, profileID: "A"},
+			scopeB:                 PolicyScope{kind: UserSetting, userID: "1234", profileID: "B"},
+			wantAContainsB:         false,
+			wantAStrictlyContainsB: false,
+		},
+		{
+			name:                   "UserScope(1234)/UserScope(A/1234)",
+			scopeA:                 PolicyScope{kind: UserSetting, userID: "1234"},
+			scopeB:                 PolicyScope{kind: UserSetting, userID: "1234", profileID: "A"},
+			wantAContainsB:         true,
+			wantAStrictlyContainsB: true,
+		},
+		{
+			name:                   "UserScope(1234)/UserScope(A/5678)",
+			scopeA:                 PolicyScope{kind: UserSetting, userID: "1234"},
+			scopeB:                 PolicyScope{kind: UserSetting, userID: "5678", profileID: "A"},
+			wantAContainsB:         false,
+			wantAStrictlyContainsB: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotContains := tt.scopeA.Contains(tt.scopeB)
+			if gotContains != tt.wantAContainsB {
+				t.Fatalf("WithinOf: got %v, want %v", gotContains, tt.wantAContainsB)
+			}
+
+			gotStrictlyContains := tt.scopeA.StrictlyContains(tt.scopeB)
+			if gotStrictlyContains != tt.wantAStrictlyContainsB {
+				t.Fatalf("StrictlyWithinOf: got %v, want %v", gotStrictlyContains, tt.wantAStrictlyContainsB)
+			}
+		})
+	}
+}
+
+func TestPolicyScopeMarshalUnmarshal(t *testing.T) {
+	tests := []struct {
+		name      string
+		in        any
+		wantJSON  string
+		wantError bool
+	}{
+		{
+			name: "null-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{},
+			wantJSON: `{"Scope":"Device"}`,
+		},
+		{
+			name: "null-scope-omit-zero",
+			in: &struct {
+				Scope PolicyScope `json:",omitzero"`
+			}{},
+			wantJSON: `{}`,
+		},
+		{
+			name: "device-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{DeviceScope},
+			wantJSON: `{"Scope":"Device"}`,
+		},
+		{
+			name: "current-profile-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{CurrentProfileScope},
+			wantJSON: `{"Scope":"Profile"}`,
+		},
+		{
+			name: "current-user-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{CurrentUserScope},
+			wantJSON: `{"Scope":"User"}`,
+		},
+		{
+			name: "specific-user-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{UserScopeOf("_")},
+			wantJSON: `{"Scope":"User(_)"}`,
+		},
+		{
+			name: "specific-user-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{UserScopeOf("S-1-5-21-3698941153-1525015703-2649197413-1001")},
+			wantJSON: `{"Scope":"User(S-1-5-21-3698941153-1525015703-2649197413-1001)"}`,
+		},
+		{
+			name: "specific-profile-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{PolicyScope{kind: ProfileSetting, profileID: "1234"}},
+			wantJSON: `{"Scope":"Profile(1234)"}`,
+		},
+		{
+			name: "specific-profile-and-user-scope",
+			in: &struct {
+				Scope PolicyScope
+			}{PolicyScope{
+				kind:      UserSetting,
+				profileID: "1234",
+				userID:    "S-1-5-21-3698941153-1525015703-2649197413-1001",
+			}},
+			wantJSON: `{"Scope":"Profile(1234)/User(S-1-5-21-3698941153-1525015703-2649197413-1001)"}`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotJSON, err := jsonv2.Marshal(tt.in)
+			if err != nil {
+				t.Fatalf("Marshal failed: %v", err)
+			}
+			if string(gotJSON) != tt.wantJSON {
+				t.Fatalf("Marshal got %s, want %s", gotJSON, tt.wantJSON)
+			}
+			wantBack := tt.in
+			gotBack := reflect.New(reflect.TypeOf(tt.in).Elem()).Interface()
+			err = jsonv2.Unmarshal(gotJSON, gotBack)
+			if err != nil {
+				t.Fatalf("Unmarshal failed: %v", err)
+			}
+			if !reflect.DeepEqual(gotBack, wantBack) {
+				t.Fatalf("Unmarshal got %+v, want %+v", gotBack, wantBack)
+			}
+		})
+	}
+}
+
+func TestPolicyScopeUnmarshalSpecial(t *testing.T) {
+	tests := []struct {
+		name      string
+		json      string
+		want      any
+		wantError bool
+	}{
+		{
+			name: "empty",
+			json: "{}",
+			want: &struct {
+				Scope PolicyScope
+			}{},
+		},
+		{
+			name:      "too-many-scopes",
+			json:      `{"Scope":"Device/Profile/User"}`,
+			wantError: true,
+		},
+		{
+			name:      "user/profile", // incorrect order
+			json:      `{"Scope":"User/Profile"}`,
+			wantError: true,
+		},
+		{
+			name: "profile-user-no-params",
+			json: `{"Scope":"Profile/User"}`,
+			want: &struct {
+				Scope PolicyScope
+			}{CurrentUserScope},
+		},
+		{
+			name:      "unknown-scope",
+			json:      `{"Scope":"Unknown"}`,
+			wantError: true,
+		},
+		{
+			name:      "unknown-scope/unknown-scope",
+			json:      `{"Scope":"Unknown/Unknown"}`,
+			wantError: true,
+		},
+		{
+			name:      "device-scope/unknown-scope",
+			json:      `{"Scope":"Device/Unknown"}`,
+			wantError: true,
+		},
+		{
+			name:      "unknown-scope/device-scope",
+			json:      `{"Scope":"Unknown/Device"}`,
+			wantError: true,
+		},
+		{
+			name:      "slash",
+			json:      `{"Scope":"/"}`,
+			wantError: true,
+		},
+		{
+			name:      "empty",
+			json:      `{"Scope": ""`,
+			wantError: true,
+		},
+		{
+			name:      "no-closing-bracket",
+			json:      `{"Scope": "user(1234"`,
+			wantError: true,
+		},
+		{
+			name:      "device-with-id",
+			json:      `{"Scope": "device(123)"`,
+			wantError: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := &struct {
+				Scope PolicyScope
+			}{}
+			err := jsonv2.Unmarshal([]byte(tt.json), got)
+			if (err != nil) != tt.wantError {
+				t.Errorf("Marshal error: got %v, want %v", err, tt.wantError)
+			}
+			if err != nil {
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Fatalf("Unmarshal got %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+
+}
+
+func TestExtractScopeAndParams(t *testing.T) {
+	tests := []struct {
+		name   string
+		s      string
+		scope  string
+		params string
+		wantOk bool
+	}{
+		{
+			name:   "empty",
+			s:      "",
+			wantOk: true,
+		},
+		{
+			name:   "scope-only",
+			s:      "device",
+			scope:  "device",
+			wantOk: true,
+		},
+		{
+			name:   "scope-with-params",
+			s:      "user(1234)",
+			scope:  "user",
+			params: "1234",
+			wantOk: true,
+		},
+		{
+			name:   "params-empty-scope",
+			s:      "(1234)",
+			scope:  "",
+			params: "1234",
+			wantOk: true,
+		},
+		{
+			name:   "params-with-brackets",
+			s:      "test()())))())",
+			scope:  "test",
+			params: ")())))()",
+			wantOk: true,
+		},
+		{
+			name:   "no-closing-bracket",
+			s:      "user(1234",
+			scope:  "",
+			params: "",
+			wantOk: false,
+		},
+		{
+			name:   "open-before-close",
+			s:      ")user(1234",
+			scope:  "",
+			params: "",
+			wantOk: false,
+		},
+		{
+			name:   "brackets-only",
+			s:      ")(",
+			scope:  "",
+			params: "",
+			wantOk: false,
+		},
+		{
+			name:   "closing-bracket",
+			s:      ")",
+			scope:  "",
+			params: "",
+			wantOk: false,
+		},
+		{
+			name:   "opening-bracket",
+			s:      ")",
+			scope:  "",
+			params: "",
+			wantOk: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scope, params, ok := extractScopeAndParams(tt.s)
+			if ok != tt.wantOk {
+				t.Logf("OK: got %v; want %v", ok, tt.wantOk)
+			}
+			if scope != tt.scope {
+				t.Logf("Scope: got %q; want %q", scope, tt.scope)
+			}
+			if params != tt.params {
+				t.Logf("Params: got %v; want %v", params, tt.params)
+			}
+		})
+	}
+}

util/syspolicy/setting/raw_item.go

@@ -0,0 +1,67 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"fmt"
+
+	"tailscale.com/types/structs"
+)
+
+// RawItem contains a raw policy setting value as read from a policy store, or an
+// error if the requested setting could not be read from the store. As a special
+// case, it may also hold a value of the [Visibility], [PreferenceOption],
+// or [time.Duration] types. While the policy store interface does not support
+// these types natively, and the values of these types have to be unmarshalled
+// or converted from strings, these setting types predate the typed policy
+// hierarchies, and must be supported at this layer.
+type RawItem struct {
+	_      structs.Incomparable
+	value  any
+	err    *ErrorText
+	origin *Origin // or nil
+}
+
+// RawItemOf returns a [RawItem] with the specified value.
+func RawItemOf(value any) RawItem {
+	return RawItemWith(value, nil, nil)
+}
+
+// RawItemWith returns a [RawItem] with the specified value, error and origin.
+func RawItemWith(value any, err *ErrorText, origin *Origin) RawItem {
+	return RawItem{value: value, err: err, origin: origin}
+}
+
+// Value returns the value of the policy setting, or nil if the policy setting
+// is not configured, or an error occurred while reading it.
+func (i RawItem) Value() any {
+	return i.value
+}
+
+// Error returns the error that occurred when reading the policy setting,
+// or nil if no error occurred.
+func (i RawItem) Error() error {
+	if i.err != nil {
+		return i.err
+	}
+	return nil
+}
+
+// Origin returns an optional [Origin] indicating where the policy setting is
+// configured.
+func (i RawItem) Origin() *Origin {
+	return i.origin
+}
+
+// String implements [fmt.Stringer].
+func (i RawItem) String() string {
+	var suffix string
+	if i.origin != nil {
+		suffix = fmt.Sprintf(" - {%v}", i.origin)
+	}
+	if i.err != nil {
+		return fmt.Sprintf("Error{%q}%s", i.err.Error(), suffix)
+	}
+	return fmt.Sprintf("%v%s", i.value, suffix)
+}

util/syspolicy/setting/setting.go

@@ -0,0 +1,348 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package setting contains types for defining and representing policy settings.
+// It facilitates the registration of setting definitions using [Register] and [RegisterDefinition],
+// and the retrieval of registered setting definitions via [Definitions] and [DefinitionOf].
+// This package is intended for use primarily within the syspolicy package hierarchy.
+package setting
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/types/lazy"
+	"tailscale.com/util/syspolicy/internal"
+)
+
+// Scope indicates the broadest scope at which a policy setting may apply,
+// and the narrowest scope at which it may be configured.
+type Scope int8
+
+const (
+	// DeviceSetting indicates a policy setting that applies to a device, regardless of
+	// which OS user or Tailscale profile is currently active, if any.
+	// It can only be configured at a [DeviceScope].
+	DeviceSetting Scope = iota
+	// ProfileSetting indicates a policy setting that applies to a Tailscale profile.
+	// It can only be configured for a specific profile or at a [DeviceScope],
+	// in which case it applies to all profiles on the device.
+	ProfileSetting
+	// UserSetting indicates a policy setting that applies to users.
+	// It can be configured for a user, profile, or the entire device.
+	UserSetting
+
+	// NumScopes is the number of possible [Scope] values.
+	NumScopes int = iota // must be the last value in the const block.
+)
+
+// String implements [fmt.Stringer].
+func (s Scope) String() string {
+	switch s {
+	case DeviceSetting:
+		return "Device"
+	case ProfileSetting:
+		return "Profile"
+	case UserSetting:
+		return "User"
+	default:
+		panic("unreachable")
+	}
+}
+
+// MarshalText implements [encoding.TextMarshaler].
+func (s Scope) MarshalText() (text []byte, err error) {
+	return []byte(s.String()), nil
+}
+
+// UnmarshalText implements [encoding.TextUnmarshaler].
+func (s *Scope) UnmarshalText(text []byte) error {
+	switch strings.ToLower(string(text)) {
+	case "device":
+		*s = DeviceSetting
+	case "profile":
+		*s = ProfileSetting
+	case "user":
+		*s = UserSetting
+	default:
+		return fmt.Errorf("%q is not a valid scope", string(text))
+	}
+	return nil
+}
+
+// Type is a policy setting value type.
+// Except for [InvalidValue], which represents an invalid policy setting type,
+// and [PreferenceOptionValue], [VisibilityValue], and [DurationValue],
+// which have special handling due to their legacy status in the package,
+// SettingTypes represent the raw value types readable from policy stores.
+type Type int
+
+const (
+	// InvalidValue indicates an invalid policy setting value type.
+	InvalidValue Type = iota
+	// BooleanValue indicates a policy setting whose underlying type is a bool.
+	BooleanValue
+	// IntegerValue indicates a policy setting whose underlying type is a uint64.
+	IntegerValue
+	// StringValue indicates a policy setting whose underlying type is a string.
+	StringValue
+	// StringListValue indicates a policy setting whose underlying type is a []string.
+	StringListValue
+	// PreferenceOptionValue indicates a three-state policy setting whose
+	// underlying type is a string, but the actual value is a [PreferenceOption].
+	PreferenceOptionValue
+	// VisibilityValue indicates a two-state boolean-like policy setting whose
+	// underlying type is a string, but the actual value is a [Visibility].
+	VisibilityValue
+	// DurationValue indicates an interval/period/duration policy setting whose
+	// underlying type is a string, but the actual value is a [time.Duration].
+	DurationValue
+)
+
+// String returns a string representation of t.
+func (t Type) String() string {
+	switch t {
+	case InvalidValue:
+		return "Invalid"
+	case BooleanValue:
+		return "Boolean"
+	case IntegerValue:
+		return "Integer"
+	case StringValue:
+		return "String"
+	case StringListValue:
+		return "StringList"
+	case PreferenceOptionValue:
+		return "PreferenceOption"
+	case VisibilityValue:
+		return "Visibility"
+	case DurationValue:
+		return "Duration"
+	default:
+		panic("unreachable")
+	}
+}
+
+// ValueType is a constraint that allows Go types corresponding to [Type].
+type ValueType interface {
+	bool | uint64 | string | []string | Visibility | PreferenceOption | time.Duration
+}
+
+// Definition defines policy key, scope and value type.
+type Definition struct {
+	key       Key
+	scope     Scope
+	typ       Type
+	platforms PlatformList
+}
+
+// NewDefinition returns a new [Definition] with the specified
+// key, scope, type and supported platforms (see [PlatformList]).
+func NewDefinition(k Key, s Scope, t Type, platforms ...string) *Definition {
+	return &Definition{key: k, scope: s, typ: t, platforms: platforms}
+}
+
+// Key returns a policy setting's identifier.
+func (d *Definition) Key() Key {
+	if d == nil {
+		return ""
+	}
+	return d.key
+}
+
+// Scope reports the broadest [Scope] the policy setting may apply to.
+func (d *Definition) Scope() Scope {
+	if d == nil {
+		return 0
+	}
+	return d.scope
+}
+
+// Type reports the underlying value type of the policy setting.
+func (d *Definition) Type() Type {
+	if d == nil {
+		return InvalidValue
+	}
+	return d.typ
+}
+
+// IsSupported reports whether the policy setting is supported on the current OS.
+func (d *Definition) IsSupported() bool {
+	if d == nil {
+		return false
+	}
+	return d.platforms.HasCurrent()
+}
+
+// SupportedPlatforms reports platforms on which the policy setting is supported.
+// An empty [PlatformList] indicates that s is available on all platforms.
+func (d *Definition) SupportedPlatforms() PlatformList {
+	if d == nil {
+		return nil
+	}
+	return d.platforms
+}
+
+// String implements [fmt.Stringer].
+func (d *Definition) String() string {
+	if d == nil {
+		return "(nil)"
+	}
+	return fmt.Sprintf("%v(%q, %v)", d.scope, d.key, d.typ)
+}
+
+// Equal reports whether d and d2 have the same key, type and scope.
+// It does not check whether both s and s2 are supported on the same platforms.
+func (d *Definition) Equal(d2 *Definition) bool {
+	if d == d2 {
+		return true
+	}
+	if d == nil || d2 == nil {
+		return false
+	}
+	return d.key == d2.key && d.typ == d2.typ && d.scope == d2.scope
+}
+
+// DefinitionMap is a map of setting [Definition] by [Key].
+type DefinitionMap map[Key]*Definition
+
+var (
+	definitions lazy.SyncValue[DefinitionMap]
+
+	definitionsMu   sync.Mutex
+	definitionsList []*Definition
+	definitionsUsed bool
+)
+
+// Register registers a policy setting with the specified key, scope, value type,
+// and an optional list of supported platforms. All policy settings must be
+// registered before any of them can be used. Register panics if called after
+// invoking any functions that use the registered policy definitions. This
+// includes calling [Definitions] or [DefinitionOf] directly, or reading any
+// policy settings via syspolicy.
+func Register(k Key, s Scope, t Type, platforms ...string) {
+	RegisterDefinition(NewDefinition(k, s, t, platforms...))
+}
+
+// RegisterDefinition is like [Register], but accepts a [Definition].
+func RegisterDefinition(d *Definition) {
+	definitionsMu.Lock()
+	defer definitionsMu.Unlock()
+	registerLocked(d)
+}
+
+func registerLocked(d *Definition) {
+	if definitionsUsed {
+		panic("policy definitions are already in use")
+	}
+	definitionsList = append(definitionsList, d)
+}
+
+func settingDefinitions() (DefinitionMap, error) {
+	return definitions.GetErr(func() (DefinitionMap, error) {
+		definitionsMu.Lock()
+		defer definitionsMu.Unlock()
+		definitionsUsed = true
+		return DefinitionMapOf(definitionsList)
+	})
+}
+
+// DefinitionMapOf returns a [DefinitionMap] with the specified settings,
+// or an error if any settings have the same key but different type or scope.
+func DefinitionMapOf(settings []*Definition) (DefinitionMap, error) {
+	m := make(DefinitionMap, len(settings))
+	for _, s := range settings {
+		if existing, exists := m[s.key]; exists {
+			if existing.Equal(s) {
+				// Ignore duplicate setting definitions if they match. It is acceptable
+				// if the same policy setting was registered more than once
+				// (e.g. by the syspolicy package itself and by iOS/Android code).
+				existing.platforms.mergeFrom(s.platforms)
+				continue
+			}
+			return nil, fmt.Errorf("duplicate policy definition: %q", s.key)
+		}
+		m[s.key] = s
+	}
+	return m, nil
+}
+
+// SetDefinitionsForTest allows to register the specified setting definitions
+// for the test duration. It is not concurrency-safe, but unlike [Register],
+// it does not panic and can be called anytime.
+// It returns an error if ds contains two different settings with the same [Key].
+func SetDefinitionsForTest(tb lazy.TB, ds ...*Definition) error {
+	m, err := DefinitionMapOf(ds)
+	if err != nil {
+		return err
+	}
+	definitions.SetForTest(tb, m, err)
+	return nil
+}
+
+// DefinitionOf returns a setting definition by key,
+// or [ErrNoSuchKey] if the specified key does not exist,
+// or an error if there are conflicting policy definitions.
+func DefinitionOf(k Key) (*Definition, error) {
+	ds, err := settingDefinitions()
+	if err != nil {
+		return nil, err
+	}
+	if d, ok := ds[k]; ok {
+		return d, nil
+	}
+	return nil, ErrNoSuchKey
+}
+
+// Definitions returns all registered setting definitions,
+// or an error if different policies were registered under the same name.
+func Definitions() ([]*Definition, error) {
+	ds, err := settingDefinitions()
+	if err != nil {
+		return nil, err
+	}
+	res := make([]*Definition, 0, len(ds))
+	for _, d := range ds {
+		res = append(res, d)
+	}
+	return res, nil
+}
+
+// PlatformList is a list of OSes.
+// An empty list indicates that all possible platforms are supported.
+type PlatformList []string
+
+// Has reports whether l contains the target platform.
+func (l PlatformList) Has(target string) bool {
+	if len(l) == 0 {
+		return true
+	}
+	return slices.ContainsFunc(l, func(os string) bool {
+		return strings.EqualFold(os, target)
+	})
+}
+
+// HasCurrent is like Has, but for the current platform.
+func (l PlatformList) HasCurrent() bool {
+	return l.Has(internal.OS())
+}
+
+// mergeFrom merges l2 into l. Since an empty list indicates no platform restrictions,
+// if either l or l2 is empty, the merged result in l will also be empty.
+func (l *PlatformList) mergeFrom(l2 PlatformList) {
+	switch {
+	case len(*l) == 0:
+		// No-op. An empty list indicates no platform restrictions.
+	case len(l2) == 0:
+		// Merging with an empty list results in an empty list.
+		*l = l2
+	default:
+		// Append, sort and dedup.
+		*l = append(*l, l2...)
+		slices.Sort(*l)
+		*l = slices.Compact(*l)
+	}
+}

util/syspolicy/setting/setting_test.go

@@ -0,0 +1,344 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"tailscale.com/types/lazy"
+	"tailscale.com/types/ptr"
+	"tailscale.com/util/syspolicy/internal"
+)
+
+func TestSettingDefinition(t *testing.T) {
+	tests := []struct {
+		name                   string
+		setting                *Definition
+		osOverride             string
+		wantKey                Key
+		wantScope              Scope
+		wantType               Type
+		wantIsSupported        bool
+		wantSupportedPlatforms PlatformList
+		wantString             string
+	}{
+		{
+			name:            "Nil",
+			setting:         nil,
+			wantKey:         "",
+			wantScope:       0,
+			wantType:        InvalidValue,
+			wantIsSupported: false,
+			wantString:      "(nil)",
+		},
+		{
+			name:            "Device/Invalid",
+			setting:         NewDefinition("TestDevicePolicySetting", DeviceSetting, InvalidValue),
+			wantKey:         "TestDevicePolicySetting",
+			wantScope:       DeviceSetting,
+			wantType:        InvalidValue,
+			wantIsSupported: true,
+			wantString:      `Device("TestDevicePolicySetting", Invalid)`,
+		},
+		{
+			name:            "Device/Integer",
+			setting:         NewDefinition("TestDevicePolicySetting", DeviceSetting, IntegerValue),
+			wantKey:         "TestDevicePolicySetting",
+			wantScope:       DeviceSetting,
+			wantType:        IntegerValue,
+			wantIsSupported: true,
+			wantString:      `Device("TestDevicePolicySetting", Integer)`,
+		},
+		{
+			name:            "Profile/String",
+			setting:         NewDefinition("TestProfilePolicySetting", ProfileSetting, StringValue),
+			wantKey:         "TestProfilePolicySetting",
+			wantScope:       ProfileSetting,
+			wantType:        StringValue,
+			wantIsSupported: true,
+			wantString:      `Profile("TestProfilePolicySetting", String)`,
+		},
+		{
+			name:            "Device/StringList",
+			setting:         NewDefinition("AllowedSuggestedExitNodes", DeviceSetting, StringListValue),
+			wantKey:         "AllowedSuggestedExitNodes",
+			wantScope:       DeviceSetting,
+			wantType:        StringListValue,
+			wantIsSupported: true,
+			wantString:      `Device("AllowedSuggestedExitNodes", StringList)`,
+		},
+		{
+			name:            "Device/PreferenceOption",
+			setting:         NewDefinition("AdvertiseExitNode", DeviceSetting, PreferenceOptionValue),
+			wantKey:         "AdvertiseExitNode",
+			wantScope:       DeviceSetting,
+			wantType:        PreferenceOptionValue,
+			wantIsSupported: true,
+			wantString:      `Device("AdvertiseExitNode", PreferenceOption)`,
+		},
+		{
+			name:            "User/Boolean",
+			setting:         NewDefinition("TestUserPolicySetting", UserSetting, BooleanValue),
+			wantKey:         "TestUserPolicySetting",
+			wantScope:       UserSetting,
+			wantType:        BooleanValue,
+			wantIsSupported: true,
+			wantString:      `User("TestUserPolicySetting", Boolean)`,
+		},
+		{
+			name:            "User/Visibility",
+			setting:         NewDefinition("AdminConsole", UserSetting, VisibilityValue),
+			wantKey:         "AdminConsole",
+			wantScope:       UserSetting,
+			wantType:        VisibilityValue,
+			wantIsSupported: true,
+			wantString:      `User("AdminConsole", Visibility)`,
+		},
+		{
+			name:            "User/Duration",
+			setting:         NewDefinition("KeyExpirationNotice", UserSetting, DurationValue),
+			wantKey:         "KeyExpirationNotice",
+			wantScope:       UserSetting,
+			wantType:        DurationValue,
+			wantIsSupported: true,
+			wantString:      `User("KeyExpirationNotice", Duration)`,
+		},
+		{
+			name:                   "SupportedSetting",
+			setting:                NewDefinition("DesktopPolicySetting", DeviceSetting, StringValue, "macos", "windows"),
+			osOverride:             "windows",
+			wantKey:                "DesktopPolicySetting",
+			wantScope:              DeviceSetting,
+			wantType:               StringValue,
+			wantIsSupported:        true,
+			wantSupportedPlatforms: PlatformList{"macos", "windows"},
+			wantString:             `Device("DesktopPolicySetting", String)`,
+		},
+		{
+			name:                   "UnsupportedSetting",
+			setting:                NewDefinition("AndroidPolicySetting", DeviceSetting, StringValue, "android"),
+			osOverride:             "macos",
+			wantKey:                "AndroidPolicySetting",
+			wantScope:              DeviceSetting,
+			wantType:               StringValue,
+			wantIsSupported:        false,
+			wantSupportedPlatforms: PlatformList{"android"},
+			wantString:             `Device("AndroidPolicySetting", String)`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.osOverride != "" {
+				internal.OSForTesting.SetForTest(t, tt.osOverride, nil)
+			}
+			if !tt.setting.Equal(tt.setting) {
+				t.Errorf("the setting should be equal to itself")
+			}
+			if tt.setting != nil && !tt.setting.Equal(ptr.To(*tt.setting)) {
+				t.Errorf("the setting should be equal to its shallow copy")
+			}
+			if gotKey := tt.setting.Key(); gotKey != tt.wantKey {
+				t.Errorf("Key: got %q, want %q", gotKey, tt.wantKey)
+			}
+			if gotScope := tt.setting.Scope(); gotScope != tt.wantScope {
+				t.Errorf("Scope: got %v, want %v", gotScope, tt.wantScope)
+			}
+			if gotType := tt.setting.Type(); gotType != tt.wantType {
+				t.Errorf("Type: got %v, want %v", gotType, tt.wantType)
+			}
+			if gotIsSupported := tt.setting.IsSupported(); gotIsSupported != tt.wantIsSupported {
+				t.Errorf("IsSupported: got %v, want %v", gotIsSupported, tt.wantIsSupported)
+			}
+			if gotSupportedPlatforms := tt.setting.SupportedPlatforms(); !slices.Equal(gotSupportedPlatforms, tt.wantSupportedPlatforms) {
+				t.Errorf("SupportedPlatforms: got %v, want %v", gotSupportedPlatforms, tt.wantSupportedPlatforms)
+			}
+			if gotString := tt.setting.String(); gotString != tt.wantString {
+				t.Errorf("String: got %v, want %v", gotString, tt.wantString)
+			}
+		})
+	}
+}
+
+func TestRegisterSettingDefinition(t *testing.T) {
+	const testPolicySettingKey Key = "TestPolicySetting"
+	tests := []struct {
+		name    string
+		key     Key
+		wantEq  *Definition
+		wantErr error
+	}{
+		{
+			name:   "GetRegistered",
+			key:    "TestPolicySetting",
+			wantEq: NewDefinition(testPolicySettingKey, DeviceSetting, StringValue),
+		},
+		{
+			name:    "GetNonRegistered",
+			key:     "OtherPolicySetting",
+			wantEq:  nil,
+			wantErr: ErrNoSuchKey,
+		},
+	}
+
+	resetSettingDefinitions(t)
+	Register(testPolicySettingKey, DeviceSetting, StringValue)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := DefinitionOf(tt.key)
+			if gotErr != tt.wantErr {
+				t.Errorf("gotErr %v, wantErr %v", gotErr, tt.wantErr)
+			}
+			if !got.Equal(tt.wantEq) {
+				t.Errorf("got %v, want %v", got, tt.wantEq)
+			}
+		})
+	}
+}
+
+func TestRegisterAfterUsePanics(t *testing.T) {
+	resetSettingDefinitions(t)
+
+	Register("TestPolicySetting", DeviceSetting, StringValue)
+	DefinitionOf("TestPolicySetting")
+
+	func() {
+		defer func() {
+			if gotPanic, wantPanic := recover(), "policy definitions are already in use"; gotPanic != wantPanic {
+				t.Errorf("gotPanic: %q, wantPanic: %q", gotPanic, wantPanic)
+			}
+		}()
+
+		Register("TestPolicySetting", DeviceSetting, StringValue)
+	}()
+}
+
+func TestRegisterDuplicateSettings(t *testing.T) {
+
+	tests := []struct {
+		name       string
+		settings   []*Definition
+		wantEq     *Definition
+		wantErrStr string
+	}{
+		{
+			name: "NoConflict/Exact",
+			settings: []*Definition{
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue),
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue),
+			},
+			wantEq: NewDefinition("TestPolicySetting", DeviceSetting, StringValue),
+		},
+		{
+			name: "NoConflict/MergeOS-First",
+			settings: []*Definition{
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue, "android", "macos"),
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue), // all platforms
+			},
+			wantEq: NewDefinition("TestPolicySetting", DeviceSetting, StringValue), // all platforms
+		},
+		{
+			name: "NoConflict/MergeOS-Second",
+			settings: []*Definition{
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue), // all platforms
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue, "android", "macos"),
+			},
+			wantEq: NewDefinition("TestPolicySetting", DeviceSetting, StringValue), // all platforms
+		},
+		{
+			name: "NoConflict/MergeOS-Both",
+			settings: []*Definition{
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue, "macos"),
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue, "windows"),
+			},
+			wantEq: NewDefinition("TestPolicySetting", DeviceSetting, StringValue, "macos", "windows"),
+		},
+		{
+			name: "Conflict/Scope",
+			settings: []*Definition{
+				NewDefinition("TestPolicySetting", DeviceSetting, StringValue),
+				NewDefinition("TestPolicySetting", UserSetting, StringValue),
+			},
+			wantEq:     nil,
+			wantErrStr: `duplicate policy definition: "TestPolicySetting"`,
+		},
+		{
+			name: "Conflict/Type",
+			settings: []*Definition{
+				NewDefinition("TestPolicySetting", UserSetting, StringValue),
+				NewDefinition("TestPolicySetting", UserSetting, IntegerValue),
+			},
+			wantEq:     nil,
+			wantErrStr: `duplicate policy definition: "TestPolicySetting"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resetSettingDefinitions(t)
+			for _, s := range tt.settings {
+				Register(s.Key(), s.Scope(), s.Type(), s.SupportedPlatforms()...)
+			}
+			got, err := DefinitionOf("TestPolicySetting")
+			var gotErrStr string
+			if err != nil {
+				gotErrStr = err.Error()
+			}
+			if gotErrStr != tt.wantErrStr {
+				t.Fatalf("ErrStr: got %q, want %q", gotErrStr, tt.wantErrStr)
+			}
+			if !got.Equal(tt.wantEq) {
+				t.Errorf("Definition got %v, want %v", got, tt.wantEq)
+			}
+			if !slices.Equal(got.SupportedPlatforms(), tt.wantEq.SupportedPlatforms()) {
+				t.Errorf("SupportedPlatforms got %v, want %v", got.SupportedPlatforms(), tt.wantEq.SupportedPlatforms())
+			}
+		})
+	}
+}
+
+func TestListSettingDefinitions(t *testing.T) {
+	definitions := []*Definition{
+		NewDefinition("TestDevicePolicySetting", DeviceSetting, IntegerValue),
+		NewDefinition("TestProfilePolicySetting", ProfileSetting, StringValue),
+		NewDefinition("TestUserPolicySetting", UserSetting, BooleanValue),
+		NewDefinition("TestStringListPolicySetting", DeviceSetting, StringListValue),
+	}
+	if err := SetDefinitionsForTest(t, definitions...); err != nil {
+		t.Fatalf("SetDefinitionsForTest failed: %v", err)
+	}
+
+	cmp := func(l, r *Definition) int {
+		return strings.Compare(string(l.Key()), string(r.Key()))
+	}
+	want := append([]*Definition{}, definitions...)
+	slices.SortFunc(want, cmp)
+
+	got, err := Definitions()
+	if err != nil {
+		t.Fatalf("Definitions failed: %v", err)
+	}
+	slices.SortFunc(got, cmp)
+
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func resetSettingDefinitions(t *testing.T) {
+	t.Cleanup(func() {
+		definitionsMu.Lock()
+		definitionsList = nil
+		definitions = lazy.SyncValue[DefinitionMap]{}
+		definitionsUsed = false
+		definitionsMu.Unlock()
+	})
+
+	definitionsMu.Lock()
+	definitionsList = nil
+	definitions = lazy.SyncValue[DefinitionMap]{}
+	definitionsUsed = false
+	definitionsMu.Unlock()
+}

util/syspolicy/setting/snapshot.go

@@ -0,0 +1,173 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"slices"
+	"strings"
+
+	xmaps "golang.org/x/exp/maps"
+	"tailscale.com/util/deephash"
+)
+
+// Snapshot is an immutable collection of ([Key], [RawItem]) pairs, representing
+// a set of policy settings applied at a specific moment in time.
+// A nil pointer to [Snapshot] is valid.
+type Snapshot struct {
+	m       map[Key]RawItem
+	sig     deephash.Sum // of m
+	summary Summary
+}
+
+// NewSnapshot returns a new [Snapshot] with the specified items and options.
+func NewSnapshot(items map[Key]RawItem, opts ...SummaryOption) *Snapshot {
+	return &Snapshot{m: xmaps.Clone(items), sig: deephash.Hash(&items), summary: SummaryWith(opts...)}
+}
+
+// All returns a map of all policy settings in s.
+// The returned map must not be modified.
+func (s *Snapshot) All() map[Key]RawItem {
+	if s == nil {
+		return nil
+	}
+	// TODO(nickkhyl): return iter.Seq2[[Key], [RawItem]] in Go 1.23,
+	// and remove [keyItemPair].
+	return s.m
+}
+
+// Get returns the value of the policy setting with the specified key
+// or nil if it is not configured or has an error.
+func (s *Snapshot) Get(k Key) any {
+	v, _ := s.GetErr(k)
+	return v
+}
+
+// GetErr returns the value of the policy setting with the specified key,
+// [ErrNotConfigured] if it is not configured, or an error returned by
+// the policy Store if the policy setting could not be read.
+func (s *Snapshot) GetErr(k Key) (any, error) {
+	if s != nil {
+		if s, ok := s.m[k]; ok {
+			return s.Value(), s.Error()
+		}
+	}
+	return nil, ErrNotConfigured
+}
+
+// GetSetting returns the untyped policy setting with the specified key and true
+// if a policy setting with such key has been configured;
+// otherwise, it returns zero, false.
+func (s *Snapshot) GetSetting(k Key) (setting RawItem, ok bool) {
+	setting, ok = s.m[k]
+	return setting, ok
+}
+
+// Equal reports whether s and s2 are equal.
+func (s *Snapshot) Equal(s2 *Snapshot) bool {
+	if !s.EqualItems(s2) {
+		return false
+	}
+	return s.Summary() == s2.Summary()
+}
+
+// EqualItems reports whether items in s and s2 are equal.
+func (s *Snapshot) EqualItems(s2 *Snapshot) bool {
+	if s == s2 {
+		return true
+	}
+	if s.Len() != s2.Len() {
+		return false
+	}
+	if s.Len() == 0 {
+		return true
+	}
+	return s.sig == s2.sig
+}
+
+// Keys return an iterator over keys in s. The iteration order is not specified
+// and is not guaranteed to be the same from one call to the next.
+func (s *Snapshot) Keys() []Key {
+	if s.m == nil {
+		return nil
+	}
+	// TODO(nickkhyl): return iter.Seq[Key] in Go 1.23.
+	return xmaps.Keys(s.m)
+}
+
+// Len reports the number of [RawItem]s in s.
+func (s *Snapshot) Len() int {
+	if s == nil {
+		return 0
+	}
+	return len(s.m)
+}
+
+// Summary returns information about s as a whole rather than about specific [RawItem]s in it.
+func (s *Snapshot) Summary() Summary {
+	if s == nil {
+		return Summary{}
+	}
+	return s.summary
+}
+
+// String implements [fmt.Stringer]
+func (s *Snapshot) String() string {
+	if s.Len() == 0 && s.Summary().IsEmpty() {
+		return "{Empty}"
+	}
+	keys := s.Keys()
+	slices.Sort(keys)
+	var sb strings.Builder
+	if !s.summary.IsEmpty() {
+		sb.WriteRune('{')
+		if s.Len() == 0 {
+			sb.WriteString("Empty, ")
+		}
+		sb.WriteString(s.summary.String())
+		sb.WriteRune('}')
+	}
+	for _, k := range keys {
+		if sb.Len() != 0 {
+			sb.WriteRune('\n')
+		}
+		sb.WriteString(string(k))
+		sb.WriteString(" = ")
+		sb.WriteString(s.m[k].String())
+	}
+	return sb.String()
+}
+
+// MergeSnapshots returns a [Snapshot] that contains all [RawItem]s
+// from snapshot1 and snapshot2 and the [Summary] with the narrower [PolicyScope].
+// If there's a conflict between policy settings in the two snapshots,
+// the policy settings from the snapshot with the broader scope take precedence.
+// In other words, policy settings configured for the [DeviceScope] win
+// over policy settings configured for a user scope.
+func MergeSnapshots(snapshot1, snapshot2 *Snapshot) *Snapshot {
+	scope1, ok1 := snapshot1.Summary().Scope().GetOk()
+	scope2, ok2 := snapshot2.Summary().Scope().GetOk()
+	if ok1 && ok2 && scope1.StrictlyContains(scope2) {
+		// Swap snapshots if snapshot1 has higher precedence than snapshot2.
+		snapshot1, snapshot2 = snapshot2, snapshot1
+	}
+	if snapshot2.Len() == 0 {
+		return snapshot1
+	}
+	summaryOpts := make([]SummaryOption, 0, 2)
+	if scope, ok := snapshot1.Summary().Scope().GetOk(); ok {
+		// Use the scope from snapshot1, if present, which is the more specific snapshot.
+		summaryOpts = append(summaryOpts, scope)
+	}
+	if snapshot1.Len() == 0 {
+		if origin, ok := snapshot2.Summary().Origin().GetOk(); ok {
+			// Use the origin from snapshot2 if snapshot1 is empty.
+			summaryOpts = append(summaryOpts, origin)
+		}
+		return &Snapshot{snapshot2.m, snapshot2.sig, SummaryWith(summaryOpts...)}
+	}
+	m := make(map[Key]RawItem, snapshot1.Len()+snapshot2.Len())
+	xmaps.Copy(m, snapshot1.m)
+	xmaps.Copy(m, snapshot2.m) // snapshot2 has higher precedence
+	return &Snapshot{m, deephash.Hash(&m), SummaryWith(summaryOpts...)}
+}

util/syspolicy/setting/snapshot_test.go

@@ -0,0 +1,435 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"testing"
+	"time"
+)
+
+func TestMergeSnapshots(t *testing.T) {
+	tests := []struct {
+		name   string
+		s1, s2 *Snapshot
+		want   *Snapshot
+	}{
+		{
+			name: "both-nil",
+			s1:   nil,
+			s2:   nil,
+			want: NewSnapshot(map[Key]RawItem{}),
+		},
+		{
+			name: "both-empty",
+			s1:   NewSnapshot(map[Key]RawItem{}),
+			s2:   NewSnapshot(map[Key]RawItem{}),
+			want: NewSnapshot(map[Key]RawItem{}),
+		},
+		{
+			name: "first-nil",
+			s1:   nil,
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}),
+		},
+		{
+			name: "first-empty",
+			s1:   NewSnapshot(map[Key]RawItem{}),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+		},
+		{
+			name: "second-nil",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}),
+			s2: nil,
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}),
+		},
+		{
+			name: "second-empty",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			s2: NewSnapshot(map[Key]RawItem{}),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+		},
+		{
+			name: "no-conflicts",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting4": {value: 2 * time.Hour},
+				"Setting5": {value: VisibleByPolicy},
+				"Setting6": {value: ShowChoiceByPolicy},
+			}),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+				"Setting4": {value: 2 * time.Hour},
+				"Setting5": {value: VisibleByPolicy},
+				"Setting6": {value: ShowChoiceByPolicy},
+			}),
+		},
+		{
+			name: "with-conflicts",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 456},
+				"Setting3": {value: false},
+				"Setting4": {value: 2 * time.Hour},
+			}),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 456},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+				"Setting4": {value: 2 * time.Hour},
+			}),
+		},
+		{
+			name: "with-scope-first-wins",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}, DeviceScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 456},
+				"Setting3": {value: false},
+				"Setting4": {value: 2 * time.Hour},
+			}, CurrentUserScope),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+				"Setting4": {value: 2 * time.Hour},
+			}, CurrentUserScope),
+		},
+		{
+			name: "with-scope-second-wins",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}, CurrentUserScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 456},
+				"Setting3": {value: false},
+				"Setting4": {value: 2 * time.Hour},
+			}, DeviceScope),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 456},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+				"Setting4": {value: 2 * time.Hour},
+			}, CurrentUserScope),
+		},
+		{
+			name: "with-scope-both-empty",
+			s1:   NewSnapshot(map[Key]RawItem{}, CurrentUserScope),
+			s2:   NewSnapshot(map[Key]RawItem{}, DeviceScope),
+			want: NewSnapshot(map[Key]RawItem{}, CurrentUserScope),
+		},
+		{
+			name: "with-scope-first-empty",
+			s1:   NewSnapshot(map[Key]RawItem{}, CurrentUserScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true}},
+				DeviceScope, NewNamedOrigin("TestPolicy", DeviceScope)),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}, CurrentUserScope, NewNamedOrigin("TestPolicy", DeviceScope)),
+		},
+		{
+			name: "with-scope-second-empty",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}, CurrentUserScope),
+			s2: NewSnapshot(map[Key]RawItem{}),
+			want: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}, CurrentUserScope),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := MergeSnapshots(tt.s1, tt.s2)
+			if !got.Equal(tt.want) {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSnapshotEqual(t *testing.T) {
+	tests := []struct {
+		name           string
+		s1, s2         *Snapshot
+		wantEqual      bool
+		wantEqualItems bool
+	}{
+		{
+			name:           "nil-nil",
+			s1:             nil,
+			s2:             nil,
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name:           "nil-empty",
+			s1:             nil,
+			s2:             NewSnapshot(map[Key]RawItem{}),
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name:           "empty-nil",
+			s1:             NewSnapshot(map[Key]RawItem{}),
+			s2:             nil,
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name:           "empty-empty",
+			s1:             NewSnapshot(map[Key]RawItem{}),
+			s2:             NewSnapshot(map[Key]RawItem{}),
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name: "first-nil",
+			s1:   nil,
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			wantEqual:      false,
+			wantEqualItems: false,
+		},
+		{
+			name: "first-empty",
+			s1:   NewSnapshot(map[Key]RawItem{}),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			wantEqual:      false,
+			wantEqualItems: false,
+		},
+		{
+			name: "second-nil",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: true},
+			}),
+			s2:             nil,
+			wantEqual:      false,
+			wantEqualItems: false,
+		},
+		{
+			name: "second-empty",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			s2:             NewSnapshot(map[Key]RawItem{}),
+			wantEqual:      false,
+			wantEqualItems: false,
+		},
+		{
+			name: "same-items-same-order-no-scope",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}),
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name: "same-items-same-order-same-scope",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}, DeviceScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}, DeviceScope),
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name: "same-items-different-order-same-scope",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}, DeviceScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting3": {value: false},
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+			}, DeviceScope),
+			wantEqual:      true,
+			wantEqualItems: true,
+		},
+		{
+			name: "same-items-same-order-different-scope",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}, DeviceScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}, CurrentUserScope),
+			wantEqual:      false,
+			wantEqualItems: true,
+		},
+		{
+			name: "different-items-same-scope",
+			s1: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 123},
+				"Setting2": {value: "String"},
+				"Setting3": {value: false},
+			}, DeviceScope),
+			s2: NewSnapshot(map[Key]RawItem{
+				"Setting4": {value: 2 * time.Hour},
+				"Setting5": {value: VisibleByPolicy},
+				"Setting6": {value: ShowChoiceByPolicy},
+			}, DeviceScope),
+			wantEqual:      false,
+			wantEqualItems: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotEqual := tt.s1.Equal(tt.s2); gotEqual != tt.wantEqual {
+				t.Errorf("WantEqual: got %v, want %v", gotEqual, tt.wantEqual)
+			}
+			if gotEqualItems := tt.s1.EqualItems(tt.s2); gotEqualItems != tt.wantEqualItems {
+				t.Errorf("WantEqualItems: got %v, want %v", gotEqualItems, tt.wantEqualItems)
+			}
+		})
+	}
+}
+
+func TestSnapshotString(t *testing.T) {
+	tests := []struct {
+		name       string
+		snapshot   *Snapshot
+		wantString string
+	}{
+		{
+			name:       "nil",
+			snapshot:   nil,
+			wantString: "{Empty}",
+		},
+		{
+			name:       "empty",
+			snapshot:   NewSnapshot(nil),
+			wantString: "{Empty}",
+		},
+		{
+			name:       "empty-with-scope",
+			snapshot:   NewSnapshot(nil, DeviceScope),
+			wantString: "{Empty, Device}",
+		},
+		{
+			name:       "empty-with-origin",
+			snapshot:   NewSnapshot(nil, NewNamedOrigin("Test Policy", DeviceScope)),
+			wantString: "{Empty, Test Policy (Device)}",
+		},
+		{
+			name: "non-empty",
+			snapshot: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 2 * time.Hour},
+				"Setting2": {value: VisibleByPolicy},
+				"Setting3": {value: ShowChoiceByPolicy},
+			}, NewNamedOrigin("Test Policy", DeviceScope)),
+			wantString: `{Test Policy (Device)}
+Setting1 = 2h0m0s
+Setting2 = show
+Setting3 = user-decides`,
+		},
+		{
+			name: "non-empty-with-item-origin",
+			snapshot: NewSnapshot(map[Key]RawItem{
+				"Setting1": {value: 42, origin: NewNamedOrigin("Test Policy", DeviceScope)},
+			}),
+			wantString: `Setting1 = 42 - {Test Policy (Device)}`,
+		},
+		{
+			name: "non-empty-with-item-error",
+			snapshot: NewSnapshot(map[Key]RawItem{
+				"Setting1": {err: NewErrorText("bang!")},
+			}),
+			wantString: `Setting1 = Error{"bang!"}`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotString := tt.snapshot.String(); gotString != tt.wantString {
+				t.Errorf("got %v\nwant %v", gotString, tt.wantString)
+			}
+		})
+	}
+}

util/syspolicy/setting/summary.go

@@ -0,0 +1,100 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	jsonv2 "github.com/go-json-experiment/json"
+	"github.com/go-json-experiment/json/jsontext"
+	"tailscale.com/types/opt"
+)
+
+// Summary is an immutable [PolicyScope] and [Origin].
+type Summary struct {
+	data summary
+}
+
+type summary struct {
+	Scope  opt.Value[PolicyScope] `json:",omitzero"`
+	Origin opt.Value[Origin]      `json:",omitzero"`
+}
+
+// SummaryWith returns a [Summary] with the specified options.
+func SummaryWith(opts ...SummaryOption) Summary {
+	var summary Summary
+	for _, o := range opts {
+		o.applySummaryOption(&summary)
+	}
+	return summary
+}
+
+// IsEmpty reports whether s is empty.
+func (s Summary) IsEmpty() bool {
+	return s == Summary{}
+}
+
+// Scope reports the [PolicyScope] in s.
+func (s Summary) Scope() opt.Value[PolicyScope] {
+	return s.data.Scope
+}
+
+// Origin reports the [Origin] in s.
+func (s Summary) Origin() opt.Value[Origin] {
+	return s.data.Origin
+}
+
+// String implements [fmt.Stringer].
+func (s Summary) String() string {
+	if s.IsEmpty() {
+		return "{Empty}"
+	}
+	if origin, ok := s.data.Origin.GetOk(); ok {
+		return origin.String()
+	}
+	return s.data.Scope.String()
+}
+
+// MarshalJSONV2 implements [jsonv2.MarshalerV2].
+func (s Summary) MarshalJSONV2(out *jsontext.Encoder, opts jsonv2.Options) error {
+	return jsonv2.MarshalEncode(out, &s.data, opts)
+}
+
+// UnmarshalJSONV2 implements [jsonv2.UnmarshalerV2].
+func (s *Summary) UnmarshalJSONV2(in *jsontext.Decoder, opts jsonv2.Options) error {
+	return jsonv2.UnmarshalDecode(in, &s.data, opts)
+}
+
+// MarshalJSON implements [json.Marshaler].
+func (s Summary) MarshalJSON() ([]byte, error) {
+	return jsonv2.Marshal(s) // uses MarshalJSONV2
+}
+
+// UnmarshalJSON implements [json.Unmarshaler].
+func (s *Summary) UnmarshalJSON(b []byte) error {
+	return jsonv2.Unmarshal(b, s) // uses UnmarshalJSONV2
+}
+
+// SummaryOption is an option that configures [Summary]
+// The following are allowed options:
+//
+//   - [Summary]
+//   - [PolicyScope]
+//   - [Origin]
+type SummaryOption interface {
+	applySummaryOption(summary *Summary)
+}
+
+func (s PolicyScope) applySummaryOption(summary *Summary) {
+	summary.data.Scope.Set(s)
+}
+
+func (o Origin) applySummaryOption(summary *Summary) {
+	summary.data.Origin.Set(o)
+	if !summary.data.Scope.IsSet() {
+		summary.data.Scope.Set(o.Scope())
+	}
+}
+
+func (s Summary) applySummaryOption(summary *Summary) {
+	*summary = s
+}

util/syspolicy/setting/types.go

@@ -0,0 +1,136 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package setting
+
+import (
+	"encoding"
+)
+
+// PreferenceOption is a policy that governs whether a boolean variable
+// is forcibly assigned an administrator-defined value, or allowed to receive
+// a user-defined value.
+type PreferenceOption byte
+
+const (
+	ShowChoiceByPolicy PreferenceOption = iota
+	NeverByPolicy
+	AlwaysByPolicy
+)
+
+// Show returns if the UI option that controls the choice administered by this
+// policy should be shown. Currently this is true if and only if the policy is
+// [ShowChoiceByPolicy].
+func (p PreferenceOption) Show() bool {
+	return p == ShowChoiceByPolicy
+}
+
+// ShouldEnable checks if the choice administered by this policy should be
+// enabled. If the administrator has chosen a setting, the administrator's
+// setting is returned, otherwise userChoice is returned.
+func (p PreferenceOption) ShouldEnable(userChoice bool) bool {
+	switch p {
+	case NeverByPolicy:
+		return false
+	case AlwaysByPolicy:
+		return true
+	default:
+		return userChoice
+	}
+}
+
+// IsAlways reports whether the preference should always be enabled.
+func (p PreferenceOption) IsAlways() bool {
+	return p == AlwaysByPolicy
+}
+
+// IsNever reports whether the preference should always be disabled.
+func (p PreferenceOption) IsNever() bool {
+	return p == NeverByPolicy
+}
+
+// WillOverride checks if the choice administered by the policy is different
+// from the user's choice.
+func (p PreferenceOption) WillOverride(userChoice bool) bool {
+	return p.ShouldEnable(userChoice) != userChoice
+}
+
+// String returns a string representation of p.
+func (p PreferenceOption) String() string {
+	switch p {
+	case AlwaysByPolicy:
+		return "always"
+	case NeverByPolicy:
+		return "never"
+	default:
+		return "user-decides"
+	}
+}
+
+// MarshalText implements [encoding.TextMarshaler].
+func (p *PreferenceOption) MarshalText() (text []byte, err error) {
+	return []byte(p.String()), nil
+}
+
+// UnmarshalText implements [encoding.TextUnmarshaler].
+// It never fails and sets p to [ShowChoiceByPolicy] if the specified text
+// does not represent a valid [PreferenceOption].
+func (p *PreferenceOption) UnmarshalText(text []byte) error {
+	switch string(text) {
+	case "always":
+		*p = AlwaysByPolicy
+	case "never":
+		*p = NeverByPolicy
+	default:
+		*p = ShowChoiceByPolicy
+	}
+	return nil
+}
+
+// Visibility is a policy that controls whether or not a particular
+// component of a user interface is to be shown.
+type Visibility byte
+
+var (
+	_ encoding.TextMarshaler   = (*Visibility)(nil)
+	_ encoding.TextUnmarshaler = (*Visibility)(nil)
+)
+
+const (
+	VisibleByPolicy Visibility = 'v'
+	HiddenByPolicy  Visibility = 'h'
+)
+
+// Show reports whether the UI option administered by this policy should be shown.
+// Currently this is true if the policy is not [hiddenByPolicy].
+func (v Visibility) Show() bool {
+	return v != HiddenByPolicy
+}
+
+// String returns a string representation of v.
+func (v Visibility) String() string {
+	switch v {
+	case 'h':
+		return "hide"
+	default:
+		return "show"
+	}
+}
+
+// MarshalText implements [encoding.TextMarshaler].
+func (v Visibility) MarshalText() (text []byte, err error) {
+	return []byte(v.String()), nil
+}
+
+// UnmarshalText implements [encoding.TextUnmarshaler].
+// It never fails and sets v to [VisibleByPolicy] if the specified text
+// does not represent a valid [Visibility].
+func (v *Visibility) UnmarshalText(text []byte) error {
+	switch string(text) {
+	case "hide":
+		*v = HiddenByPolicy
+	default:
+		*v = VisibleByPolicy
+	}
+	return nil
+}

util/syspolicy/syspolicy.go

@@ -7,6 +7,8 @@ package syspolicy
 import (
 	"errors"
 	"time"
+
+	"tailscale.com/util/syspolicy/setting"
 )
 
 func GetString(key Key, defaultValue string) (string, error) {
@@ -45,78 +47,20 @@ func GetStringArray(key Key, defaultValue []string) ([]string, error) {
 	return v, err
 }
 
-// PreferenceOption is a policy that governs whether a boolean variable
-// is forcibly assigned an administrator-defined value, or allowed to receive
-// a user-defined value.
-type PreferenceOption int
-
-const (
-	showChoiceByPolicy PreferenceOption = iota
-	neverByPolicy
-	alwaysByPolicy
-)
-
-// Show returns if the UI option that controls the choice administered by this
-// policy should be shown. Currently this is true if and only if the policy is
-// showChoiceByPolicy.
-func (p PreferenceOption) Show() bool {
-	return p == showChoiceByPolicy
-}
-
-// ShouldEnable checks if the choice administered by this policy should be
-// enabled. If the administrator has chosen a setting, the administrator's
-// setting is returned, otherwise userChoice is returned.
-func (p PreferenceOption) ShouldEnable(userChoice bool) bool {
-	switch p {
-	case neverByPolicy:
-		return false
-	case alwaysByPolicy:
-		return true
-	default:
-		return userChoice
-	}
-}
-
-// WillOverride checks if the choice administered by the policy is different
-// from the user's choice.
-func (p PreferenceOption) WillOverride(userChoice bool) bool {
-	return p.ShouldEnable(userChoice) != userChoice
-}
-
 // GetPreferenceOption loads a policy from the registry that can be
 // managed by an enterprise policy management system and allows administrative
 // overrides of users' choices in a way that we do not want tailcontrol to have
 // the authority to set. It describes user-decides/always/never options, where
 // "always" and "never" remove the user's ability to make a selection. If not
 // present or set to a different value, "user-decides" is the default.
-func GetPreferenceOption(name Key) (PreferenceOption, error) {
-	opt, err := GetString(name, "user-decides")
+func GetPreferenceOption(name Key) (setting.PreferenceOption, error) {
+	s, err := GetString(name, "user-decides")
 	if err != nil {
-		return showChoiceByPolicy, err
+		return setting.ShowChoiceByPolicy, err
 	}
-	switch opt {
-	case "always":
-		return alwaysByPolicy, nil
-	case "never":
-		return neverByPolicy, nil
-	default:
-		return showChoiceByPolicy, nil
-	}
-}
-
-// Visibility is a policy that controls whether or not a particular
-// component of a user interface is to be shown.
-type Visibility byte
-
-const (
-	visibleByPolicy Visibility = 'v'
-	hiddenByPolicy  Visibility = 'h'
-)
-
-// Show reports whether the UI option administered by this policy should be shown.
-// Currently this is true if and only if the policy is visibleByPolicy.
-func (p Visibility) Show() bool {
-	return p == visibleByPolicy
+	var opt setting.PreferenceOption
+	err = opt.UnmarshalText([]byte(s))
+	return opt, err
 }
 
 // GetVisibility loads a policy from the registry that can be managed
@@ -124,17 +68,14 @@ func (p Visibility) Show() bool {
 // for UI elements. The registry value should be a string set to "show" (return
 // true) or "hide" (return true). If not present or set to a different value,
 // "show" (return false) is the default.
-func GetVisibility(name Key) (Visibility, error) {
-	opt, err := GetString(name, "show")
+func GetVisibility(name Key) (setting.Visibility, error) {
+	s, err := GetString(name, "show")
 	if err != nil {
-		return visibleByPolicy, err
-	}
-	switch opt {
-	case "hide":
-		return hiddenByPolicy, nil
-	default:
-		return visibleByPolicy, nil
+		return setting.VisibleByPolicy, err
 	}
+	var visibility setting.Visibility
+	visibility.UnmarshalText([]byte(s))
+	return visibility, nil
 }
 
 // GetDuration loads a policy from the registry that can be managed

util/syspolicy/syspolicy_test.go

@@ -8,6 +8,8 @@ import (
 	"slices"
 	"testing"
 	"time"
+
+	"tailscale.com/util/syspolicy/setting"
 )
 
 // testHandler encompasses all data types returned when testing any of the syspolicy
@@ -230,38 +232,38 @@ func TestGetPreferenceOption(t *testing.T) {
 		key          Key
 		handlerValue string
 		handlerError error
-		wantValue    PreferenceOption
+		wantValue    setting.PreferenceOption
 		wantError    error
 	}{
 		{
 			name:         "always by policy",
 			key:          EnableIncomingConnections,
 			handlerValue: "always",
-			wantValue:    alwaysByPolicy,
+			wantValue:    setting.AlwaysByPolicy,
 		},
 		{
 			name:         "never by policy",
 			key:          EnableIncomingConnections,
 			handlerValue: "never",
-			wantValue:    neverByPolicy,
+			wantValue:    setting.NeverByPolicy,
 		},
 		{
 			name:         "use default",
 			key:          EnableIncomingConnections,
 			handlerValue: "",
-			wantValue:    showChoiceByPolicy,
+			wantValue:    setting.ShowChoiceByPolicy,
 		},
 		{
 			name:         "read non-existing value",
 			key:          EnableIncomingConnections,
 			handlerError: ErrNoSuchKey,
-			wantValue:    showChoiceByPolicy,
+			wantValue:    setting.ShowChoiceByPolicy,
 		},
 		{
 			name:         "other error is returned",
 			key:          EnableIncomingConnections,
 			handlerError: someOtherError,
-			wantValue:    showChoiceByPolicy,
+			wantValue:    setting.ShowChoiceByPolicy,
 			wantError:    someOtherError,
 		},
 	}
@@ -291,34 +293,34 @@ func TestGetVisibility(t *testing.T) {
 		key          Key
 		handlerValue string
 		handlerError error
-		wantValue    Visibility
+		wantValue    setting.Visibility
 		wantError    error
 	}{
 		{
 			name:         "hidden by policy",
 			key:          AdminConsoleVisibility,
 			handlerValue: "hide",
-			wantValue:    hiddenByPolicy,
+			wantValue:    setting.HiddenByPolicy,
 		},
 		{
 			name:         "visibility default",
 			key:          AdminConsoleVisibility,
 			handlerValue: "show",
-			wantValue:    visibleByPolicy,
+			wantValue:    setting.VisibleByPolicy,
 		},
 		{
 			name:         "read non-existing value",
 			key:          AdminConsoleVisibility,
 			handlerValue: "show",
 			handlerError: ErrNoSuchKey,
-			wantValue:    visibleByPolicy,
+			wantValue:    setting.VisibleByPolicy,
 		},
 		{
 			name:         "other error is returned",
 			key:          AdminConsoleVisibility,
 			handlerValue: "show",
 			handlerError: someOtherError,
-			wantValue:    visibleByPolicy,
+			wantValue:    setting.VisibleByPolicy,
 			wantError:    someOtherError,
 		},
 	}

wgengine/magicsock/magicsock_linux.go

@@ -32,6 +32,9 @@ const (
 // Enable/disable using raw sockets to receive disco traffic.
 var debugDisableRawDisco = envknob.RegisterBool("TS_DEBUG_DISABLE_RAW_DISCO")
 
+// debugRawDiscoReads enables logging of raw disco reads.
+var debugRawDiscoReads = envknob.RegisterBool("TS_DEBUG_RAW_DISCO")
+
 // These are our BPF filters that we use for testing packets.
 var (
 	magicsockFilterV4 = []bpf.Instruction{
@@ -211,6 +214,9 @@ func (c *Conn) receiveDisco(pc net.PacketConn, isIPV6 bool) {
 	var buf [1500]byte
 	for {
 		n, src, err := pc.ReadFrom(buf[:])
+		if debugRawDiscoReads() {
+			c.logf("raw disco read from %v = (%v, %v)", src, n, err)
+		}
 		if errors.Is(err, net.ErrClosed) {
 			return
 		} else if err != nil {