net/netmon: add netmon hook for os-derived network change notifications

Update tailscale/corp#20677

While hunting down DNS issues, it was observed that
on apple platforms, w attempt to rebind to interfaces before they're
completely configured by the system.

The AF_ROUTE messages aren't lying, they're just not
taking into account other setup that has to happen,
notably the system setting the interface nameservers.
When we react based on the route updates, it's often
too quickly.  This happens frequently on wakes-from-sleep.

This replumbs netmon so that (optionally) rather than reacting to every
system message by checking for interesting changes, we
use that only to update our state, and recheck for interesting
network changes when we get a poke from the upper layers
indicating that a network change has 'completed'.  Notably on
Apple, NWPathMonitor seems to call us back at the correct
time(s) to check for potentially interesting interface changes
that have been observed via AF_ROUTE and to react to them.

This seems to reduce thrashing during network
transitions, and avoids some of the bad states we were
previously ending up in.

This is entirely opt in during the construction of the netmon
and will have no effect until/unless a specific client chooses
to use it.

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

.gitignore

@@ -43,3 +43,9 @@ client/web/build/assets
 
 /gocross
 /dist
+
+# Ignore xcode userstate and workspace data
+*.xcuserstate
+*.xcworkspacedata
+/tstest/tailmac/bin
+/tstest/tailmac/build

Dockerfile

@@ -42,7 +42,7 @@ RUN go install \
     gvisor.dev/gvisor/pkg/tcpip/stack \
     golang.org/x/crypto/ssh \
     golang.org/x/crypto/acme \
-    nhooyr.io/websocket \
+    github.com/coder/websocket \
     github.com/mdlayher/netlink
 
 COPY . .

Makefile

@@ -117,7 +117,8 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
+	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
 
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"

VERSION.txt

@@ -1 +1 @@
-1.71.0
+1.73.0

client/tailscale/acl.go

@@ -286,6 +286,9 @@ type UserRuleMatch struct {
 	Users      []string `json:"users"`
 	Ports      []string `json:"ports"`
 	LineNumber int      `json:"lineNumber"`
+	// Via is the list of targets through which Users can access Ports.
+	// See https://tailscale.com/kb/1378/via for more information.
+	Via []string `json:"via,omitempty"`
 
 	// Postures is a list of posture policies that are
 	// associated with this match. The rules can be looked

client/tailscale/localclient.go

@@ -69,6 +69,14 @@ type LocalClient struct {
 	// connecting to the GUI client variants.
 	UseSocketOnly bool
 
+	// OmitAuth, if true, omits sending the local Tailscale daemon any
+	// authentication token that might be required by the platform.
+	//
+	// As of 2024-08-12, only macOS uses an authentication token. OmitAuth is
+	// meant for when Dial is set and the LocalAPI is being proxied to a
+	// different operating system, such as in integration tests.
+	OmitAuth bool
+
 	// tsClient does HTTP requests to the local Tailscale daemon.
 	// It's lazily initialized on first use.
 	tsClient     *http.Client
@@ -124,8 +132,10 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 			},
 		}
 	})
-	if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
-		req.SetBasicAuth("", token)
+	if !lc.OmitAuth {
+		if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
+			req.SetBasicAuth("", token)
+		}
 	}
 	return lc.tsClient.Do(req)
 }

cmd/containerboot/main.go

@@ -52,6 +52,12 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
+//   - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
+//     served at /healthz at the provided address, which should be in form [<address>]:<port>.
+//     If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
+//     The health endpoint will return 200 OK if this node has at least one tailnet IP address,
+//     otherwise returns 503.
+//     NB: the health criteria might change in the future.
 //   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
 //     directory that containers tailscaled config in file. The config file needs to be
 //     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
@@ -95,6 +101,7 @@ import (
 	"log"
 	"math"
 	"net"
+	"net/http"
 	"net/netip"
 	"os"
 	"os/exec"
@@ -158,6 +165,7 @@ func main() {
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
 		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -349,6 +357,9 @@ authLoop:
 
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
+
+		h             = &healthz{} // http server for the healthz endpoint
+		healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
 	)
 	if cfg.ServeConfigPath != "" {
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
@@ -565,6 +576,13 @@ runLoop:
 						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
+
+				if cfg.HealthCheckAddrPort != "" {
+					h.Lock()
+					h.hasAddrs = len(addrs) != 0
+					h.Unlock()
+					healthzRunner()
+				}
 			}
 			if !startupTasksDone {
 				// For containerboot instances that act as TCP
@@ -1152,7 +1170,8 @@ type settings struct {
 	// PodIP is the IP of the Pod if running in Kubernetes. This is used
 	// when setting up rules to proxy cluster traffic to cluster ingress
 	// target.
-	PodIP string
+	PodIP               string
+	HealthCheckAddrPort string
 }
 
 func (s *settings) validate() error {
@@ -1201,6 +1220,11 @@ func (s *settings) validate() error {
 	if s.EnableForwardingOptimizations && s.UserspaceMode {
 		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
 	}
+	if s.HealthCheckAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+		}
+	}
 	return nil
 }
 
@@ -1374,3 +1398,41 @@ func tailscaledConfigFilePath() string {
 	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
 	return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer))
 }
+
+// healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type healthz struct {
+	sync.Mutex
+	hasAddrs bool
+}
+
+func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+	if h.hasAddrs {
+		w.Write([]byte("ok"))
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
+	}
+}
+
+// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
+// provided address. A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func runHealthz(addr string, h *healthz) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", h)
+	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
+	hs := &http.Server{Handler: mux}
+
+	go func() {
+		if err := hs.Serve(lis); err != nil {
+			log.Fatalf("failed running health endpoint: %v", err)
+		}
+	}()
+}

cmd/derper/depaware.txt

@@ -7,10 +7,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/cmd/derper+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -82,10 +86,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoiface                from google.golang.org/protobuf/internal/impl+
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
-        nhooyr.io/websocket                                          from tailscale.com/cmd/derper+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -146,9 +146,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -159,6 +161,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
@@ -180,6 +184,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/derper/websocket.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"strings"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/derp"
 	"tailscale.com/net/wsconn"
 )

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/k8s-operator/depaware.txt

@@ -80,6 +80,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -96,7 +100,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -310,7 +314,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
-        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack/gro
         gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
         gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
@@ -417,6 +421,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/util/naming                          from k8s.io/apimachinery/pkg/runtime+
         k8s.io/apimachinery/pkg/util/net                             from k8s.io/apimachinery/pkg/watch+
         k8s.io/apimachinery/pkg/util/rand                            from k8s.io/apiserver/pkg/storage/names
+        k8s.io/apimachinery/pkg/util/remotecommand                   from tailscale.com/k8s-operator/sessionrecording/ws
         k8s.io/apimachinery/pkg/util/runtime                         from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
         k8s.io/apimachinery/pkg/util/sets                            from k8s.io/apimachinery/pkg/api/meta+
         k8s.io/apimachinery/pkg/util/strategicpatch                  from k8s.io/client-go/tools/record+
@@ -594,10 +599,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
         k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
         k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
         sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
         sigs.k8s.io/controller-runtime/pkg/cache/internal            from sigs.k8s.io/controller-runtime/pkg/cache
@@ -686,9 +687,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/k8s-operator/apis                              from tailscale.com/k8s-operator/apis/v1alpha1
         tailscale.com/k8s-operator/apis/v1alpha1                     from tailscale.com/cmd/k8s-operator+
         tailscale.com/k8s-operator/sessionrecording                  from tailscale.com/cmd/k8s-operator
-        tailscale.com/k8s-operator/sessionrecording/conn             from tailscale.com/k8s-operator/sessionrecording/spdy
         tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
         tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
+        tailscale.com/k8s-operator/sessionrecording/ws               from tailscale.com/k8s-operator/sessionrecording
         tailscale.com/kube                                           from tailscale.com/cmd/k8s-operator+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
@@ -741,13 +742,13 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
-        tailscale.com/sessionrecording                               from tailscale.com/cmd/k8s-operator+
+        tailscale.com/sessionrecording                               from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
         tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
@@ -804,6 +805,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
@@ -825,6 +828,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/tsnet
+        tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -843,7 +847,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
@@ -861,6 +865,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
+        golang.org/x/net/websocket                                   from tailscale.com/k8s-operator/sessionrecording/ws
         golang.org/x/oauth2                                          from golang.org/x/oauth2/clientcredentials+
         golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
         golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -77,6 +77,10 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            {{- if .Values.proxyConfig.defaultProxyClass }}
+            - name: PROXY_DEFAULT_CLASS
+              value: {{ .Values.proxyConfig.defaultProxyClass }}
+            {{- end }}
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -14,10 +14,10 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["events", "services", "services/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingresses", "ingresses/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
@@ -49,10 +49,10 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
   verbs: ["get", "list", "watch"]

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -15,7 +15,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -78,6 +78,9 @@ proxyConfig:
   # Note that if you pass multiple tags to this field via `--set` flag to helm upgrade/install commands you must escape the comma (for example, "tag:k8s-proxies\,tag:prod"). See https://github.com/helm/helm/issues/1556
   defaultTags: "tag:k8s"
   firewallMode: auto
+  # If defined, this proxy class will be used as the default proxy class for
+  # service and ingress resources that do not have a proxy class defined.
+  defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2428,14 +2428,28 @@ rules:
         - services
         - services/status
       verbs:
-        - '*'
+        - create
+        - delete
+        - deletecollection
+        - get
+        - list
+        - patch
+        - update
+        - watch
     - apiGroups:
         - networking.k8s.io
       resources:
         - ingresses
         - ingresses/status
       verbs:
-        - '*'
+        - create
+        - delete
+        - deletecollection
+        - get
+        - list
+        - patch
+        - update
+        - watch
     - apiGroups:
         - networking.k8s.io
       resources:
@@ -2493,14 +2507,28 @@ rules:
         - serviceaccounts
         - configmaps
       verbs:
-        - '*'
+        - create
+        - delete
+        - deletecollection
+        - get
+        - list
+        - patch
+        - update
+        - watch
     - apiGroups:
         - apps
       resources:
         - statefulsets
         - deployments
       verbs:
-        - '*'
+        - create
+        - delete
+        - deletecollection
+        - get
+        - list
+        - patch
+        - update
+        - watch
     - apiGroups:
         - discovery.k8s.io
       resources:
@@ -2521,7 +2549,14 @@ rules:
       resources:
         - secrets
       verbs:
-        - '*'
+        - create
+        - delete
+        - deletecollection
+        - get
+        - list
+        - patch
+        - update
+        - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/dnsrecords.go

@@ -24,6 +24,7 @@ import (
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
 )
 
 const (
@@ -167,36 +168,49 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 		}
 	}
 
-	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
-	// headless Service.
+	// Get the Pod IP addresses for the proxy from the EndpointSlices for
+	// the headless Service. The Service can have multiple EndpointSlices
+	// associated with it, for example in dual-stack clusters.
 	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
-	if err != nil {
-		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
+	var eps = new(discoveryv1.EndpointSliceList)
+	if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error listing EndpointSlices for the proxy's headless Service: %w", err)
 	}
-	if eps == nil {
+	if len(eps.Items) == 0 {
 		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
 		return nil
 	}
-	// An EndpointSlice for a Service can have a list of endpoints that each
+	// Each EndpointSlice for a Service can have a list of endpoints that each
 	// can have multiple addresses - these are the IP addresses of any Pods
 	// selected by that Service. Pick all the IPv4 addresses.
-	ips := make([]string, 0)
-	for _, ep := range eps.Endpoints {
-		for _, ip := range ep.Addresses {
-			if !net.IsIPv4String(ip) {
-				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
-			} else {
-				ips = append(ips, ip)
+	// It is also possible that multiple EndpointSlices have overlapping addresses.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints
+	ips := make(set.Set[string], 0)
+	for _, slice := range eps.Items {
+		if slice.AddressType != discoveryv1.AddressTypeIPv4 {
+			logger.Infof("EndpointSlice is for AddressType %s, currently only IPv4 address type is supported", slice.AddressType)
+			continue
+		}
+		for _, ep := range slice.Endpoints {
+			if !epIsReady(&ep) {
+				logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String())
+				continue
+			}
+			for _, ip := range ep.Addresses {
+				if !net.IsIPv4String(ip) {
+					logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+				} else {
+					ips.Add(ip)
+				}
 			}
 		}
 	}
-	if len(ips) == 0 {
+	if ips.Len() == 0 {
 		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
 		return nil
 	}
 	updateFunc := func(rec *operatorutils.Records) {
-		mak.Set(&rec.IP4, fqdn, ips)
+		mak.Set(&rec.IP4, fqdn, ips.Slice())
 	}
 	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
 		return fmt.Errorf("error updating DNS records: %w", err)
@@ -204,6 +218,17 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 	return nil
 }
 
+// epIsReady reports whether the endpoint is currently in a state to receive new
+// traffic. As per kube docs, only explicitly set 'false' for 'Ready' or
+// 'Serving' conditions or explicitly set 'true' for 'Terminating' condition
+// means that the Endpoint is NOT ready.
+// https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/apis/discovery/types.go#L109-L131
+func epIsReady(ep *discoveryv1.Endpoint) bool {
+	return (ep.Conditions.Ready == nil || *ep.Conditions.Ready) &&
+		(ep.Conditions.Serving == nil || *ep.Conditions.Serving) &&
+		(ep.Conditions.Terminating == nil || !*ep.Conditions.Terminating)
+}
+
 // maybeCleanup ensures that the DNS record for the proxy has been removed from
 // dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
 // has been removed from the Service. If the record is not found in the

cmd/k8s-operator/dnsrecords_test.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -87,13 +88,16 @@ func TestDNSRecordsReconciler(t *testing.T) {
 		},
 	}
 	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
-	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4)
+	epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6)
+
 	mustCreate(t, fc, egressSvcFQDN)
 	mustCreate(t, fc, headlessForEgressSvcFQDN)
 	mustCreate(t, fc, ep)
+	mustCreate(t, fc, epv6)
 	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
 	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
-	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} // IPv6 endpoint is currently ignored
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
@@ -106,7 +110,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 3. DNS record is updated if the IP address of the proxy Pod changes.
-	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4)
 	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
 		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
 	})
@@ -116,7 +120,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 
 	// 4. DNS record is created for an ingress proxy configured via Ingress
 	headlessForIngress := headlessSvcForParent(ing, "ingress")
-	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4)
 	mustCreate(t, fc, headlessForIngress)
 	mustCreate(t, fc, ep)
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
@@ -140,6 +144,17 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
 	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
 	expectHostsRecords(t, fc, wantHosts)
+
+	// 7. A not-ready Endpoint is removed from DNS config.
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Conditions.Ready = ptr.To(false)
+		ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{
+			Addresses: []string{"1.2.3.4"},
+		})
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
+	wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"}
+	expectHostsRecords(t, fc, wantHosts)
 }
 
 func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
@@ -162,15 +177,21 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 	}
 }
 
-func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
+func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.AddressType) *discoveryv1.EndpointSlice {
 	return &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      svc.Name,
+			Name:      fmt.Sprintf("%s-%s", svc.Name, string(fam)),
 			Namespace: svc.Namespace,
 			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
 		},
+		AddressType: fam,
 		Endpoints: []discoveryv1.Endpoint{{
 			Addresses: []string{ip},
+			Conditions: discoveryv1.EndpointConditions{
+				Ready:       ptr.To(true),
+				Serving:     ptr.To(true),
+				Terminating: ptr.To(false),
+			},
 		}},
 	}
 }

cmd/k8s-operator/ingress.go

@@ -46,6 +46,8 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
+
+	proxyDefaultClass string
 }
 
 var (
@@ -133,7 +135,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing)
+	proxyClass := proxyClassForObject(ing, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/operator.go

@@ -66,6 +66,7 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
@@ -106,6 +107,7 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
+		proxyDefaultClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -279,6 +281,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
+			proxyDefaultClass:     opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -301,6 +304,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
 			logger:   opts.log.Named("ingress-reconciler"),
+			proxyDefaultClass: opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -424,6 +428,10 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
+	// proxyDefaultClass is the name of the ProxyClass to use as the default
+	// class for proxies that do not have a ProxyClass set.
+	// this is defined by an operator env variable.
+	proxyDefaultClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each

cmd/k8s-operator/proxy.go

@@ -22,9 +22,8 @@ import (
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
-	kubesessionrecording "tailscale.com/k8s-operator/sessionrecording"
+	ksr "tailscale.com/k8s-operator/sessionrecording"
 	tskube "tailscale.com/kube"
-	"tailscale.com/sessionrecording"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -168,7 +167,8 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", ap.serveDefault)
-	mux.HandleFunc("/api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExec)
+	mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
+	mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
 
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
@@ -209,9 +209,19 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
 	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
-// serveExec serves 'kubectl exec' requests, optionally configuring the kubectl
-// exec sessions to be recorded.
-func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
+// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.SPDYProtocol)
+}
+
+// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.WSProtocol)
+}
+
+func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
 	who, err := ap.whoIs(r)
 	if err != nil {
 		ap.authError(w, err)
@@ -227,15 +237,17 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 		return
 	}
-	kubesessionrecording.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
+	ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
 	if !failOpen && len(addrs) == 0 {
 		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
 		ap.log.Error(msg)
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-	if r.Method != "POST" || r.Header.Get("Upgrade") != "SPDY/3.1" {
-		msg := "'kubectl exec' session recording is configured, but the request is not over SPDY. Session recording is currently only supported for SPDY based clients"
+
+	wantsHeader := upgradeHeaderForProto[proto]
+	if h := r.Header.Get("Upgrade"); h != wantsHeader {
+		msg := fmt.Sprintf("[unexpected] unable to verify that streaming protocol is %s, wants Upgrade header %q, got: %q", proto, wantsHeader, h)
 		if failOpen {
 			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
 			ap.log.Warn(msg)
@@ -247,9 +259,22 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-	spdyH := kubesessionrecording.New(ap.ts, r, who, w, r.PathValue("pod"), r.PathValue("namespace"), kubesessionrecording.SPDYProtocol, addrs, failOpen, sessionrecording.ConnectToRecorder, ap.log)
 
-	ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	opts := ksr.HijackerOpts{
+		Req:       r,
+		W:         w,
+		Proto:     proto,
+		TS:        ap.ts,
+		Who:       who,
+		Addrs:     addrs,
+		FailOpen:  failOpen,
+		Pod:       r.PathValue("pod"),
+		Namespace: r.PathValue("namespace"),
+		Log:       ap.log,
+	}
+	h := ksr.New(opts)
+
+	ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
 func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
@@ -382,3 +407,8 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 	}
 	return failOpen, recorderAddresses, nil
 }
+
+var upgradeHeaderForProto = map[ksr.Protocol]string{
+	ksr.SPDYProtocol: "SPDY/3.1",
+	ksr.WSProtocol:   "websocket",
+}

cmd/k8s-operator/svc.go

@@ -62,6 +62,8 @@ type ServiceReconciler struct {
 	tsNamespace string
 
 	clock tstime.Clock
+
+	proxyDefaultClass string
 }
 
 var (
@@ -208,7 +210,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc)
+	proxyClass := proxyClassForObject(svc, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -404,8 +406,14 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-func proxyClassForObject(o client.Object) string {
-	return o.GetLabels()[LabelProxyClass]
+// proxyClassForObject returns the proxy class for the given object. If the
+// object does not have a proxy class label, it returns the default proxy class
+func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+	proxyClass, exists := o.GetLabels()[LabelProxyClass]
+	if !exists {
+		proxyClass = proxyDefaultClass
+	}
+	return proxyClass
 }
 
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {

cmd/stunstamp/stunstamp.go

@@ -1,13 +1,14 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-// The stunstamp binary measures STUN round-trip latency with DERPs.
+// The stunstamp binary measures round-trip latency with DERPs.
 package main
 
 import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -22,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"runtime"
 	"slices"
 	"strconv"
 	"strings"
@@ -31,8 +33,10 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/prompb"
+	"github.com/tcnksm/go-httpstat"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/stun"
+	"tailscale.com/net/tcpinfo"
 	"tailscale.com/tailcfg"
 )
 
@@ -42,7 +46,10 @@ var (
 	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
 	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
 	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
-	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+	flagSTUNDstPorts   = flag.String("stun-dst-ports", "", "comma-separated list of STUN destination ports to monitor")
+	flagHTTPSDstPorts  = flag.String("https-dst-ports", "", "comma-separated list of HTTPS destination ports to monitor")
+	flagTCPDstPorts    = flag.String("tcp-dst-ports", "", "comma-separated list of TCP destination ports to monitor")
+	flagICMP           = flag.Bool("icmp", false, "probe ICMP")
 )
 
 const (
@@ -89,12 +96,22 @@ func (t timestampSource) String() string {
 	}
 }
 
+type protocol string
+
+const (
+	protocolSTUN  protocol = "stun"
+	protocolICMP  protocol = "icmp"
+	protocolHTTPS protocol = "https"
+	protocolTCP   protocol = "tcp"
+)
+
 // resultKey contains the stable dimensions and their values for a given
 // timeseries, i.e. not time and not rtt/timeout.
 type resultKey struct {
 	meta            nodeMeta
 	timestampSource timestampSource
 	connStability   connStability
+	protocol        protocol
 	dstPort         int
 }
 
@@ -104,7 +121,203 @@ type result struct {
 	rtt *time.Duration // nil signifies failure, e.g. timeout
 }
 
-func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+type lportsPool struct {
+	sync.Mutex
+	ports []int
+}
+
+func (l *lportsPool) get() int {
+	l.Lock()
+	defer l.Unlock()
+	ret := l.ports[0]
+	l.ports = append(l.ports[:0], l.ports[1:]...)
+	return ret
+}
+
+func (l *lportsPool) put(i int) {
+	l.Lock()
+	defer l.Unlock()
+	l.ports = append(l.ports, int(i))
+}
+
+var (
+	lports *lportsPool
+)
+
+const (
+	lportPoolSize = 16000
+	lportBase     = 2048
+)
+
+func init() {
+	lports = &lportsPool{
+		ports: make([]int, 0, lportPoolSize),
+	}
+	for i := lportBase; i < lportBase+lportPoolSize; i++ {
+		lports.ports = append(lports.ports, i)
+	}
+}
+
+// lportForTCPConn satisfies io.ReadWriteCloser, but is really just used to pass
+// around a persistent laddr for stableConn purposes. The underlying TCP
+// connection is not created until measurement time as in some cases we need to
+// measure dial time.
+type lportForTCPConn int
+
+func (l *lportForTCPConn) Close() error {
+	if *l == 0 {
+		return nil
+	}
+	lports.put(int(*l))
+	return nil
+}
+
+func (l *lportForTCPConn) Write([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func (l *lportForTCPConn) Read([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func addrInUse(err error, lport *lportForTCPConn) bool {
+	if errors.Is(err, syscall.EADDRINUSE) {
+		old := int(*lport)
+		// abandon port, don't return it to pool
+		*lport = lportForTCPConn(lports.get()) // get a new port
+		log.Printf("EADDRINUSE: %v old: %d new: %d", err, old, *lport)
+		return true
+	}
+	return false
+}
+
+func tcpDial(ctx context.Context, lport *lportForTCPConn, dst netip.AddrPort) (net.Conn, error) {
+	for {
+		var opErr error
+		dialer := &net.Dialer{
+			LocalAddr: &net.TCPAddr{
+				Port: int(*lport),
+			},
+			Control: func(network, address string, c syscall.RawConn) error {
+				return c.Control(func(fd uintptr) {
+					// we may restart faster than TIME_WAIT can clear
+					opErr = setSOReuseAddr(fd)
+				})
+			},
+		}
+		if opErr != nil {
+			panic(opErr)
+		}
+		tcpConn, err := dialer.DialContext(ctx, "tcp", dst.String())
+		if err != nil {
+			if addrInUse(err, lport) {
+				continue
+			}
+			return nil, err
+		}
+		return tcpConn, nil
+	}
+}
+
+type tempError struct {
+	error
+}
+
+func (t tempError) Temporary() bool {
+	return true
+}
+
+func measureTCPRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	// Set a dial timeout < 1s (TCP_TIMEOUT_INIT on Linux) as a means to avoid
+	// SYN retries, which can contribute to tcpi->rtt below. This simply limits
+	// retries from the initiator, but SYN+ACK on the reverse path can also
+	// time out and be retransmitted.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*750)
+	defer cancel()
+	tcpConn, err := tcpDial(ctx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	// This is an unreliable method to measure TCP RTT. The Linux kernel
+	// describes it as such in tcp_rtt_estimator(). We take some care in how we
+	// hold tcp_info->rtt here, e.g. clamping dial timeout, but if we are to
+	// actually use this elsewhere as an input to some decision it warrants a
+	// deeper study and consideration for alternative methods. Its usefulness
+	// here is as a point of comparison against the other methods.
+	rtt, err = tcpinfo.RTT(tcpConn)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	return rtt, nil
+}
+
+func measureHTTPSRTT(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	var httpResult httpstat.Result
+	// 5s mirrors net/netcheck.overallProbeTimeout used in net/netcheck.Client.measureHTTPSLatency.
+	reqCtx, cancel := context.WithTimeout(httpstat.WithHTTPStat(context.Background(), &httpResult), time.Second*5)
+	defer cancel()
+	reqURL := "https://" + dst.String() + "/derp/latency-check"
+	req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil)
+	if err != nil {
+		return 0, err
+	}
+	client := &http.Client{}
+	// 1.5s mirrors derp/derphttp.dialnodeTimeout used in derp/derphttp.DialNode().
+	dialCtx, dialCancel := context.WithTimeout(reqCtx, time.Millisecond*1500)
+	defer dialCancel()
+	tcpConn, err := tcpDial(dialCtx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	tlsConn := tls.Client(tcpConn, &tls.Config{
+		ServerName: hostname,
+	})
+	// Mirror client/netcheck behavior, which handshakes before handing the
+	// tlsConn over to the http.Client via http.Transport
+	err = tlsConn.Handshake()
+	if err != nil {
+		return 0, tempError{err}
+	}
+	tlsConnCh := make(chan net.Conn, 1)
+	tlsConnCh <- tlsConn
+	tr := &http.Transport{
+		DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
+			select {
+			case tlsConn := <-tlsConnCh:
+				return tlsConn, nil
+			default:
+				return nil, errors.New("unexpected second call of DialTLSContext")
+			}
+		},
+	}
+	client.Transport = tr
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	if resp.StatusCode/100 != 2 {
+		return 0, tempError{fmt.Errorf("unexpected status code: %d", resp.StatusCode)}
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(io.Discard, io.LimitReader(resp.Body, 8<<10))
+	if err != nil {
+		return 0, tempError{err}
+	}
+	httpResult.End(time.Now())
+	return httpResult.ServerProcessing, nil
+}
+
+func measureSTUNRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	uconn, ok := conn.(*net.UDPConn)
 	if !ok {
 		return 0, fmt.Errorf("unexpected conn type: %T", conn)
@@ -116,7 +329,10 @@ func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, e
 	txID := stun.NewTxID()
 	req := stun.Request(txID)
 	txAt := time.Now()
-	_, err = uconn.WriteToUDP(req, dst)
+	_, err = uconn.WriteToUDP(req, &net.UDPAddr{
+		IP:   dst.Addr().AsSlice(),
+		Port: int(dst.Port()),
+	})
 	if err != nil {
 		return 0, fmt.Errorf("error writing to udp socket: %w", err)
 	}
@@ -153,20 +369,19 @@ type nodeMeta struct {
 	addr       netip.Addr
 }
 
-type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
+type measureFn func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error)
 
-// probe measures STUN round trip time for the node described by meta over
-// conn against dstPort. It may return a nil duration and nil error if the
-// STUN request timed out. A non-nil error indicates an unrecoverable or
-// non-temporary error.
-func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
+// probe measures round trip time for the node described by meta over cf against
+// dstPort. It may return a nil duration and nil error in the event of a
+// timeout. A non-nil error indicates an unrecoverable or non-temporary error.
+func probe(meta nodeMeta, cf *connAndMeasureFn, dstPort int) (*time.Duration, error) {
 	ua := &net.UDPAddr{
 		IP:   net.IP(meta.addr.AsSlice()),
 		Port: dstPort,
 	}
 
 	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
-	rtt, err := fn(conn, ua)
+	rtt, err := cf.fn(cf.conn, meta.hostname, netip.AddrPortFrom(meta.addr, uint16(dstPort)))
 	if err != nil {
 		if isTemporaryOrTimeoutErr(err) {
 			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
@@ -237,43 +452,138 @@ func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]node
 	return stale, nil
 }
 
-func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
-	conns := [2]io.ReadWriteCloser{}
-	byDstPort, ok := stableConns[addr]
-	if ok {
-		conns, ok = byDstPort[dstPort]
-		if ok {
-			return conns, nil
-		}
-	}
-	if supportsKernelTS() {
-		kconn, err := getConnKernelTimestamp()
-		if err != nil {
-			return conns, err
-		}
-		conns[timestampSourceKernel] = kconn
-	}
-	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
-	if err != nil {
-		if supportsKernelTS() {
-			conns[timestampSourceKernel].Close()
-		}
-		return conns, err
-	}
-	conns[timestampSourceUserspace] = uconn
-	if byDstPort == nil {
-		byDstPort = make(map[int][2]io.ReadWriteCloser)
-	}
-	byDstPort[dstPort] = conns
-	stableConns[addr] = byDstPort
-	return conns, nil
+type connAndMeasureFn struct {
+	conn io.ReadWriteCloser
+	fn   measureFn
 }
 
-// probeNodes measures the round-trip time for STUN binding requests against the
-// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
-// UDP sockets that should be recycled across runs. It returns the results or
-// an error if one occurs.
-func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+// newConnAndMeasureFn returns a connAndMeasureFn or an error. It may return
+// nil for both if some combination of the supplied timestampSource, protocol,
+// or connStability is unsupported.
+func newConnAndMeasureFn(source timestampSource, protocol protocol, stable connStability) (*connAndMeasureFn, error) {
+	info := getProtocolSupportInfo(protocol)
+	if !info.stableConn && bool(stable) {
+		return nil, nil
+	}
+	if !info.userspaceTS && source == timestampSourceUserspace {
+		return nil, nil
+	}
+	if !info.kernelTS && source == timestampSourceKernel {
+		return nil, nil
+	}
+	switch protocol {
+	case protocolSTUN:
+		if source == timestampSourceKernel {
+			conn, err := getUDPConnKernelTimestamp()
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTTKernel,
+			}, nil
+		} else {
+			conn, err := net.ListenUDP("udp", &net.UDPAddr{})
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTT,
+			}, nil
+		}
+	case protocolICMP:
+		// TODO(jwhited): implement
+		return nil, nil
+	case protocolHTTPS:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureHTTPSRTT,
+		}, nil
+	case protocolTCP:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureTCPRTT,
+		}, nil
+	}
+	return nil, errors.New("unknown protocol")
+}
+
+type stableConnKey struct {
+	node     netip.Addr
+	protocol protocol
+	port     int
+}
+
+type protocolSupportInfo struct {
+	kernelTS    bool
+	userspaceTS bool
+	stableConn  bool
+}
+
+func getConns(
+	stableConns map[stableConnKey][2]*connAndMeasureFn,
+	addr netip.Addr,
+	protocol protocol,
+	dstPort int,
+) (stable, unstable [2]*connAndMeasureFn, err error) {
+	key := stableConnKey{addr, protocol, dstPort}
+	defer func() {
+		if err != nil {
+			for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+				c := stable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+				c = unstable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+			}
+		}
+	}()
+
+	var ok bool
+	stable, ok = stableConns[key]
+	if !ok {
+		for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+			var cf *connAndMeasureFn
+			cf, err = newConnAndMeasureFn(source, protocol, stableConn)
+			if err != nil {
+				return
+			}
+			stable[source] = cf
+		}
+		stableConns[key] = stable
+	}
+
+	for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+		var cf *connAndMeasureFn
+		cf, err = newConnAndMeasureFn(source, protocol, unstableConn)
+		if err != nil {
+			return
+		}
+		unstable[source] = cf
+	}
+	return stable, unstable, nil
+}
+
+// probeNodes measures the round-trip time for the protocols and ports described
+// by portsByProtocol against the DERP nodes described by nodeMetaByAddr.
+// stableConns are used to recycle connections across calls to probeNodes.
+// probeNodes is also responsible for trimming stableConns based on node
+// lifetime in nodeMetaByAddr. It returns the results or an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[stableConnKey][2]*connAndMeasureFn, portsByProtocol map[protocol][]int) ([]result, error) {
 	wg := sync.WaitGroup{}
 	results := make([]result, 0)
 	resultsCh := make(chan result)
@@ -283,40 +593,19 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 	at := time.Now()
 	addrsToProbe := make(map[netip.Addr]bool)
 
-	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+	doProbe := func(cf *connAndMeasureFn, meta nodeMeta, source timestampSource, stable connStability, protocol protocol, dstPort int) {
 		defer wg.Done()
 		r := result{
 			key: resultKey{
 				meta:            meta,
 				timestampSource: source,
+				connStability:   stable,
 				dstPort:         dstPort,
+				protocol:        protocol,
 			},
 			at: at,
 		}
-		if conn == nil {
-			var err error
-			if source == timestampSourceKernel {
-				conn, err = getConnKernelTimestamp()
-			} else {
-				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
-			}
-			if err != nil {
-				select {
-				case <-doneCh:
-					return
-				case errCh <- err:
-					return
-				}
-			}
-			defer conn.Close()
-		} else {
-			r.key.connStability = stableConn
-		}
-		fn := measureRTT
-		if source == timestampSourceKernel {
-			fn = measureRTTKernel
-		}
-		rtt, err := probe(meta, conn, fn, dstPort)
+		rtt, err := probe(meta, cf, dstPort)
 		if err != nil {
 			select {
 			case <-doneCh:
@@ -334,37 +623,42 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 
 	for _, meta := range nodeMetaByAddr {
 		addrsToProbe[meta.addr] = true
-		for _, port := range dstPorts {
-			stable, err := getStableConns(stableConns, meta.addr, port)
-			if err != nil {
-				close(doneCh)
-				wg.Wait()
-				return nil, err
-			}
+		for p, ports := range portsByProtocol {
+			for _, port := range ports {
+				stable, unstable, err := getConns(stableConns, meta.addr, p, port)
+				if err != nil {
+					close(doneCh)
+					wg.Wait()
+					return nil, err
+				}
 
-			wg.Add(2)
-			numProbes += 2
-			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
-			go doProbe(nil, meta, timestampSourceUserspace, port)
-			if supportsKernelTS() {
-				wg.Add(2)
-				numProbes += 2
-				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
-				go doProbe(nil, meta, timestampSourceKernel, port)
+				for i, cf := range stable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), stableConn, p, port)
+					}
+				}
+
+				for i, cf := range unstable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), unstableConn, p, port)
+					}
+				}
 			}
 		}
 	}
 
 	// cleanup conns we no longer need
-	for k, byDstPort := range stableConns {
-		if !addrsToProbe[k] {
-			for _, conns := range byDstPort {
-				if conns[timestampSourceKernel] != nil {
-					conns[timestampSourceKernel].Close()
-				}
-				conns[timestampSourceUserspace].Close()
-				delete(stableConns, k)
+	for k, cf := range stableConns {
+		if !addrsToProbe[k.node] {
+			if cf[timestampSourceKernel] != nil {
+				cf[timestampSourceKernel].conn.Close()
 			}
+			cf[timestampSourceUserspace].conn.Close()
+			delete(stableConns, k)
 		}
 	}
 
@@ -391,11 +685,11 @@ const (
 )
 
 const (
-	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
-	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+	rttMetricName      = "stunstamp_derp_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_timeouts_total"
 )
 
-func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, protocol protocol, dstPort int) []prompb.Label {
 	addressFamily := "ipv4"
 	if meta.addr.Is6() {
 		addressFamily = "ipv6"
@@ -425,6 +719,10 @@ func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source
 		Name:  "hostname",
 		Value: meta.hostname,
 	})
+	labels = append(labels, prompb.Label{
+		Name:  "protocol",
+		Value: string(protocol),
+	})
 	labels = append(labels, prompb.Label{
 		Name:  "dst_port",
 		Value: strconv.Itoa(dstPort),
@@ -453,53 +751,35 @@ const (
 	staleNaN uint64 = 0x7ff0000000000002
 )
 
-func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, portsByProtocol map[protocol][]int) []prompb.TimeSeries {
 	staleMarkers := make([]prompb.TimeSeries, 0)
 	now := time.Now()
-	for _, s := range stale {
-		for _, dstPort := range dstPorts {
-			samples := []prompb.Sample{
-				{
-					Timestamp: now.UnixMilli(),
-					Value:     math.Float64frombits(staleNaN),
-				},
-			}
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			if supportsKernelTS() {
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
+
+	for p, ports := range portsByProtocol {
+		for _, port := range ports {
+			for _, s := range stale {
+				samples := []prompb.Sample{
+					{
+						Timestamp: now.UnixMilli(),
+						Value:     math.Float64frombits(staleNaN),
+					},
+				}
+				// We send stale markers for all combinations in the interest
+				// of simplicity.
+				for _, name := range []string{rttMetricName, timeoutsMetricName} {
+					for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+						for _, stable := range []connStability{unstableConn, stableConn} {
+							staleMarkers = append(staleMarkers, prompb.TimeSeries{
+								Labels:  timeSeriesLabels(name, s, instance, source, stable, p, port),
+								Samples: samples,
+							})
+						}
+					}
+				}
 			}
 		}
 	}
+
 	return staleMarkers
 }
 
@@ -513,7 +793,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 	for _, r := range results {
 		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
 		seenKeys[r.key] = true
-		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		rttSamples := make([]prompb.Sample, 1)
 		rttSamples[0].Timestamp = r.at.UnixMilli()
 		if r.rtt != nil {
@@ -528,7 +808,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 		}
 		all = append(all, rttTS)
 		timeouts[r.key] = timeoutsCount
-		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		timeoutsSamples := make([]prompb.Sample, 1)
 		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
 		timeoutsSamples[0].Value = float64(timeoutsCount)
@@ -620,22 +900,66 @@ func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSer
 	}
 }
 
-func main() {
-	flag.Parse()
-	if len(*flagDstPorts) == 0 {
-		log.Fatal("dst-ports flag is unset")
+func getPortsFromFlag(f string) ([]int, error) {
+	if len(f) == 0 {
+		return nil, nil
 	}
-	dstPortsSplit := strings.Split(*flagDstPorts, ",")
-	slices.Sort(dstPortsSplit)
-	dstPortsSplit = slices.Compact(dstPortsSplit)
-	dstPorts := make([]int, 0, len(dstPortsSplit))
-	for _, d := range dstPortsSplit {
-		i, err := strconv.ParseUint(d, 10, 16)
+	split := strings.Split(f, ",")
+	slices.Sort(split)
+	split = slices.Compact(split)
+	ports := make([]int, 0)
+	for _, portStr := range split {
+		port, err := strconv.ParseUint(portStr, 10, 16)
 		if err != nil {
-			log.Fatal("invalid dst-ports")
+			return nil, err
 		}
-		dstPorts = append(dstPorts, int(i))
+		ports = append(ports, int(port))
 	}
+	return ports, nil
+}
+
+func main() {
+	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
+		log.Fatal("unsupported platform")
+	}
+	flag.Parse()
+
+	portsByProtocol := make(map[protocol][]int)
+	stunPorts, err := getPortsFromFlag(*flagSTUNDstPorts)
+	if err != nil {
+		log.Fatalf("invalid stun-dst-ports flag value: %v", err)
+	}
+	if len(stunPorts) > 0 {
+		portsByProtocol[protocolSTUN] = stunPorts
+	}
+	httpsPorts, err := getPortsFromFlag(*flagHTTPSDstPorts)
+	if err != nil {
+		log.Fatalf("invalid https-dst-ports flag value: %v", err)
+	}
+	if len(httpsPorts) > 0 {
+		portsByProtocol[protocolHTTPS] = httpsPorts
+	}
+	tcpPorts, err := getPortsFromFlag(*flagTCPDstPorts)
+	if err != nil {
+		log.Fatalf("invalid tcp-dst-ports flag value: %v", err)
+	}
+	if len(tcpPorts) > 0 {
+		portsByProtocol[protocolTCP] = tcpPorts
+	}
+	if *flagICMP {
+		portsByProtocol[protocolICMP] = []int{0}
+	}
+	if len(portsByProtocol) == 0 {
+		log.Fatal("nothing to probe")
+	}
+
+	// TODO(jwhited): remove protocol restriction
+	for k := range portsByProtocol {
+		if k != protocolSTUN && k != protocolHTTPS && k != protocolTCP {
+			log.Fatal("ICMP is not yet supported")
+		}
+	}
+
 	if len(*flagDERPMap) < 1 {
 		log.Fatal("derp-map flag is unset")
 	}
@@ -645,7 +969,7 @@ func main() {
 	if len(*flagRemoteWriteURL) < 1 {
 		log.Fatal("rw-url flag is unset")
 	}
-	_, err := url.Parse(*flagRemoteWriteURL)
+	_, err = url.Parse(*flagRemoteWriteURL)
 	if err != nil {
 		log.Fatalf("invalid rw-url flag value: %v", err)
 	}
@@ -707,7 +1031,7 @@ func main() {
 		for _, v := range nodeMetaByAddr {
 			staleMeta = append(staleMeta, v)
 		}
-		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 		if len(staleMarkers) > 0 {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 			rwc.write(ctx, staleMarkers)
@@ -723,8 +1047,8 @@ func main() {
 	// in a higher probability of the packets traversing the same underlay path.
 	// Comparison of stable and unstable 5-tuple results can shed light on
 	// differences between paths where hashing (multipathing/load balancing)
-	// comes into play.
-	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+	// comes into play. The inner 2 element array index is timestampSource.
+	stableConns := make(map[stableConnKey][2]*connAndMeasureFn)
 
 	// timeouts holds counts of timeout events. Values are persisted for the
 	// lifetime of the related node in the DERP map.
@@ -738,7 +1062,7 @@ func main() {
 	for {
 		select {
 		case <-probeTicker.C:
-			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			results, err := probeNodes(nodeMetaByAddr, stableConns, portsByProtocol)
 			if err != nil {
 				log.Printf("unrecoverable error while probing: %v", err)
 				shutdown()
@@ -761,7 +1085,7 @@ func main() {
 				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
 				continue
 			}
-			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 			if len(staleMarkers) < 1 {
 				continue
 			}
@@ -780,7 +1104,7 @@ func main() {
 				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 				defer cancel()
 				updatedDM, err := getDERPMap(ctx, *flagDERPMap)
-				if err != nil {
+				if err == nil {
 					dmCh <- updatedDM
 				}
 			}()

cmd/stunstamp/stunstamp_default.go

@@ -8,18 +8,42 @@ package main
 import (
 	"errors"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unimplemented")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	return 0, errors.New("unimplemented")
 }
 
-func supportsKernelTS() bool {
-	return false
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	return nil
 }

cmd/stunstamp/stunstamp_linux.go

@@ -10,7 +10,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
+	"net/netip"
+	"syscall"
 	"time"
 
 	"github.com/mdlayher/socket"
@@ -24,7 +25,7 @@ const (
 		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
 	if err != nil {
 		return nil, err
@@ -56,24 +57,23 @@ func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
 	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	sconn, ok := conn.(*socket.Conn)
 	if !ok {
 		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
 	}
 
 	var to unix.Sockaddr
-	to4 := dst.IP.To4()
-	if to4 != nil {
+	if dst.Addr().Is4() {
 		to = &unix.SockaddrInet4{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
 	} else {
 		to = &unix.SockaddrInet6{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
 	}
 
 	txID := stun.NewTxID()
@@ -138,6 +138,32 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 
 }
 
-func supportsKernelTS() bool {
-	return true
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+		// TODO(jwhited): add ICMP
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	// we may restart faster than TIME_WAIT can clear
+	return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
 }

cmd/tailscale/cli/cli.go

@@ -7,7 +7,6 @@ package cli
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -85,6 +84,13 @@ var localClient = tailscale.LocalClient{
 
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) (err error) {
+	if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 {
+		// We're running on gokrazy and it's the first start.
+		// Don't run the tailscale CLI as a service; just exit.
+		// See https://gokrazy.org/development/process-interface/
+		os.Exit(0)
+	}
+
 	args = CleanUpArgs(args)
 
 	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
@@ -160,10 +166,8 @@ func newRootCmd() *ffcli.Command {
 		return nil
 	})
 	rootfs.Lookup("socket").DefValue = localClient.Socket
-	jsonDocs := rootfs.Bool("json-docs", false, hidden+"print JSON-encoded docs for all subcommands and flags")
 
-	var rootCmd *ffcli.Command
-	rootCmd = &ffcli.Command{
+	rootCmd := &ffcli.Command{
 		Name:       "tailscale",
 		ShortUsage: "tailscale [flags] <subcommand> [command flags]",
 		ShortHelp:  "The easiest, most secure way to use WireGuard.",
@@ -205,9 +209,6 @@ change in the future.
 		},
 		FlagSet: rootfs,
 		Exec: func(ctx context.Context, args []string) error {
-			if *jsonDocs {
-				return printJSONDocs(rootCmd)
-			}
 			if len(args) > 0 {
 				return fmt.Errorf("tailscale: unknown subcommand: %s", args[0])
 			}
@@ -407,54 +408,3 @@ func colorableOutput() (w io.Writer, ok bool) {
 	}
 	return colorable.NewColorableStdout(), true
 }
-
-type commandDoc struct {
-	Name        string
-	Desc        string
-	Subcommands []commandDoc `json:",omitempty"`
-	Flags       []flagDoc    `json:",omitempty"`
-}
-
-type flagDoc struct {
-	Name string
-	Desc string
-}
-
-func printJSONDocs(root *ffcli.Command) error {
-	docs := jsonDocsWalk(root)
-	return json.NewEncoder(os.Stdout).Encode(docs)
-}
-
-func jsonDocsWalk(cmd *ffcli.Command) *commandDoc {
-	res := &commandDoc{
-		Name: cmd.Name,
-	}
-	if cmd.LongHelp != "" {
-		res.Desc = cmd.LongHelp
-	} else if cmd.ShortHelp != "" {
-		res.Desc = cmd.ShortHelp
-	} else {
-		res.Desc = cmd.ShortUsage
-	}
-	if strings.HasPrefix(res.Desc, hidden) {
-		return nil
-	}
-	if cmd.FlagSet != nil {
-		cmd.FlagSet.VisitAll(func(f *flag.Flag) {
-			if strings.HasPrefix(f.Usage, hidden) {
-				return
-			}
-			res.Flags = append(res.Flags, flagDoc{
-				Name: f.Name,
-				Desc: f.Usage,
-			})
-		})
-	}
-	for _, sub := range cmd.Subcommands {
-		subj := jsonDocsWalk(sub)
-		if subj != nil {
-			res.Subcommands = append(res.Subcommands, *subj)
-		}
-	}
-	return res
-}

cmd/tailscale/cli/network-lock.go

@@ -20,6 +20,7 @@ import (
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tka"
+	"tailscale.com/tsconst"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
 )
@@ -443,15 +444,33 @@ func runNetworkLockModify(ctx context.Context, addArgs, removeArgs []string) err
 
 var nlSignCmd = &ffcli.Command{
 	Name:       "sign",
-	ShortUsage: "tailscale lock sign <node-key> [<rotation-key>] or sign <auth-key>",
+	ShortUsage: "tailscale lock sign <node-key> [<rotation-key>]\ntailscale lock sign <auth-key>",
 	ShortHelp:  "Signs a node or pre-approved auth key",
 	LongHelp: `Either:
-  - signs a node key and transmits the signature to the coordination server, or
-  - signs a pre-approved auth key, printing it in a form that can be used to bring up nodes under tailnet lock`,
+  - signs a node key and transmits the signature to the coordination
+    server, or
+  - signs a pre-approved auth key, printing it in a form that can be
+    used to bring up nodes under tailnet lock
+
+If any of the key arguments begin with "file:", the key is retrieved from
+the file at the path specified in the argument suffix.`,
 	Exec: runNetworkLockSign,
 }
 
 func runNetworkLockSign(ctx context.Context, args []string) error {
+	// If any of the arguments start with "file:", replace that argument
+	// with the contents of the file. We do this early, before the check
+	// to see if the first argument is an auth key.
+	for i, arg := range args {
+		if filename, ok := strings.CutPrefix(arg, "file:"); ok {
+			b, err := os.ReadFile(filename)
+			if err != nil {
+				return err
+			}
+			args[i] = strings.TrimSpace(string(b))
+		}
+	}
+
 	if len(args) > 0 && strings.HasPrefix(args[0], "tskey-auth-") {
 		return runTskeyWrapCmd(ctx, args)
 	}
@@ -476,7 +495,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error {
 	err := localClient.NetworkLockSign(ctx, nodeKey, []byte(rotationKey.Verifier()))
 	// Provide a better help message for when someone clicks through the signing flow
 	// on the wrong device.
-	if err != nil && strings.Contains(err.Error(), "this node is not trusted by network lock") {
+	if err != nil && strings.Contains(err.Error(), tsconst.TailnetLockNotTrustedMsg) {
 		fmt.Fprintln(Stderr, "Error: Signing is not available on this device because it does not have a trusted tailnet lock key.")
 		fmt.Fprintln(Stderr)
 		fmt.Fprintln(Stderr, "Try again on a signing device instead. Tailnet admins can see signing devices on the admin panel.")

cmd/tailscale/depaware.txt

@@ -5,11 +5,15 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -66,10 +70,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         go4.org/netipx                                               from tailscale.com/net/tsaddr
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         k8s.io/client-go/util/homedir                                from tailscale.com/cmd/tailscale/cli
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         sigs.k8s.io/yaml                                             from tailscale.com/cmd/tailscale/cli
         sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml
         software.sslmate.com/src/go-pkcs12                           from tailscale.com/cmd/tailscale/cli
@@ -128,7 +128,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/tempfork/spf13/cobra                           from tailscale.com/cmd/tailscale/cli/ffcomplete+
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tstime                                         from tailscale.com/control/controlhttp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
@@ -152,9 +152,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dnscache+
         tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy+
         tailscale.com/util/ctxkey                                    from tailscale.com/types/logger
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/groupmember                               from tailscale.com/client/web
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -167,6 +169,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
@@ -191,7 +195,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
    W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
-        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli
+        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
@@ -308,7 +312,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
-        runtime/debug                                                from nhooyr.io/websocket/internal/xsync+
+        runtime/debug                                                from github.com/coder/websocket/internal/xsync+
         slices                                                       from tailscale.com/client/web+
         sort                                                         from archive/tar+
         strconv                                                      from archive/tar+

cmd/tailscaled/depaware.txt

@@ -79,6 +79,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -90,7 +94,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 github.com/djherbis/times                                    from tailscale.com/drive/driveimpl
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/tstun+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -221,7 +225,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
-        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack/gro
         gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
         gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
@@ -232,10 +236,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack           from gvisor.dev/gvisor/pkg/tcpip/stack
         gvisor.dev/gvisor/pkg/tcpip/transport/udp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
         gvisor.dev/gvisor/pkg/waiter                                 from gvisor.dev/gvisor/pkg/context+
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
@@ -338,7 +338,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/cmd/tailscaled+
         tailscale.com/tstime                                         from tailscale.com/control/controlclient+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
@@ -396,6 +396,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/cmd/tailscaled+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
@@ -418,6 +420,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
+        tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -436,7 +439,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
   LD    golang.org/x/crypto/ssh                                      from github.com/pkg/sftp+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+

cmd/tailscaled/tailscaled.go

@@ -35,6 +35,7 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive/driveimpl"
 	"tailscale.com/envknob"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/conffile"
 	"tailscale.com/ipn/ipnlocal"
@@ -154,9 +155,11 @@ var beCLI func() // non-nil if CLI is linked in
 func main() {
 	envknob.PanicIfAnyEnvCheckedInInit()
 	envknob.ApplyDiskConfig()
+	applyIntegrationTestEnvKnob()
 
+	defaultVerbosity := envknob.RegisterInt("TS_LOG_VERBOSITY")
 	printVersion := false
-	flag.IntVar(&args.verbose, "verbose", 0, "log verbosity level; 0 is default, 1 or higher are increasingly verbose")
+	flag.IntVar(&args.verbose, "verbose", defaultVerbosity(), "log verbosity level; 0 is default, 1 or higher are increasingly verbose")
 	flag.BoolVar(&args.cleanUp, "cleanup", false, "clean up system state and exit")
 	flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server")
 	flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`)
@@ -895,3 +898,24 @@ func dieOnPipeReadErrorOfFD(fd int) {
 	f.Read(make([]byte, 1))
 	os.Exit(1)
 }
+
+// applyIntegrationTestEnvKnob applies the tailscaled.env=... environment
+// variables specified on the Linux kernel command line, if the VM is being
+// run in NATLab integration tests.
+//
+// They're specified as: tailscaled.env=FOO=bar tailscaled.env=BAR=baz
+func applyIntegrationTestEnvKnob() {
+	if runtime.GOOS != "linux" || !hostinfo.IsNATLabGuestVM() {
+		return
+	}
+	cmdLine, _ := os.ReadFile("/proc/cmdline")
+	for _, s := range strings.Fields(string(cmdLine)) {
+		suf, ok := strings.CutPrefix(s, "tailscaled.env=")
+		if !ok {
+			continue
+		}
+		if k, v, ok := strings.Cut(suf, "="); ok {
+			envknob.Setenv(k, v)
+		}
+	}
+}

cmd/tl-longchain/tl-longchain.go

@@ -0,0 +1,93 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Program tl-longchain prints commands to re-sign Tailscale nodes that have
+// long rotation signature chains.
+//
+// There is an implicit limit on the number of rotation signatures that can
+// be chained before the signature becomes too long. This program helps
+// tailnet admins to identify nodes that have signatures with long chains and
+// prints commands to re-sign those node keys with a fresh direct signature.
+// Commands are printed to stdout, while log messages are printed to stderr.
+//
+// Note that the Tailscale client this command is executed on must have
+// ACL visibility to all other nodes to be able to see their signatures.
+// https://tailscale.com/kb/1087/device-visibility
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tka"
+	"tailscale.com/types/key"
+)
+
+var (
+	flagSocket   = flag.String("socket", "", "custom path to tailscaled socket")
+	maxRotations = flag.Int("rotations", 10, "number of rotation signatures before re-signing (max 16)")
+	showFiltered = flag.Bool("show-filtered", false, "include nodes with invalid signatures")
+)
+
+func main() {
+	flag.Parse()
+
+	lc := tailscale.LocalClient{Socket: *flagSocket}
+	if lc.Socket != "" {
+		lc.UseSocketOnly = true
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	st, err := lc.NetworkLockStatus(ctx)
+	if err != nil {
+		log.Fatalf("could not get Tailnet Lock status: %v", err)
+	}
+	if !st.Enabled {
+		log.Print("Tailnet Lock is not enabled")
+		return
+	}
+	print("Self", *st.NodeKey, *st.NodeKeySignature)
+	if len(st.VisiblePeers) > 0 {
+		log.Print("Visible peers with valid signatures:")
+		for _, peer := range st.VisiblePeers {
+			print(peerInfo(peer), peer.NodeKey, peer.NodeKeySignature)
+		}
+	}
+	if *showFiltered && len(st.FilteredPeers) > 0 {
+		log.Print("Visible peers with invalid signatures:")
+		for _, peer := range st.FilteredPeers {
+			print(peerInfo(peer), peer.NodeKey, peer.NodeKeySignature)
+		}
+	}
+}
+
+// peerInfo returns a string with information about a peer.
+func peerInfo(peer *ipnstate.TKAPeer) string {
+	return fmt.Sprintf("Peer %s (%s) nodeid=%s, current signature kind=%v", peer.Name, peer.TailscaleIPs[0], peer.StableID, peer.NodeKeySignature.SigKind)
+}
+
+// print prints a message about a node key signature and a re-signing command if needed.
+func print(info string, nodeKey key.NodePublic, sig tka.NodeKeySignature) {
+	if l := chainLength(sig); l > *maxRotations {
+		log.Printf("%s: chain length %d, printing command to re-sign", info, l)
+		wrapping, _ := sig.UnverifiedWrappingPublic()
+		fmt.Printf("tailscale lock sign %s %s\n", nodeKey, key.NLPublicFromEd25519Unsafe(wrapping).CLIString())
+	} else {
+		log.Printf("%s: does not need re-signing", info)
+	}
+}
+
+// chainLength returns the length of the rotation signature chain.
+func chainLength(sig tka.NodeKeySignature) int {
+	if sig.SigKind != tka.SigRotation {
+		return 1
+	}
+	return 1 + chainLength(*sig.Nested)
+}

cmd/tsidp/tsidp.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/rsa"
@@ -16,6 +17,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -25,6 +27,7 @@ import (
 	"net/netip"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +38,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -44,13 +48,22 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 	"tailscale.com/util/rands"
+	"tailscale.com/version"
 )
 
+// ctxConn is a key to look up a net.Conn stored in an HTTP request's context.
+type ctxConn struct{}
+
+// funnelClientsFile is the file where client IDs and secrets for OIDC clients
+// accessing the IDP over Funnel are persisted.
+const funnelClientsFile = "oidc-funnel-clients.json"
+
 var (
 	flagVerbose            = flag.Bool("verbose", false, "be verbose")
 	flagPort               = flag.Int("port", 443, "port to listen on")
 	flagLocalPort          = flag.Int("local-port", -1, "allow requests from localhost")
 	flagUseLocalTailscaled = flag.Bool("use-local-tailscaled", false, "use local tailscaled instead of tsnet")
+	flagFunnel             = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet")
 )
 
 func main() {
@@ -61,9 +74,11 @@ func main() {
 	}
 
 	var (
-		lc  *tailscale.LocalClient
-		st  *ipnstate.Status
-		err error
+		lc          *tailscale.LocalClient
+		st          *ipnstate.Status
+		err         error
+		watcherChan chan error
+		cleanup     func()
 
 		lns []net.Listener
 	)
@@ -90,6 +105,18 @@ func main() {
 		if !anySuccess {
 			log.Fatalf("failed to listen on any of %v", st.TailscaleIPs)
 		}
+
+		// tailscaled needs to be setting an HTTP header for funneled requests
+		// that older versions don't provide.
+		// TODO(naman): is this the correct check?
+		if *flagFunnel && !version.AtLeast(st.Version, "1.71.0") {
+			log.Fatalf("Local tailscaled not new enough to support -funnel. Update Tailscale or use tsnet mode.")
+		}
+		cleanup, watcherChan, err = serveOnLocalTailscaled(ctx, lc, st, uint16(*flagPort), *flagFunnel)
+		if err != nil {
+			log.Fatalf("could not serve on local tailscaled: %v", err)
+		}
+		defer cleanup()
 	} else {
 		ts := &tsnet.Server{
 			Hostname: "idp",
@@ -105,7 +132,15 @@ func main() {
 		if err != nil {
 			log.Fatalf("getting local client: %v", err)
 		}
-		ln, err := ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		var ln net.Listener
+		if *flagFunnel {
+			if err := ipn.CheckFunnelAccess(uint16(*flagPort), st.Self); err != nil {
+				log.Fatalf("%v", err)
+			}
+			ln, err = ts.ListenFunnel("tcp", fmt.Sprintf(":%d", *flagPort))
+		} else {
+			ln, err = ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		}
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -113,13 +148,26 @@ func main() {
 	}
 
 	srv := &idpServer{
-		lc: lc,
+		lc:          lc,
+		funnel:      *flagFunnel,
+		localTSMode: *flagUseLocalTailscaled,
 	}
 	if *flagPort != 443 {
 		srv.serverURL = fmt.Sprintf("https://%s:%d", strings.TrimSuffix(st.Self.DNSName, "."), *flagPort)
 	} else {
 		srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, "."))
 	}
+	if *flagFunnel {
+		f, err := os.Open(funnelClientsFile)
+		if err == nil {
+			srv.funnelClients = make(map[string]*funnelClient)
+			if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil {
+				log.Fatalf("could not parse %s: %v", funnelClientsFile, err)
+			}
+		} else if !errors.Is(err, os.ErrNotExist) {
+			log.Fatalf("could not open %s: %v", funnelClientsFile, err)
+		}
+	}
 
 	log.Printf("Running tsidp at %s ...", srv.serverURL)
 
@@ -134,35 +182,129 @@ func main() {
 	}
 
 	for _, ln := range lns {
-		go http.Serve(ln, srv)
+		server := http.Server{
+			Handler: srv,
+			ConnContext: func(ctx context.Context, c net.Conn) context.Context {
+				return context.WithValue(ctx, ctxConn{}, c)
+			},
+		}
+		go server.Serve(ln)
 	}
-	select {}
+	// need to catch os.Interrupt, otherwise deferred cleanup code doesn't run
+	exitChan := make(chan os.Signal, 1)
+	signal.Notify(exitChan, os.Interrupt)
+	select {
+	case <-exitChan:
+		log.Printf("interrupt, exiting")
+		return
+	case <-watcherChan:
+		if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
+			log.Printf("watcher closed, exiting")
+			return
+		}
+		log.Fatalf("watcher error: %v", err)
+		return
+	}
+}
+
+// serveOnLocalTailscaled starts a serve session using an already-running
+// tailscaled instead of starting a fresh tsnet server, making something
+// listening on clientDNSName:dstPort accessible over serve/funnel.
+func serveOnLocalTailscaled(ctx context.Context, lc *tailscale.LocalClient, st *ipnstate.Status, dstPort uint16, shouldFunnel bool) (cleanup func(), watcherChan chan error, err error) {
+	// In order to support funneling out in local tailscaled mode, we need
+	// to add a serve config to forward the listeners we bound above and
+	// allow those forwarders to be funneled out.
+	sc, err := lc.GetServeConfig(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get serve config: %v", err)
+	}
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+
+	// We watch the IPN bus just to get a session ID. The session expires
+	// when we stop watching the bus, and that auto-deletes the foreground
+	// serve/funnel configs we are creating below.
+	watcher, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not set up ipn bus watcher: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+	n, err := watcher.Next()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get initial state from ipn bus watcher: %v", err)
+	}
+	if n.SessionID == "" {
+		err = fmt.Errorf("missing sessionID in ipn.Notify")
+		return nil, nil, err
+	}
+	watcherChan = make(chan error)
+	go func() {
+		for {
+			_, err = watcher.Next()
+			if err != nil {
+				watcherChan <- err
+				return
+			}
+		}
+	}()
+
+	// Create a foreground serve config that gets cleaned up when tsidp
+	// exits and the session ID associated with this config is invalidated.
+	foregroundSc := new(ipn.ServeConfig)
+	mak.Set(&sc.Foreground, n.SessionID, foregroundSc)
+	serverURL := strings.TrimSuffix(st.Self.DNSName, ".")
+	fmt.Printf("setting funnel for %s:%v\n", serverURL, dstPort)
+
+	foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel)
+	foregroundSc.SetWebHandler(&ipn.HTTPHandler{
+		Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))),
+	}, serverURL, uint16(*flagPort), "/", true)
+	err = lc.SetServeConfig(ctx, sc)
+	if err != nil {
+		return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err)
+	}
+
+	return func() { watcher.Close() }, watcherChan, nil
 }
 
 type idpServer struct {
 	lc          *tailscale.LocalClient
 	loopbackURL string
 	serverURL   string // "https://foo.bar.ts.net"
+	funnel      bool
+	localTSMode bool
 
 	lazyMux        lazy.SyncValue[*http.ServeMux]
 	lazySigningKey lazy.SyncValue[*signingKey]
 	lazySigner     lazy.SyncValue[jose.Signer]
 
-	mu          sync.Mutex              // guards the fields below
-	code        map[string]*authRequest // keyed by random hex
-	accessToken map[string]*authRequest // keyed by random hex
+	mu            sync.Mutex               // guards the fields below
+	code          map[string]*authRequest  // keyed by random hex
+	accessToken   map[string]*authRequest  // keyed by random hex
+	funnelClients map[string]*funnelClient // keyed by client ID
 }
 
 type authRequest struct {
 	// localRP is true if the request is from a relying party running on the
-	// same machine as the idp server. It is mutually exclusive with rpNodeID.
+	// same machine as the idp server. It is mutually exclusive with rpNodeID
+	// and funnelRP.
 	localRP bool
 
 	// rpNodeID is the NodeID of the relying party (who requested the auth, such
 	// as Proxmox or Synology), not the user node who is being authenticated. It
-	// is mutually exclusive with localRP.
+	// is mutually exclusive with localRP and funnelRP.
 	rpNodeID tailcfg.NodeID
 
+	// funnelRP is non-nil if the request is from a relying party outside the
+	// tailnet, via Tailscale Funnel. It is mutually exclusive with rpNodeID
+	// and localRP.
+	funnelRP *funnelClient
+
 	// clientID is the "client_id" sent in the authorized request.
 	clientID string
 
@@ -181,9 +323,12 @@ type authRequest struct {
 	validTill time.Time
 }
 
-func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string, lc *tailscale.LocalClient) error {
+// allowRelyingParty validates that a relying party identified either by a
+// known remoteAddr or a valid client ID/secret pair is allowed to proceed
+// with the authorization flow associated with this authRequest.
+func (ar *authRequest) allowRelyingParty(r *http.Request, lc *tailscale.LocalClient) error {
 	if ar.localRP {
-		ra, err := netip.ParseAddrPort(remoteAddr)
+		ra, err := netip.ParseAddrPort(r.RemoteAddr)
 		if err != nil {
 			return err
 		}
@@ -192,7 +337,18 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 		}
 		return nil
 	}
-	who, err := lc.WhoIs(ctx, remoteAddr)
+	if ar.funnelRP != nil {
+		clientID, clientSecret, ok := r.BasicAuth()
+		if !ok {
+			clientID = r.FormValue("client_id")
+			clientSecret = r.FormValue("client_secret")
+		}
+		if ar.funnelRP.ID != clientID || ar.funnelRP.Secret != clientSecret {
+			return fmt.Errorf("tsidp: invalid client credentials")
+		}
+		return nil
+	}
+	who, err := lc.WhoIs(r.Context(), r.RemoteAddr)
 	if err != nil {
 		return fmt.Errorf("tsidp: error getting WhoIs: %w", err)
 	}
@@ -203,24 +359,60 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 }
 
 func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
-	who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr)
+	// This URL is visited by the user who is being authenticated. If they are
+	// visiting the URL over Funnel, that means they are not part of the
+	// tailnet that they are trying to be authenticated for.
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	uq := r.URL.Query()
+
+	redirectURI := uq.Get("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must specify redirect_uri", http.StatusBadRequest)
+		return
+	}
+
+	var remoteAddr string
+	if s.localTSMode {
+		// in local tailscaled mode, the local tailscaled is forwarding us
+		// HTTP requests, so reading r.RemoteAddr will just get us our own
+		// address.
+		remoteAddr = r.Header.Get("X-Forwarded-For")
+	} else {
+		remoteAddr = r.RemoteAddr
+	}
+	who, err := s.lc.WhoIs(r.Context(), remoteAddr)
 	if err != nil {
 		log.Printf("Error getting WhoIs: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	uq := r.URL.Query()
-
 	code := rands.HexString(32)
 	ar := &authRequest{
 		nonce:       uq.Get("nonce"),
 		remoteUser:  who,
-		redirectURI: uq.Get("redirect_uri"),
+		redirectURI: redirectURI,
 		clientID:    uq.Get("client_id"),
 	}
 
-	if r.URL.Path == "/authorize/localhost" {
+	if r.URL.Path == "/authorize/funnel" {
+		s.mu.Lock()
+		c, ok := s.funnelClients[ar.clientID]
+		s.mu.Unlock()
+		if !ok {
+			http.Error(w, "tsidp: invalid client ID", http.StatusBadRequest)
+			return
+		}
+		if ar.redirectURI != c.RedirectURI {
+			http.Error(w, "tsidp: redirect_uri mismatch", http.StatusBadRequest)
+			return
+		}
+		ar.funnelRP = c
+	} else if r.URL.Path == "/authorize/localhost" {
 		ar.localRP = true
 	} else {
 		var ok bool
@@ -237,8 +429,10 @@ func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
 
 	q := make(url.Values)
 	q.Set("code", code)
-	q.Set("state", uq.Get("state"))
-	u := uq.Get("redirect_uri") + "?" + q.Encode()
+	if state := uq.Get("state"); state != "" {
+		q.Set("state", state)
+	}
+	u := redirectURI + "?" + q.Encode()
 	log.Printf("Redirecting to %q", u)
 
 	http.Redirect(w, r, u, http.StatusFound)
@@ -251,6 +445,7 @@ func (s *idpServer) newMux() *http.ServeMux {
 	mux.HandleFunc("/authorize/", s.authorize)
 	mux.HandleFunc("/userinfo", s.serveUserInfo)
 	mux.HandleFunc("/token", s.serveToken)
+	mux.HandleFunc("/clients/", s.serveClients)
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
 			io.WriteString(w, "<html><body><h1>Tailscale OIDC IdP</h1>")
@@ -284,11 +479,6 @@ func (s *idpServer) serveUserInfo(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: invalid token", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
-		log.Printf("Error allowing relying party: %v", err)
-		http.Error(w, err.Error(), http.StatusForbidden)
-		return
-	}
 
 	if ar.validTill.Before(time.Now()) {
 		http.Error(w, "tsidp: token expired", http.StatusBadRequest)
@@ -348,7 +538,7 @@ func (s *idpServer) serveToken(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: code not found", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
+	if err := ar.allowRelyingParty(r, s.lc); err != nil {
 		log.Printf("Error allowing relying party: %v", err)
 		http.Error(w, err.Error(), http.StatusForbidden)
 		return
@@ -581,7 +771,9 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 	var authorizeEndpoint string
 	rpEndpoint := s.serverURL
-	if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
+	if isFunnelRequest(r) {
+		authorizeEndpoint = fmt.Sprintf("%s/authorize/funnel", s.serverURL)
+	} else if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
 		authorizeEndpoint = fmt.Sprintf("%s/authorize/%d", s.serverURL, who.Node.ID)
 	} else if ap.Addr().IsLoopback() {
 		rpEndpoint = s.loopbackURL
@@ -611,6 +803,148 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// funnelClient represents an OIDC client/relying party that is accessing the
+// IDP over Funnel.
+type funnelClient struct {
+	ID          string `json:"client_id"`
+	Secret      string `json:"client_secret,omitempty"`
+	Name        string `json:"name,omitempty"`
+	RedirectURI string `json:"redirect_uri"`
+}
+
+// /clients is a privileged endpoint that allows the visitor to create new
+// Funnel-capable OIDC clients, so it is only accessible over the tailnet.
+func (s *idpServer) serveClients(w http.ResponseWriter, r *http.Request) {
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	path := strings.TrimPrefix(r.URL.Path, "/clients/")
+
+	if path == "new" {
+		s.serveNewClient(w, r)
+		return
+	}
+
+	if path == "" {
+		s.serveGetClientsList(w, r)
+		return
+	}
+
+	s.mu.Lock()
+	c, ok := s.funnelClients[path]
+	s.mu.Unlock()
+	if !ok {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	switch r.Method {
+	case "DELETE":
+		s.serveDeleteClient(w, r, path)
+	case "GET":
+		json.NewEncoder(w).Encode(&funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	default:
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *idpServer) serveNewClient(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	redirectURI := r.FormValue("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must provide redirect_uri", http.StatusBadRequest)
+		return
+	}
+	clientID := rands.HexString(32)
+	clientSecret := rands.HexString(64)
+	newClient := funnelClient{
+		ID:          clientID,
+		Secret:      clientSecret,
+		Name:        r.FormValue("name"),
+		RedirectURI: redirectURI,
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.funnelClients, clientID, &newClient)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// delete the new client to avoid inconsistent state between memory
+		// and disk
+		delete(s.funnelClients, clientID)
+		return
+	}
+	json.NewEncoder(w).Encode(newClient)
+}
+
+func (s *idpServer) serveGetClientsList(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	redactedClients := make([]funnelClient, 0, len(s.funnelClients))
+	for _, c := range s.funnelClients {
+		redactedClients = append(redactedClients, funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	}
+	s.mu.Unlock()
+	json.NewEncoder(w).Encode(redactedClients)
+}
+
+func (s *idpServer) serveDeleteClient(w http.ResponseWriter, r *http.Request, clientID string) {
+	if r.Method != "DELETE" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.funnelClients == nil {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	if _, ok := s.funnelClients[clientID]; !ok {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	deleted := s.funnelClients[clientID]
+	delete(s.funnelClients, clientID)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// restore the deleted value to avoid inconsistent state between memory
+		// and disk
+		s.funnelClients[clientID] = deleted
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// storeFunnelClientsLocked writes the current mapping of OIDC client ID/secret
+// pairs for RPs that access the IDP over funnel. s.mu must be held while
+// calling this.
+func (s *idpServer) storeFunnelClientsLocked() error {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(s.funnelClients); err != nil {
+		return err
+	}
+	return os.WriteFile(funnelClientsFile, buf.Bytes(), 0600)
+}
+
 const (
 	minimumRSAKeySize = 2048
 )
@@ -700,3 +1034,24 @@ func parseID[T ~int64](input string) (_ T, ok bool) {
 	}
 	return T(i), true
 }
+
+// isFunnelRequest checks if an HTTP request is coming over Tailscale Funnel.
+func isFunnelRequest(r *http.Request) bool {
+	// If we're funneling through the local tailscaled, it will set this HTTP
+	// header.
+	if r.Header.Get("Tailscale-Funnel-Request") != "" {
+		return true
+	}
+
+	// If the funneled connection is from tsnet, then the net.Conn will be of
+	// type ipn.FunnelConn.
+	netConn := r.Context().Value(ctxConn{})
+	// if the conn is wrapped inside TLS, unwrap it
+	if tlsConn, ok := netConn.(*tls.Conn); ok {
+		netConn = tlsConn.NetConn()
+	}
+	if _, ok := netConn.(*ipn.FunnelConn); ok {
+		return true
+	}
+	return false
+}

cmd/tta/fw_linux.go

@@ -0,0 +1,128 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"encoding/binary"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/expr"
+	"tailscale.com/types/ptr"
+)
+
+func init() {
+	addFirewall = addFirewallLinux
+}
+
+func addFirewallLinux() error {
+	c, err := nftables.New()
+	if err != nil {
+		return err
+	}
+
+	// Create a new table
+	table := &nftables.Table{
+		Family: nftables.TableFamilyIPv4, // TableFamilyINet doesn't work (why?. oh well.)
+		Name:   "filter",
+	}
+	c.AddTable(table)
+
+	// Create a new chain for incoming traffic
+	inputChain := &nftables.Chain{
+		Name:     "input",
+		Table:    table,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookInput,
+		Priority: nftables.ChainPriorityFilter,
+		Policy:   ptr.To(nftables.ChainPolicyDrop),
+	}
+	c.AddChain(inputChain)
+
+	// Allow traffic from the loopback interface
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte("lo"),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	// Accept established and related connections
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Ct{
+				Register: 1,
+				Key:      expr.CtKeySTATE,
+			},
+			&expr.Bitwise{
+				SourceRegister: 1,
+				DestRegister:   1,
+				Len:            4,
+				Mask:           binary.NativeEndian.AppendUint32(nil, 0x06), // CT_STATE_BIT_ESTABLISHED | CT_STATE_BIT_RELATED
+				Xor:            binary.NativeEndian.AppendUint32(nil, 0),
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     binary.NativeEndian.AppendUint32(nil, 0x00),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	// Allow TCP packets in that don't have the SYN bit set, even if they're not
+	// ESTABLISHED or RELATED. This is because the test suite gets TCP
+	// connections up & idle (for HTTP) before it conditionally installs these
+	// firewall rules. But because conntrack wasn't previously active, existing
+	// TCP flows aren't ESTABLISHED and get dropped. So this rule allows
+	// previously established TCP connections that predates the firewall rules
+	// to continue working, as they don't have conntrack state.
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte{0x06}, // TCP
+			},
+			&expr.Payload{ // get TCP flags
+				DestRegister: 1,
+				Base:         2,
+				Offset:       13, // flags
+				Len:          1,
+			},
+			&expr.Bitwise{
+				SourceRegister: 1,
+				DestRegister:   1,
+				Len:            1,
+				Mask:           []byte{2}, // TCP_SYN
+				Xor:            []byte{0},
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     []byte{2}, // TCP_SYN
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	return c.Flush()
+}

cmd/tta/tta.go

@@ -0,0 +1,238 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The tta server is the Tailscale Test Agent.
+//
+// It runs on each Tailscale node being integration tested and permits the test
+// harness to control the node. It connects out to the test drver (rather than
+// accepting any TCP connections inbound, which might be blocked depending on
+// the scenario being tested) and then the test driver turns the TCP connection
+// around and sends request back.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/hostinfo"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+	"tailscale.com/version/distro"
+)
+
+var (
+	driverAddr = flag.String("driver", "test-driver.tailscale:8008", "address of the test driver; by default we use the DNS name test-driver.tailscale which is special cased in the emulated network's DNS server")
+)
+
+func absify(cmd string) string {
+	if distro.Get() == distro.Gokrazy && !strings.Contains(cmd, "/") {
+		return "/user/" + cmd
+	}
+	return cmd
+}
+
+func serveCmd(w http.ResponseWriter, cmd string, args ...string) {
+	log.Printf("Got serveCmd for %q %v", cmd, args)
+	out, err := exec.Command(absify(cmd), args...).CombinedOutput()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if err != nil {
+		w.Header().Set("Exec-Err", err.Error())
+		w.WriteHeader(500)
+		log.Printf("Err on serveCmd for %q %v, %d bytes of output: %v", cmd, args, len(out), err)
+	} else {
+		log.Printf("Did serveCmd for %q %v, %d bytes of output", cmd, args, len(out))
+	}
+	w.Write(out)
+}
+
+type localClientRoundTripper struct {
+	lc tailscale.LocalClient
+}
+
+func (rt *localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.RequestURI = ""
+	return rt.lc.DoLocalRequest(req)
+}
+
+func main() {
+	if distro.Get() == distro.Gokrazy {
+		if !hostinfo.IsNATLabGuestVM() {
+			// "Exiting immediately with status code 0 when the
+			// GOKRAZY_FIRST_START=1 environment variable is set means “don’t
+			// start the program on boot”"
+			return
+		}
+	}
+	flag.Parse()
+
+	if distro.Get() == distro.Gokrazy {
+		nsRx := regexp.MustCompile(`(?m)^nameserver (.*)`)
+		for t := time.Now(); time.Since(t) < 10*time.Second; time.Sleep(10 * time.Millisecond) {
+			all, _ := os.ReadFile("/etc/resolv.conf")
+			if nsRx.Match(all) {
+				break
+			}
+		}
+	}
+
+	logc, err := net.Dial("tcp", "9.9.9.9:124")
+	if err == nil {
+		log.SetOutput(logc)
+	}
+
+	log.Printf("Tailscale Test Agent running.")
+
+	gokRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+	gokRP.Transport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if network != "tcp" {
+				return nil, errors.New("unexpected network")
+			}
+			if addr != "gokrazy:80" {
+				return nil, errors.New("unexpected addr")
+			}
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", "/run/gokrazy-http.sock")
+		},
+	}
+
+	var ttaMux http.ServeMux // agent mux
+	var serveMux http.ServeMux
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("X-TTA-GoKrazy") == "1" {
+			gokRP.ServeHTTP(w, r)
+			return
+		}
+		ttaMux.ServeHTTP(w, r)
+	})
+	var hs http.Server
+	hs.Handler = &serveMux
+	var (
+		stMu   sync.Mutex
+		newSet = set.Set[net.Conn]{} // conns in StateNew
+	)
+	needConnCh := make(chan bool, 1)
+	hs.ConnState = func(c net.Conn, s http.ConnState) {
+		stMu.Lock()
+		defer stMu.Unlock()
+		oldLen := len(newSet)
+		switch s {
+		case http.StateNew:
+			newSet.Add(c)
+		default:
+			newSet.Delete(c)
+		}
+		if oldLen != 0 && len(newSet) == 0 {
+			select {
+			case needConnCh <- true:
+			default:
+			}
+		}
+	}
+	conns := make(chan net.Conn, 1)
+
+	lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock")))
+	lcRP.Transport = new(localClientRoundTripper)
+	ttaMux.HandleFunc("/localapi/", func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("Got localapi request: %v", r.URL)
+		t0 := time.Now()
+		lcRP.ServeHTTP(w, r)
+		log.Printf("Did localapi request in %v: %v", time.Since(t0).Round(time.Millisecond), r.URL)
+	})
+
+	ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "TTA\n")
+		return
+	})
+	ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale")
+	})
+	ttaMux.HandleFunc("/fw", addFirewallHandler)
+
+	go hs.Serve(chanListener(conns))
+
+	// For doing agent operations locally from gokrazy:
+	// (e.g. with "wget -O - localhost:8123/fw")
+	go func() {
+		err := http.ListenAndServe("127.0.0.1:8123", &ttaMux)
+		if err != nil {
+			log.Fatalf("ListenAndServe: %v", err)
+		}
+	}()
+
+	var lastErr string
+	needConnCh <- true
+	for {
+		<-needConnCh
+		c, err := connect()
+		if err != nil {
+			s := err.Error()
+			if s != lastErr {
+				log.Printf("Connect failure: %v", s)
+			}
+			lastErr = s
+			time.Sleep(time.Second)
+			continue
+		}
+		conns <- c
+	}
+}
+
+func connect() (net.Conn, error) {
+	c, err := net.Dial("tcp", *driverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+type chanListener <-chan net.Conn
+
+func (cl chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl
+	if !ok {
+		return nil, errors.New("closed")
+	}
+	return c, nil
+}
+
+func (cl chanListener) Close() error {
+	return nil
+}
+
+func (cl chanListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("52.0.0.34"), // TS..DR(iver)
+		Port: 123,
+	}
+}
+
+func addFirewallHandler(w http.ResponseWriter, r *http.Request) {
+	if addFirewall == nil {
+		http.Error(w, "firewall not supported", 500)
+		return
+	}
+	err := addFirewall()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	io.WriteString(w, "OK\n")
+}
+
+var addFirewall func() error // set by fw_linux.go

cmd/viewer/tests/tests.go

@@ -152,13 +152,53 @@ func ContainerViewOf[T views.ViewCloner[T, V], V views.StructView[T]](c *Contain
 	return ContainerView[T, V]{c}
 }
 
+// MapContainer is a predefined map-like container type.
+// Unlike [Container], it has two type parameters, where the value
+// is the second parameter.
+type MapContainer[K comparable, V views.Cloner[V]] struct {
+	Items map[K]V
+}
+
+func (c *MapContainer[K, V]) Clone() *MapContainer[K, V] {
+	if c == nil {
+		return nil
+	}
+	var m map[K]V
+	if c.Items != nil {
+		m = make(map[K]V, len(c.Items))
+		for i := range m {
+			m[i] = c.Items[i].Clone()
+		}
+	}
+	return &MapContainer[K, V]{m}
+}
+
+// MapContainerView is a pre-defined readonly view of a [MapContainer][K, T].
+type MapContainerView[K comparable, T views.ViewCloner[T, V], V views.StructView[T]] struct {
+	// ж is the underlying mutable value, named with a hard-to-type
+	// character that looks pointy like a pointer.
+	// It is named distinctively to make you think of how dangerous it is to escape
+	// to callers. You must not let callers be able to mutate it.
+	ж *MapContainer[K, T]
+}
+
+func (cv MapContainerView[K, T, V]) Items() views.MapFn[K, T, V] {
+	return views.MapFnOf(cv.ж.Items, func(t T) V { return t.View() })
+}
+
+func MapContainerViewOf[K comparable, T views.ViewCloner[T, V], V views.StructView[T]](c *MapContainer[K, T]) MapContainerView[K, T, V] {
+	return MapContainerView[K, T, V]{c}
+}
+
 type GenericBasicStruct[T BasicType] struct {
 	Value T
 }
 
 type StructWithContainers struct {
-	IntContainer             Container[int]
-	CloneableContainer       Container[*StructWithPtrs]
-	BasicGenericContainer    Container[GenericBasicStruct[int]]
-	ClonableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	IntContainer              Container[int]
+	CloneableContainer        Container[*StructWithPtrs]
+	BasicGenericContainer     Container[GenericBasicStruct[int]]
+	CloneableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	CloneableMap              MapContainer[int, *StructWithPtrs]
+	CloneableGenericMap       MapContainer[int, *GenericNoPtrsStruct[int]]
 }

cmd/viewer/tests/tests_clone.go

@@ -426,14 +426,18 @@ func (src *StructWithContainers) Clone() *StructWithContainers {
 	dst := new(StructWithContainers)
 	*dst = *src
 	dst.CloneableContainer = *src.CloneableContainer.Clone()
-	dst.ClonableGenericContainer = *src.ClonableGenericContainer.Clone()
+	dst.CloneableGenericContainer = *src.CloneableGenericContainer.Clone()
+	dst.CloneableMap = *src.CloneableMap.Clone()
+	dst.CloneableGenericMap = *src.CloneableGenericMap.Clone()
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _StructWithContainersCloneNeedsRegeneration = StructWithContainers(struct {
-	IntContainer             Container[int]
-	CloneableContainer       Container[*StructWithPtrs]
-	BasicGenericContainer    Container[GenericBasicStruct[int]]
-	ClonableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	IntContainer              Container[int]
+	CloneableContainer        Container[*StructWithPtrs]
+	BasicGenericContainer     Container[GenericBasicStruct[int]]
+	CloneableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	CloneableMap              MapContainer[int, *StructWithPtrs]
+	CloneableGenericMap       MapContainer[int, *GenericNoPtrsStruct[int]]
 }{})

cmd/viewer/tests/tests_view.go

@@ -657,14 +657,22 @@ func (v StructWithContainersView) CloneableContainer() ContainerView[*StructWith
 func (v StructWithContainersView) BasicGenericContainer() Container[GenericBasicStruct[int]] {
 	return v.ж.BasicGenericContainer
 }
-func (v StructWithContainersView) ClonableGenericContainer() ContainerView[*GenericNoPtrsStruct[int], GenericNoPtrsStructView[int]] {
-	return ContainerViewOf(&v.ж.ClonableGenericContainer)
+func (v StructWithContainersView) CloneableGenericContainer() ContainerView[*GenericNoPtrsStruct[int], GenericNoPtrsStructView[int]] {
+	return ContainerViewOf(&v.ж.CloneableGenericContainer)
+}
+func (v StructWithContainersView) CloneableMap() MapContainerView[int, *StructWithPtrs, StructWithPtrsView] {
+	return MapContainerViewOf(&v.ж.CloneableMap)
+}
+func (v StructWithContainersView) CloneableGenericMap() MapContainerView[int, *GenericNoPtrsStruct[int], GenericNoPtrsStructView[int]] {
+	return MapContainerViewOf(&v.ж.CloneableGenericMap)
 }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _StructWithContainersViewNeedsRegeneration = StructWithContainers(struct {
-	IntContainer             Container[int]
-	CloneableContainer       Container[*StructWithPtrs]
-	BasicGenericContainer    Container[GenericBasicStruct[int]]
-	ClonableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	IntContainer              Container[int]
+	CloneableContainer        Container[*StructWithPtrs]
+	BasicGenericContainer     Container[GenericBasicStruct[int]]
+	CloneableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	CloneableMap              MapContainer[int, *StructWithPtrs]
+	CloneableGenericMap       MapContainer[int, *GenericNoPtrsStruct[int]]
 }{})

cmd/viewer/viewer.go

@@ -448,7 +448,7 @@ func viewTypeForContainerType(typ types.Type) (*types.Named, *types.Func) {
 	}
 	// ...and add the element view type.
 	// For that, we need to first determine the named elem type...
-	elemType, ok := baseType(containerType.TypeArgs().At(0)).(*types.Named)
+	elemType, ok := baseType(containerType.TypeArgs().At(containerType.TypeArgs().Len() - 1)).(*types.Named)
 	if !ok {
 		return nil, nil
 	}

cmd/vnet/run-krazy.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+echo "Type 'C-a c' to enter monitor; q to quit."
+
+set -eux
+qemu-system-x86_64 -M microvm,isa-serial=off \
+    -m 1G \
+    -nodefaults -no-user-config -nographic \
+    -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
+    -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1 tailscaled.env=TS_DEBUG_RAW_DISCO=1" \
+    -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/natlabapp.img,format=raw \
+    -device virtio-blk-device,drive=blk0 \
+    -device virtio-rng-device \
+    -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \
+    -device virtio-serial-device \
+    -device virtio-net-device,netdev=net0,mac=52:cc:cc:cc:cc:01 \
+    -chardev stdio,id=virtiocon0,mux=on \
+    -device virtconsole,chardev=virtiocon0 \
+    -mon chardev=virtiocon0,mode=readline \
+    -audio none
+

cmd/vnet/vnet-main.go

@@ -0,0 +1,118 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The vnet binary runs a virtual network stack in userspace for qemu instances
+// to connect to and simulate various network conditions.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+)
+
+var (
+	listen  = flag.String("listen", "/tmp/qemu.sock", "path to listen on")
+	nat     = flag.String("nat", "easy", "type of NAT to use")
+	nat2    = flag.String("nat2", "hard", "type of NAT to use for second network")
+	portmap = flag.Bool("portmap", false, "enable portmapping")
+	dgram   = flag.Bool("dgram", false, "enable datagram mode; for use with macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment")
+)
+
+func main() {
+	flag.Parse()
+
+	if _, err := os.Stat(*listen); err == nil {
+		os.Remove(*listen)
+	}
+
+	var srv net.Listener
+	var err error
+	var conn *net.UnixConn
+	if *dgram {
+		addr, err := net.ResolveUnixAddr("unixgram", *listen)
+		if err != nil {
+			log.Fatalf("ResolveUnixAddr: %v", err)
+		}
+		conn, err = net.ListenUnixgram("unixgram", addr)
+		if err != nil {
+			log.Fatalf("ListenUnixgram: %v", err)
+		}
+		defer conn.Close()
+	} else {
+		srv, err = net.Listen("unix", *listen)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var c vnet.Config
+	node1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.NAT(*nat)))
+	c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", vnet.NAT(*nat2)))
+	if *portmap {
+		node1.Network().AddService(vnet.NATPMP)
+	}
+
+	s, err := vnet.New(&c)
+	if err != nil {
+		log.Fatalf("newServer: %v", err)
+	}
+
+	if err := s.PopulateDERPMapIPs(); err != nil {
+		log.Printf("warning: ignoring failure to populate DERP map: %v", err)
+	}
+
+	s.WriteStartingBanner(os.Stdout)
+	nc := s.NodeAgentClient(node1)
+	go func() {
+		rp := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+		d := rp.Director
+		rp.Director = func(r *http.Request) {
+			d(r)
+			r.Header.Set("X-TTA-GoKrazy", "1")
+		}
+		rp.Transport = nc.HTTPClient.Transport
+		http.ListenAndServe(":8080", rp)
+	}()
+	go func() {
+		getStatus := func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			st, err := nc.Status(ctx)
+			if err != nil {
+				log.Printf("NodeStatus: %v", err)
+				return
+			}
+			log.Printf("NodeStatus: %v", logger.AsJSON(st))
+		}
+		for {
+			time.Sleep(5 * time.Second)
+			//continue
+			getStatus()
+		}
+	}()
+
+	if conn != nil {
+		s.ServeUnixConn(conn, vnet.ProtocolUnixDGRAM)
+		return
+	}
+
+	for {
+		c, err := srv.Accept()
+		if err != nil {
+			log.Printf("Accept: %v", err)
+			continue
+		}
+		go s.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+	}
+}

control/controlhttp/client_js.go

@@ -10,7 +10,7 @@ import (
 	"net"
 	"net/url"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
 	"tailscale.com/net/wsconn"
 )

control/controlhttp/server.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"time"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/wsconn"

derp/derphttp/derphttp_server.go

@@ -18,6 +18,7 @@ import (
 // following its HTTP request.
 const fastStartHeader = "Derp-Fast-Start"
 
+// Handler returns an http.Handler to be mounted at /derp, serving s.
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// These are installed both here and in cmd/derper. The check here
@@ -79,3 +80,29 @@ func ProbeHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
 	}
 }
+
+// ServeNoContent generates the /generate_204 response used by Tailscale's
+// captive portal detection.
+func ServeNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(NoContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(NoContentResponseHeader, "response "+challenge)
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
+const (
+	NoContentChallengeHeader = "X-Tailscale-Challenge"
+	NoContentResponseHeader  = "X-Tailscale-Response"
+)

derp/derphttp/websocket.go

@@ -10,7 +10,7 @@ import (
 	"log"
 	"net"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/net/wsconn"
 )
 

flake.nix

@@ -120,4 +120,4 @@
   in
     flake-utils.lib.eachDefaultSystem (system: flakeForSystem nixpkgs system);
 }
-# nix-direnv cache busting line: sha256-1hekcJr1jEJFu4ZnapNkbAAv+8phTQuMloULIZ0f018=
+# nix-direnv cache busting line: sha256-M5e5dE1gGW3ly94r3SxCsBmVwbBmhVtaVDW691vxG/8=

go.mod

@@ -15,9 +15,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7
 	github.com/bramvdbogaerde/go-scp v1.4.0
 	github.com/cilium/ebpf v0.15.0
+	github.com/coder/websocket v1.8.12
 	github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/creack/pty v1.1.21
+	github.com/creack/pty v1.1.23
 	github.com/dave/courtney v0.4.0
 	github.com/dave/patsy v0.0.0-20210517141501-957256f50cba
 	github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa
@@ -39,6 +40,7 @@ require (
 	github.com/golangci/golangci-lint v1.52.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.18.0
+	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
 	github.com/goreleaser/nfpm/v2 v2.33.1
@@ -110,7 +112,6 @@ require (
 	k8s.io/apimachinery v0.30.3
 	k8s.io/apiserver v0.30.3
 	k8s.io/client-go v0.30.3
-	nhooyr.io/websocket v1.8.10
 	sigs.k8s.io/controller-runtime v0.18.4
 	sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab
 	sigs.k8s.io/yaml v1.4.0

go.mod.sri

@@ -1 +1 @@
-sha256-1hekcJr1jEJFu4ZnapNkbAAv+8phTQuMloULIZ0f018=
+sha256-M5e5dE1gGW3ly94r3SxCsBmVwbBmhVtaVDW691vxG/8=

go.sum

@@ -218,6 +218,8 @@ github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBS
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
+github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
@@ -228,8 +230,8 @@ github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pq
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
-github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0=
+github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
 github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
@@ -477,6 +479,8 @@ github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -1528,8 +1532,6 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8 h1:VuJo4Mt0EVPychre4fNlDWDuE5AjXtPJpRUWqZDQhaI=
 mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8/go.mod h1:Oh/d7dEtzsNHGOq1Cdv8aMm3KdKhVvPbRQcM8WFpBR8=
-nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=
-nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

go.toolchain.rev

@@ -1 +1 @@
-2f152a4eff5875655a9a84fce8f8d329f8d9a321
+22ef9eb38e9a2d21b4a45f7adc75addb05f3efb8

gokrazy/.gitignore

@@ -1,2 +1,3 @@
-tsapp.img
+*.qcow2
+*.img
 go.work

gokrazy/Makefile

@@ -6,3 +6,8 @@ image:
 
 qemu: image
 	qemu-system-x86_64 -m 1G -drive file=tsapp.img,format=raw -boot d -netdev user,id=user.0 -device virtio-net-pci,netdev=user.0 -serial mon:stdio -audio none
+
+# For natlab integration tests:
+natlab:
+	go run build.go --build --app=natlabapp
+	qemu-img convert -O qcow2 natlabapp.img natlabapp.qcow2

gokrazy/build.go

@@ -22,10 +22,12 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strings"
 	"time"
 )
 
 var (
+	app    = flag.String("app", "tsapp", "appliance name; one of the subdirectories of gokrazy/")
 	bucket = flag.String("bucket", "tskrazy-import", "S3 bucket to upload disk image to while making AMI")
 	build  = flag.Bool("build", false, "if true, just build locally and stop, without uploading")
 )
@@ -53,6 +55,10 @@ func findMkfsExt4() (string, error) {
 func main() {
 	flag.Parse()
 
+	if *app == "" || strings.Contains(*app, "/") {
+		log.Fatalf("--app must be non-empty name such as 'tsapp' or 'natlabapp'")
+	}
+
 	if err := buildImage(); err != nil {
 		log.Fatalf("build image: %v", err)
 	}
@@ -75,7 +81,7 @@ func main() {
 	}
 	log.Printf("snap ID: %v", snapID)
 
-	ami, err := makeAMI(fmt.Sprintf("tsapp-%d", time.Now().Unix()), snapID)
+	ami, err := makeAMI(fmt.Sprintf(*app+"-%d", time.Now().Unix()), snapID)
 	if err != nil {
 		log.Fatalf("makeAMI: %v", err)
 	}
@@ -92,8 +98,8 @@ func buildImage() error {
 	if err != nil {
 		return err
 	}
-	if fi, err := os.Stat(filepath.Join(dir, "tsapp")); err != nil || !fi.IsDir() {
-		return fmt.Errorf("in wrong directorg %v; no tsapp subdirectory found", dir)
+	if fi, err := os.Stat(filepath.Join(dir, *app)); err != nil || !fi.IsDir() {
+		return fmt.Errorf("in wrong directorg %v; no %q subdirectory found", dir, *app)
 	}
 	// Build the tsapp.img
 	var buf bytes.Buffer
@@ -101,9 +107,9 @@ func buildImage() error {
 		"-exec=env GOOS=linux GOARCH=amd64 ",
 		"github.com/gokrazy/tools/cmd/gok",
 		"--parent_dir="+dir,
-		"--instance=tsapp",
+		"--instance="+*app,
 		"overwrite",
-		"--full", "tsapp.img",
+		"--full", *app+".img",
 		"--target_storage_bytes=1258299392")
 	cmd.Stdout = io.MultiWriter(os.Stdout, &buf)
 	cmd.Stderr = os.Stderr
@@ -135,14 +141,14 @@ func buildImage() error {
 }
 
 func copyToS3() error {
-	cmd := exec.Command("aws", "s3", "cp", "tsapp.img", "s3://"+*bucket+"/")
+	cmd := exec.Command("aws", "s3", "cp", *app+".img", "s3://"+*bucket+"/")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	return cmd.Run()
 }
 
 func startImportSnapshot() (importTaskID string, err error) {
-	out, err := exec.Command("aws", "ec2", "import-snapshot", "--disk-container", "Url=s3://"+*bucket+"/tsappp.img").CombinedOutput()
+	out, err := exec.Command("aws", "ec2", "import-snapshot", "--disk-container", "Url=s3://"+*bucket+"/"+*app+".img").CombinedOutput()
 	if err != nil {
 		return "", fmt.Errorf("import snapshot: %v: %s", err, out)
 	}

gokrazy/go.mod

@@ -2,12 +2,12 @@ module tailscale.com/gokrazy
 
 go 1.22
 
-require github.com/gokrazy/tools v0.0.0-20240510170341-34b02e215bc2
+require github.com/gokrazy/tools v0.0.0-20240730192548-9f81add3a91e
 
 require (
 	github.com/breml/rootcerts v0.2.10 // indirect
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 // indirect
-	github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
 	github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 // indirect
 	github.com/google/renameio/v2 v2.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -18,8 +18,6 @@ require (
 	golang.org/x/sys v0.20.0 // indirect
 )
 
-replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
+replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678
 
-replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf
-
-replace github.com/gokrazy/internal => github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd
+replace github.com/gokrazy/tools => github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e

gokrazy/go.sum

@@ -3,6 +3,8 @@ github.com/breml/rootcerts v0.2.10/go.mod h1:24FDtzYMpqIeYC7QzaE8VPRQaFZU5TIUDly
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao=
 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2 h1:kBY5R1tSf+EYZ+QaSrofLaVJtBqYsVNVBWkdMq3Smcg=
 github.com/gokrazy/updater v0.0.0-20230215172637-813ccc7f21e2/go.mod h1:PYOvzGOL4nlBmuxu7IyKQTFLaxr61+WPRNRzVtuYOHw=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
@@ -17,10 +19,8 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd h1:ZJplHHhYSzxYmrXuDPCNChGRZbLkPqRkYqRBM7KNyng=
-github.com/tailscale/gokrazy-internal v0.0.0-20240602195241-04c5eda9f6cd/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf h1:lmAGqLbIVoMK1TYWqJvxKFsu+Tb1OecgvXTmypZGAZY=
-github.com/tailscale/gokrazy-tools v0.0.0-20240602210012-933640538dcf/go.mod h1:+PSix9a8BHqAz6RV/9+tiE3C1ou0GA1ViR8pqAZVfwI=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e h1:3/xIc1QCvnKL7BCLng9od98HEvxCadjvqiI/bN+Twso=
+github.com/tailscale/gokrazy-tools v0.0.0-20240730192548-9f81add3a91e/go.mod h1:eTZ0QsugEPFU5UAQ/87bKMkPxQuTNa7+iFAIahOFwRg=
 golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=

gokrazy/natlabapp/README.md

@@ -0,0 +1,6 @@
+# NATLab Linux test Appliance
+
+This is the definition of the NATLab Linux test appliance image.
+It's similar to ../tsapp, but optimized for running in qemu in NATLab.
+
+See ../tsapp/README.md for more info.

gokrazy/natlabapp/builddir/github.com/gokrazy/gokrazy/cmd/dhcp/go.mod

@@ -0,0 +1,18 @@
+module gokrazy/build/tsapp
+
+go 1.22.2
+
+require (
+	github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/josharian/native v1.0.0 // indirect
+	github.com/mdlayher/packet v1.0.0 // indirect
+	github.com/mdlayher/socket v0.2.3 // indirect
+	github.com/rtr7/dhcp4 v0.0.0-20220302171438-18c84d089b46 // indirect
+	github.com/vishvananda/netlink v1.1.0 // indirect
+	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)

gokrazy/natlabapp/builddir/github.com/gokrazy/gokrazy/cmd/dhcp/go.sum

@@ -0,0 +1,39 @@
+github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZD931Lh75mdJfzEEXrL+Dvi97Ck3A=
+github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
+github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
+github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk=
+github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/mdlayher/packet v1.0.0 h1:InhZJbdShQYt6XV2GPj5XHxChzOfhJJOMbvnGAmOfQ8=
+github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU=
+github.com/mdlayher/socket v0.2.3 h1:XZA2X2TjdOwNoNPVPclRCURoX/hokBY8nkTmRZFEheM=
+github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY=
+github.com/rtr7/dhcp4 v0.0.0-20220302171438-18c84d089b46 h1:3psQveH4RUiv5yc3p7kRySilf1nSXLQhAvJFwg4fgnE=
+github.com/rtr7/dhcp4 v0.0.0-20220302171438-18c84d089b46/go.mod h1:Ng1F/s+z0zCMsbEFEneh+30LJa9DrTfmA+REbEqcTPk=
+github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
+github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

gokrazy/natlabapp/builddir/github.com/gokrazy/gokrazy/go.mod

@@ -0,0 +1,15 @@
+module gokrazy/build/tsapp
+
+go 1.22.2
+
+require (
+	github.com/gokrazy/gokrazy v0.0.0-20240802144848-676865a4e84f // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/kenshaw/evdev v0.1.0 // indirect
+	github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
+
+replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678

gokrazy/natlabapp/builddir/github.com/gokrazy/gokrazy/go.sum

@@ -2,6 +2,8 @@ github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZ
 github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a h1:FKeN678rNpKTpWRdFbAhYL9mWzPu57R5XPXCR3WmXdI=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/kenshaw/evdev v0.1.0 h1:wmtceEOFfilChgdNT+c/djPJ2JineVsQ0N14kGzFRUo=
@@ -10,5 +12,12 @@ github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b h1:7tUBfsEEBWfFe
 github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b/go.mod h1:bmoJUS6qOA3uKFvF3KVuhf7mU1KQirzQMeHXtPyKEqg=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a h1:7dnA8x14JihQmKbPr++Y5CCN/XSyDmOB6cXUxcIj6VQ=
+github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f h1:ZSAGWpgs+6dK2oIz5OR+HUul3oJbnhFn8YNgcZ3d9SQ=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678 h1:2B8/FbIRqmVgRUulQ4iu1EojniufComYe5Yj4BtIn1c=
+github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+golang.org/x/sys v0.0.0-20201005065044-765f4ea38db3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gokrazy/natlabapp/builddir/github.com/gokrazy/serial-busybox/go.mod

@@ -0,0 +1,5 @@
+module gokrazy/build/tsapp
+
+go 1.22.2
+
+require github.com/gokrazy/serial-busybox v0.0.0-20220918193710-d728912733ca // indirect

gokrazy/natlabapp/builddir/github.com/gokrazy/serial-busybox/go.sum

@@ -0,0 +1,26 @@
+github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gokrazy/gokrazy v0.0.0-20200501080617-f3445e01a904 h1:eqfH4A/LLgxv5RvqEXwVoFvfmpRa8+TokRjB5g6xBkk=
+github.com/gokrazy/gokrazy v0.0.0-20200501080617-f3445e01a904/go.mod h1:pq6rGHqxMRPSaTXaCMzIZy0wLDusAJyoVNyNo05RLs0=
+github.com/gokrazy/internal v0.0.0-20200407075822-660ad467b7c9 h1:x5jR/nNo4/kMSoNo/nwa2xbL7PN1an8S3oIn4OZJdec=
+github.com/gokrazy/internal v0.0.0-20200407075822-660ad467b7c9/go.mod h1:LA5TQy7LcvYGQOy75tkrYkFUhbV2nl5qEBP47PSi2JA=
+github.com/gokrazy/serial-busybox v0.0.0-20220918193710-d728912733ca h1:x0eSjuFy8qsRctVHeWm3EC474q3xm4h3OOOrYpcqyyA=
+github.com/gokrazy/serial-busybox v0.0.0-20220918193710-d728912733ca/go.mod h1:OYcG5tSb+QrelmUOO4EZVUFcIHyyZb0QDbEbZFUp1TA=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gopacket v1.1.16/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw=
+github.com/mdlayher/raw v0.0.0-20190303161257-764d452d77af/go.mod h1:rC/yE65s/DoHB6BzVOUBNYBGTg772JVytyAytffIZkY=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rtr7/dhcp4 v0.0.0-20181120124042-778e8c2e24a5/go.mod h1:FwstIpm6vX98QgtR8KEwZcVjiRn2WP76LjXAHj84fK0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0=
+golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gokrazy/natlabapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod

@@ -0,0 +1,5 @@
+module gokrazy/build/tsapp
+
+go 1.22.2
+
+require github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e // indirect

gokrazy/natlabapp/builddir/github.com/tailscale/gokrazy-kernel/go.sum

@@ -0,0 +1,4 @@
+github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 h1:xzf+cMvBJBcA/Av7OTWBa0Tjrbfcy00TeatJeJt6zrY=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e h1:tyUUgeRPGHjCZWycRnhdx8Lx9DRkjl3WsVUxYMrVBOw=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=

gokrazy/natlabapp/builddir/tailscale.com/go.mod

@@ -0,0 +1,9 @@
+module gokrazy/build/tsapp
+
+go 1.22.0
+
+toolchain go1.22.2
+
+replace tailscale.com => ../../../..
+
+require tailscale.com v0.0.0-00010101000000-000000000000 // indirect

gokrazy/natlabapp/builddir/tailscale.com/go.sum

@@ -0,0 +1,188 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
+github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/config v1.26.5 h1:lodGSevz7d+kkFJodfauThRxK9mdJbyutUxGq1NNhvw=
+github.com/aws/aws-sdk-go-v2/config v1.26.5/go.mod h1:DxHrz6diQJOc9EwDslVRh84VjjrE17g+pVZXUeSxaDU=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7 h1:a8HvP/+ew3tKwSXqL3BCSjiuicr+XTU2eFYeogV9GJE=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7/go.mod h1:Q7XIWsMo0JcMpI/6TGD6XXcXcV1DbTj6e9BKNntIMIM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
+github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6 h1:8h5+bWd7R6AYUslN6c6iuZWTKsKxUFDlpnmilO6R2n0=
+github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
+github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
+github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q=
+github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A=
+github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
+github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
+github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA=
+github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/gaissmai/bart v0.11.1 h1:5Uv5XwsaFBRo4E5VBcb9TzY8B7zxFf+U7isDxqOrRfc=
+github.com/gaissmai/bart v0.11.1/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
+github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg=
+github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA=
+github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 h1:sQspH8M4niEijh3PFscJRLDnkL547IeP7kpPe3uUhEg=
+github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466/go.mod h1:ZiQxhyQ+bbbfxUKVvjfO498oPYvtYhZzycal3G/NHmU=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
+github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 h1:wG8RYIyctLhdFk6Vl1yPGtSRtwGpVkWyZww1OCil2MI=
+github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/csrf v1.7.2 h1:oTUjx0vyf2T+wkrx09Trsev1TE+/EbDAeHtSTbtC2eI=
+github.com/gorilla/csrf v1.7.2/go.mod h1:F1Fj3KG23WYHE6gozCmBAezKookxbIvUJT+121wTuLk=
+github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA=
+github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
+github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU=
+github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo=
+github.com/illarion/gonotify v1.0.1 h1:F1d+0Fgbq/sDWjj/r66ekjDG+IDeecQKUFH4wNwsoio=
+github.com/illarion/gonotify v1.0.1/go.mod h1:zt5pmDofZpU1f8aqlK0+95eQhoEAn/d4G4B/FjVW4jE=
+github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 h1:9K06NfxkBh25x56yVhWWlKFE8YpicaSfHwoV8SFbueA=
+github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI=
+github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g=
+github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 h1:elKwZS1OcdQ0WwEDBeqxKwb7WB62QX8bvZ/FJnVXIfk=
+github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86/go.mod h1:aFAMtuldEgx/4q7iSGazk22+IcgvtiC+HIimFO9XlS8=
+github.com/jsimonetti/rtnetlink v1.4.0 h1:Z1BF0fRgcETPEa0Kt0MRk3yV5+kF1FWTni6KUFKrq2I=
+github.com/jsimonetti/rtnetlink v1.4.0/go.mod h1:5W1jDvWdnthFJ7fxYX1GMK07BUpI4oskfOqvPteYS6E=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a h1:+RR6SqnTkDLWyICxS1xpjCi/3dhyV+TgZwA6Ww3KncQ=
+github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a/go.mod h1:YTtCCM3ryyfiu4F7t8HQ1mxvp1UBdWM2r6Xa+nGWvDk=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
+github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
+github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
+github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/sdnotify v1.0.0 h1:Ma9XeLVN/l0qpyx1tNeMSeTjCPH6NtuD6/N9XdTlQ3c=
+github.com/mdlayher/sdnotify v1.0.0/go.mod h1:HQUmpM4XgYkhDLtd+Uad8ZFK1T9D5+pNxnXQjCeJlGE=
+github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI=
+github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI=
+github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
+github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
+github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc=
+github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
+github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
+github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0=
+github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
+github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4 h1:rXZGgEa+k2vJM8xT0PoSKfVXwFGPQ3z3CJfmnHJkZZw=
+github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4/go.mod h1:ikbF+YT089eInTp9f2vmvy4+ZVnW5hzX1q2WknxSprQ=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw=
+github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
+github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 h1:Gz0rz40FvFVLTBk/K8UNAenb36EbDSnh+q7Z9ldcC8w=
+github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4/go.mod h1:phI29ccmHQBc+wvroosENp1IF9195449VDnFDhJ4rJU=
+github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:tdUdyPqJ0C97SJfjB9tW6EylTtreyee9C44de+UBG0g=
+github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
+github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1 h1:ycpNCSYwzZ7x4G4ioPNtKQmIY0G/3o4pVf8wCZq6blY=
+github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98 h1:RNpJrXfI5u6e+uzyIzvmnXbhmhdRkVf//90sMBH3lso=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
+github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
+github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
+github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
+github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
+github.com/u-root/u-root v0.12.0 h1:K0AuBFriwr0w/PGS3HawiAw89e3+MU7ks80GpghAsNs=
+github.com/u-root/u-root v0.12.0/go.mod h1:FYjTOh4IkIZHhjsd17lb8nYW6udgXdJhG1c0r6u0arI=
+github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e h1:BA9O3BmlTmpjbvajAwzWx4Wo2TRVdpPXZEeemGQcajw=
+github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
+github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
+github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
+github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go4.org/mem v0.0.0-20220726221520-4f986261bf13 h1:CbZeCBZ0aZj8EfVgnqQcYZgf0lpZ3H9rmp5nkDTAst8=
+go4.org/mem v0.0.0-20220726221520-4f986261bf13/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
+golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
+golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3 h1:/8/t5pz/mgdRXhYOIeqqYhFAQLE4DDGegc0Y4ZjyFJM=
+gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
+k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
+k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
+k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=
+k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U=
+nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=
+nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k=
+software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=

gokrazy/natlabapp/config.json

@@ -0,0 +1,27 @@
+{
+    "Hostname": "natlabapp",
+    "Update": {
+        "NoPassword": true
+    },
+    "SerialConsole": "ttyS0,115200",
+    "GokrazyPackages": [
+        "github.com/gokrazy/gokrazy/cmd/dhcp"
+    ],
+    "Packages": [
+        "github.com/gokrazy/serial-busybox",
+        "tailscale.com/cmd/tailscale",
+        "tailscale.com/cmd/tailscaled",
+        "tailscale.com/cmd/tta"
+    ],
+    "PackageConfig": {
+        "tailscale.com/cmd/tailscale": {
+            "ExtraFilePaths": {
+                "/usr": "usr-dir"
+            }
+        }
+    },
+    "KernelPackage": "github.com/tailscale/gokrazy-kernel",
+    "FirmwarePackage": "",
+    "EEPROMPackage": "",
+    "InternalCompatibilityFlags": {}
+}

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/cmd/heartbeat/go.mod

@@ -1,5 +0,0 @@
-module gokrazy/build/tsapp
-
-go 1.22.2
-
-require github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/cmd/randomd/go.mod

@@ -1,5 +0,0 @@
-module gokrazy/build/tsapp
-
-go 1.22.2
-
-require github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/cmd/randomd/go.sum

@@ -1,4 +0,0 @@
-github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZD931Lh75mdJfzEEXrL+Dvi97Ck3A=
-github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.mod

@@ -2,6 +2,14 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 // indirect
+require (
+	github.com/gokrazy/gokrazy v0.0.0-20240802144848-676865a4e84f // indirect
+	github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 // indirect
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/kenshaw/evdev v0.1.0 // indirect
+	github.com/mdlayher/watchdog v0.0.0-20201005150459-8bdc4f41966b // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
 
-replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a
+replace github.com/gokrazy/gokrazy => github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678

gokrazy/tsapp/builddir/github.com/gokrazy/gokrazy/go.sum

@@ -2,6 +2,8 @@ github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803 h1:gdGRW/wXHPJuZgZ
 github.com/gokrazy/gokrazy v0.0.0-20240525065858-dedadaf38803/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a h1:FKeN678rNpKTpWRdFbAhYL9mWzPu57R5XPXCR3WmXdI=
 github.com/gokrazy/internal v0.0.0-20240510165500-68dd68393b7a/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5 h1:XDklMxV0pE5jWiNaoo5TzvWfqdoiRRScmr4ZtDzE4Uw=
+github.com/gokrazy/internal v0.0.0-20240629150625-a0f1dee26ef5/go.mod h1:t3ZirVhcs9bH+fPAJuGh51rzT7sVCZ9yfXvszf0ZjF0=
 github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/kenshaw/evdev v0.1.0 h1:wmtceEOFfilChgdNT+c/djPJ2JineVsQ0N14kGzFRUo=
@@ -12,5 +14,10 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a h1:7dnA8x14JihQmKbPr++Y5CCN/XSyDmOB6cXUxcIj6VQ=
 github.com/tailscale/gokrazy v0.0.0-20240602215456-7b9b6bbf726a/go.mod h1:NHROeDlzn0icUl3f+tEYvGGpcyBDMsr3AvKLHOWRe5M=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f h1:ZSAGWpgs+6dK2oIz5OR+HUul3oJbnhFn8YNgcZ3d9SQ=
+github.com/tailscale/gokrazy v0.0.0-20240802144848-676865a4e84f/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678 h1:2B8/FbIRqmVgRUulQ4iu1EojniufComYe5Yj4BtIn1c=
+github.com/tailscale/gokrazy v0.0.0-20240812224643-6b21ddf64678/go.mod h1:+/WWMckeuQt+DG6690A6H8IgC+HpBFq2fmwRKcSbxdk=
+golang.org/x/sys v0.0.0-20201005065044-765f4ea38db3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gokrazy/tsapp/builddir/github.com/gokrazy/mkfs/go.mod

@@ -1,5 +0,0 @@
-module gokrazy/build/tsapp
-
-go 1.22.2
-
-require github.com/gokrazy/mkfs v0.0.0-20230114091253-b6755f9e9632 // indirect

gokrazy/tsapp/builddir/github.com/gokrazy/mkfs/go.sum

@@ -1,4 +0,0 @@
-github.com/gokrazy/internal v0.0.0-20210621162516-1b3b5687a06d h1:qk95CKJfxvU5oi3lbrVkEgID5ak1pOjTyPTdaXs6Q9E=
-github.com/gokrazy/internal v0.0.0-20210621162516-1b3b5687a06d/go.mod h1:Gqv1x1DNrObmBvVvblpZbvZizZ0dU5PwiwYHipmtY9Y=
-github.com/gokrazy/mkfs v0.0.0-20230114091253-b6755f9e9632 h1:Vt3rJdB4p56yjK4CKhb/awHT6Qj7LoC3CwPoOaiNS6k=
-github.com/gokrazy/mkfs v0.0.0-20230114091253-b6755f9e9632/go.mod h1:O2w1ipGvg78u3F61FnLp36Db3MsUbdy8E/ciG64jbGY=

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.mod

@@ -2,4 +2,4 @@ module gokrazy/build/tsapp
 
 go 1.22.2
 
-require github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 // indirect
+require github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e // indirect

gokrazy/tsapp/builddir/github.com/tailscale/gokrazy-kernel/go.sum

@@ -1,2 +1,4 @@
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2 h1:xzf+cMvBJBcA/Av7OTWBa0Tjrbfcy00TeatJeJt6zrY=
 github.com/tailscale/gokrazy-kernel v0.0.0-20240530042707-3f95c886bcf2/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e h1:tyUUgeRPGHjCZWycRnhdx8Lx9DRkjl3WsVUxYMrVBOw=
+github.com/tailscale/gokrazy-kernel v0.0.0-20240728225134-3d23beabda2e/go.mod h1:7Mth+m9bq2IHusSsexMNyupHWPL8RxwOuSvBlSGtgDY=

gokrazy/tsapp/builddir/tailscale.com/go.sum

@@ -122,8 +122,12 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1 h1:ycpNCSYwzZ7x4G4ioPNtKQmIY0G/3o4pVf8wCZq6blY=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98 h1:RNpJrXfI5u6e+uzyIzvmnXbhmhdRkVf//90sMBH3lso=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -170,6 +174,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3 h1:/8/t5pz/mgdRXhYOIeqqYhFAQLE4DDGegc0Y4ZjyFJM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=

gokrazy/tsapp/config.json

@@ -1,7 +1,14 @@
 {
     "Hostname": "tsapp",
-    "Update": { "NoPassword": true },
+    "Update": {
+        "NoPassword": true
+    },
     "SerialConsole": "ttyS0,115200",
+    "GokrazyPackages": [
+        "github.com/gokrazy/gokrazy/cmd/dhcp",
+        "github.com/gokrazy/gokrazy/cmd/randomd",
+        "github.com/gokrazy/gokrazy/cmd/ntp"
+    ],
     "Packages": [
         "github.com/gokrazy/serial-busybox",
         "github.com/gokrazy/breakglass",
@@ -10,7 +17,9 @@
     ],
     "PackageConfig": {
         "github.com/gokrazy/breakglass": {
-            "CommandLineFlags": [ "-authorized_keys=ec2" ]
+            "CommandLineFlags": [
+                "-authorized_keys=ec2"
+            ]
         },
         "tailscale.com/cmd/tailscale": {
             "ExtraFilePaths": {

health/health.go

@@ -65,6 +65,11 @@ type Tracker struct {
 	// magicsock receive functions: IPv4, IPv6, and DERP.
 	MagicSockReceiveFuncs [3]ReceiveFuncStats // indexed by ReceiveFunc values
 
+	// initOnce guards the initialization of the Tracker.
+	// Notably, it initializes the MagicSockReceiveFuncs names.
+	// mu should not be held during init.
+	initOnce sync.Once
+
 	// mu guards everything that follows.
 	mu sync.Mutex
 
@@ -433,6 +438,7 @@ func (t *Tracker) RegisterWatcher(cb func(w *Warnable, r *UnhealthyState)) (unre
 	if t.nil() {
 		return func() {}
 	}
+	t.initOnce.Do(t.doOnceInit)
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.watchers == nil {
@@ -859,6 +865,7 @@ func (t *Tracker) timerSelfCheck() {
 	if t.nil() {
 		return
 	}
+	t.initOnce.Do(t.doOnceInit)
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	t.checkReceiveFuncsLocked()
@@ -1168,6 +1175,11 @@ type ReceiveFuncStats struct {
 	missing bool
 }
 
+// Name returns the name of the receive func ("ReceiveIPv4", "ReceiveIPv6", etc).
+func (s *ReceiveFuncStats) Name() string {
+	return s.name
+}
+
 func (s *ReceiveFuncStats) Enter() {
 	s.numCalls.Add(1)
 	s.inCall.Store(true)
@@ -1185,15 +1197,20 @@ func (t *Tracker) ReceiveFuncStats(which ReceiveFunc) *ReceiveFuncStats {
 	if t == nil {
 		return nil
 	}
+	t.initOnce.Do(t.doOnceInit)
 	return &t.MagicSockReceiveFuncs[which]
 }
 
+func (t *Tracker) doOnceInit() {
+	for i := range t.MagicSockReceiveFuncs {
+		f := &t.MagicSockReceiveFuncs[i]
+		f.name = (ReceiveFunc(i)).String()
+	}
+}
+
 func (t *Tracker) checkReceiveFuncsLocked() {
 	for i := range t.MagicSockReceiveFuncs {
 		f := &t.MagicSockReceiveFuncs[i]
-		if f.name == "" {
-			f.name = (ReceiveFunc(i)).String()
-		}
 		if runtime.GOOS == "js" && i < 2 {
 			// Skip IPv4 and IPv6 on js.
 			continue

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/lineread"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 )
 
 var started = time.Now()
@@ -462,3 +463,15 @@ func IsSELinuxEnforcing() bool {
 	out, _ := exec.Command("getenforce").Output()
 	return string(bytes.TrimSpace(out)) == "Enforcing"
 }
+
+// IsNATLabGuestVM reports whether the current host is a NAT Lab guest VM.
+func IsNATLabGuestVM() bool {
+	if runtime.GOOS == "linux" && distro.Get() == distro.Gokrazy {
+		cmdLine, _ := os.ReadFile("/proc/cmdline")
+		return bytes.Contains(cmdLine, []byte("tailscale-tta=1"))
+	}
+	return false
+}
+
+// NAT Lab VMs have a unique MAC address prefix.
+// See

ipn/ipnlocal/local.go

@@ -1868,6 +1868,14 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		opts.AuthKey = v
 	}
 
+	if b.state != ipn.Running && b.conf == nil && opts.AuthKey == "" {
+		sysak, _ := syspolicy.GetString(syspolicy.AuthKey, "")
+		if sysak != "" {
+			b.logf("Start: setting opts.AuthKey by syspolicy, len=%v", len(sysak))
+			opts.AuthKey = strings.TrimSpace(sysak)
+		}
+	}
+
 	hostinfo := hostinfo.New()
 	applyConfigToHostinfo(hostinfo, b.conf)
 	hostinfo.BackendLogID = b.backendLogID.String()
@@ -3781,7 +3789,7 @@ func (b *LocalBackend) TCPHandlerForDst(src, dst netip.AddrPort) (handler func(c
 			return nil
 		}, opts
 	}
-	if handler := b.tcpHandlerForServe(dst.Port(), src); handler != nil {
+	if handler := b.tcpHandlerForServe(dst.Port(), src, nil); handler != nil {
 		return handler, opts
 	}
 	return nil, nil

ipn/ipnlocal/network-lock.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
+	"tailscale.com/tsconst"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -52,7 +53,7 @@ type tkaState struct {
 	profile   ipn.ProfileID
 	authority *tka.Authority
 	storage   *tka.FS
-	filtered  []ipnstate.TKAFilteredPeer
+	filtered  []ipnstate.TKAPeer
 }
 
 // tkaFilterNetmapLocked checks the signatures on each node key, dropping
@@ -98,7 +99,7 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 	// nm.Peers is ordered, so deletion must be order-preserving.
 	if len(toDelete) > 0 || len(obsoleteByRotation) > 0 {
 		peers := make([]tailcfg.NodeView, 0, len(nm.Peers))
-		filtered := make([]ipnstate.TKAFilteredPeer, 0, len(toDelete)+len(obsoleteByRotation))
+		filtered := make([]ipnstate.TKAPeer, 0, len(toDelete)+len(obsoleteByRotation))
 		for i, p := range nm.Peers {
 			if !toDelete[i] && !obsoleteByRotation.Contains(p.Key()) {
 				peers = append(peers, p)
@@ -107,20 +108,7 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 					b.logf("Network lock is dropping peer %v(%v) due to key rotation", p.ID(), p.StableID())
 				}
 				// Record information about the node we filtered out.
-				fp := ipnstate.TKAFilteredPeer{
-					Name:         p.Name(),
-					ID:           p.ID(),
-					StableID:     p.StableID(),
-					TailscaleIPs: make([]netip.Addr, p.Addresses().Len()),
-					NodeKey:      p.Key(),
-				}
-				for i := range p.Addresses().Len() {
-					addr := p.Addresses().At(i)
-					if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.Addr()) {
-						fp.TailscaleIPs[i] = addr.Addr()
-					}
-				}
-				filtered = append(filtered, fp)
+				filtered = append(filtered, tkaStateFromPeer(p))
 			}
 		}
 		nm.Peers = peers
@@ -254,7 +242,10 @@ func (b *LocalBackend) tkaSyncIfNeeded(nm *netmap.NetworkMap, prefs ipn.PrefsVie
 		b.logf("tkaSyncIfNeeded: enabled=%v, head=%v", nm.TKAEnabled, nm.TKAHead)
 	}
 
-	ourNodeKey := prefs.Persist().PublicNodeKey()
+	ourNodeKey, ok := prefs.Persist().PublicNodeKeyOK()
+	if !ok {
+		return errors.New("tkaSyncIfNeeded: no node key in prefs")
+	}
 
 	isEnabled := b.tka != nil
 	wantEnabled := nm.TKAEnabled
@@ -542,11 +533,17 @@ func (b *LocalBackend) NetworkLockStatus() *ipnstate.NetworkLockStatus {
 		}
 	}
 
-	filtered := make([]*ipnstate.TKAFilteredPeer, len(b.tka.filtered))
+	filtered := make([]*ipnstate.TKAPeer, len(b.tka.filtered))
 	for i := range len(filtered) {
 		filtered[i] = b.tka.filtered[i].Clone()
 	}
 
+	visible := make([]*ipnstate.TKAPeer, len(b.netMap.Peers))
+	for i, p := range b.netMap.Peers {
+		s := tkaStateFromPeer(p)
+		visible[i] = &s
+	}
+
 	stateID1, _ := b.tka.authority.StateIDs()
 
 	return &ipnstate.NetworkLockStatus{
@@ -558,10 +555,32 @@ func (b *LocalBackend) NetworkLockStatus() *ipnstate.NetworkLockStatus {
 		NodeKeySignature: nodeKeySignature,
 		TrustedKeys:      outKeys,
 		FilteredPeers:    filtered,
+		VisiblePeers:     visible,
 		StateID:          stateID1,
 	}
 }
 
+func tkaStateFromPeer(p tailcfg.NodeView) ipnstate.TKAPeer {
+	fp := ipnstate.TKAPeer{
+		Name:         p.Name(),
+		ID:           p.ID(),
+		StableID:     p.StableID(),
+		TailscaleIPs: make([]netip.Addr, 0, p.Addresses().Len()),
+		NodeKey:      p.Key(),
+	}
+	for i := range p.Addresses().Len() {
+		addr := p.Addresses().At(i)
+		if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.Addr()) {
+			fp.TailscaleIPs = append(fp.TailscaleIPs, addr.Addr())
+		}
+	}
+	var decoded tka.NodeKeySignature
+	if err := decoded.Unserialize(p.KeySignature().AsSlice()); err == nil {
+		fp.NodeKeySignature = decoded
+	}
+	return fp
+}
+
 // NetworkLockInit enables network-lock for the tailnet, with the tailnets'
 // key authority initialized to trust the provided keys.
 //
@@ -716,7 +735,7 @@ func (b *LocalBackend) NetworkLockSign(nodeKey key.NodePublic, rotationPublic []
 			return key.NodePublic{}, tka.NodeKeySignature{}, errNetworkLockNotActive
 		}
 		if !b.tka.authority.KeyTrusted(nlPriv.KeyID()) {
-			return key.NodePublic{}, tka.NodeKeySignature{}, errors.New("this node is not trusted by network lock")
+			return key.NodePublic{}, tka.NodeKeySignature{}, errors.New(tsconst.TailnetLockNotTrustedMsg)
 		}
 
 		p, err := nodeKey.MarshalBinary()

ipn/ipnlocal/serve.go

@@ -56,6 +56,16 @@ var serveHTTPContextKey ctxkey.Key[*serveHTTPContext]
 type serveHTTPContext struct {
 	SrcAddr  netip.AddrPort
 	DestPort uint16
+
+	// provides funnel-specific context, nil if not funneled
+	Funnel *funnelFlow
+}
+
+// funnelFlow represents a funneled connection initiated via IngressPeer
+// to Host.
+type funnelFlow struct {
+	Host        string
+	IngressPeer tailcfg.NodeView
 }
 
 // localListener is the state of host-level net.Listen for a specific (Tailscale IP, port)
@@ -91,7 +101,7 @@ func (b *LocalBackend) newServeListener(ctx context.Context, ap netip.AddrPort,
 
 		handler: func(conn net.Conn) error {
 			srcAddr := conn.RemoteAddr().(*net.TCPAddr).AddrPort()
-			handler := b.tcpHandlerForServe(ap.Port(), srcAddr)
+			handler := b.tcpHandlerForServe(ap.Port(), srcAddr, nil)
 			if handler == nil {
 				b.logf("[unexpected] local-serve: no handler for %v to port %v", srcAddr, ap.Port())
 				conn.Close()
@@ -382,7 +392,7 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 		return
 	}
 
-	_, port, err := net.SplitHostPort(string(target))
+	host, port, err := net.SplitHostPort(string(target))
 	if err != nil {
 		logf("got ingress conn for bad target %q; rejecting", target)
 		sendRST()
@@ -407,9 +417,10 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 			return
 		}
 	}
-	// TODO(bradfitz): pass ingressPeer etc in context to tcpHandlerForServe,
-	// extend serveHTTPContext or similar.
-	handler := b.tcpHandlerForServe(dport, srcAddr)
+	handler := b.tcpHandlerForServe(dport, srcAddr, &funnelFlow{
+		Host:        host,
+		IngressPeer: ingressPeer,
+	})
 	if handler == nil {
 		logf("[unexpected] no matching ingress serve handler for %v to port %v", srcAddr, dport)
 		sendRST()
@@ -424,8 +435,9 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 }
 
 // tcpHandlerForServe returns a handler for a TCP connection to be served via
-// the ipn.ServeConfig.
-func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort) (handler func(net.Conn) error) {
+// the ipn.ServeConfig. The funnelFlow can be nil if this is not a funneled
+// connection.
+func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort, f *funnelFlow) (handler func(net.Conn) error) {
 	b.mu.Lock()
 	sc := b.serveConfig
 	b.mu.Unlock()
@@ -444,6 +456,7 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort)
 			Handler: http.HandlerFunc(b.serveWebHandler),
 			BaseContext: func(_ net.Listener) context.Context {
 				return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{
+					Funnel:   f,
 					SrcAddr:  srcAddr,
 					DestPort: dport,
 				})
@@ -712,15 +725,20 @@ func (b *LocalBackend) addTailscaleIdentityHeaders(r *httputil.ProxyRequest) {
 	r.Out.Header.Del("Tailscale-User-Login")
 	r.Out.Header.Del("Tailscale-User-Name")
 	r.Out.Header.Del("Tailscale-User-Profile-Pic")
+	r.Out.Header.Del("Tailscale-Funnel-Request")
 	r.Out.Header.Del("Tailscale-Headers-Info")
 
 	c, ok := serveHTTPContextKey.ValueOk(r.Out.Context())
 	if !ok {
 		return
 	}
+	if c.Funnel != nil {
+		r.Out.Header.Set("Tailscale-Funnel-Request", "?1")
+		return
+	}
 	node, user, ok := b.WhoIs("tcp", c.SrcAddr)
 	if !ok {
-		return // traffic from outside of Tailnet (funneled)
+		return // traffic from outside of Tailnet (funneled or local machine)
 	}
 	if node.IsTagged() {
 		// 2023-06-14: Not setting identity headers for tagged nodes.

ipn/ipnstate/ipnstate.go

@@ -26,7 +26,7 @@ import (
 	"tailscale.com/version"
 )
 
-//go:generate go run tailscale.com/cmd/cloner  -clonefunc=false -type=TKAFilteredPeer
+//go:generate go run tailscale.com/cmd/cloner  -clonefunc=false -type=TKAPeer
 
 // Status represents the entire state of the IPN network.
 type Status struct {
@@ -94,15 +94,14 @@ type TKAKey struct {
 	Votes    uint
 }
 
-// TKAFilteredPeer describes a peer which was removed from the netmap
-// (i.e. no connectivity) because it failed tailnet lock
-// checks.
-type TKAFilteredPeer struct {
-	Name         string // DNS
-	ID           tailcfg.NodeID
-	StableID     tailcfg.StableNodeID
-	TailscaleIPs []netip.Addr // Tailscale IP(s) assigned to this node
-	NodeKey      key.NodePublic
+// TKAPeer describes a peer and its network lock details.
+type TKAPeer struct {
+	Name             string // DNS
+	ID               tailcfg.NodeID
+	StableID         tailcfg.StableNodeID
+	TailscaleIPs     []netip.Addr // Tailscale IP(s) assigned to this node
+	NodeKey          key.NodePublic
+	NodeKeySignature tka.NodeKeySignature
 }
 
 // NetworkLockStatus represents whether network-lock is enabled,
@@ -134,10 +133,14 @@ type NetworkLockStatus struct {
 	// to network-lock.
 	TrustedKeys []TKAKey
 
+	// VisiblePeers describes peers which are visible in the netmap that
+	// have valid Tailnet Lock signatures signatures.
+	VisiblePeers []*TKAPeer
+
 	// FilteredPeers describes peers which were removed from the netmap
 	// (i.e. no connectivity) because they failed tailnet lock
 	// checks.
-	FilteredPeers []*TKAFilteredPeer
+	FilteredPeers []*TKAPeer
 
 	// StateID is a nonce associated with the network lock authority,
 	// generated upon enablement. This field is not populated if the

ipn/ipnstate/ipnstate_clone.go

@@ -9,26 +9,29 @@ import (
 	"net/netip"
 
 	"tailscale.com/tailcfg"
+	"tailscale.com/tka"
 	"tailscale.com/types/key"
 )
 
-// Clone makes a deep copy of TKAFilteredPeer.
+// Clone makes a deep copy of TKAPeer.
 // The result aliases no memory with the original.
-func (src *TKAFilteredPeer) Clone() *TKAFilteredPeer {
+func (src *TKAPeer) Clone() *TKAPeer {
 	if src == nil {
 		return nil
 	}
-	dst := new(TKAFilteredPeer)
+	dst := new(TKAPeer)
 	*dst = *src
 	dst.TailscaleIPs = append(src.TailscaleIPs[:0:0], src.TailscaleIPs...)
+	dst.NodeKeySignature = *src.NodeKeySignature.Clone()
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
-var _TKAFilteredPeerCloneNeedsRegeneration = TKAFilteredPeer(struct {
-	Name         string
-	ID           tailcfg.NodeID
-	StableID     tailcfg.StableNodeID
-	TailscaleIPs []netip.Addr
-	NodeKey      key.NodePublic
+var _TKAPeerCloneNeedsRegeneration = TKAPeer(struct {
+	Name             string
+	ID               tailcfg.NodeID
+	StableID         tailcfg.StableNodeID
+	TailscaleIPs     []netip.Addr
+	NodeKey          key.NodePublic
+	NodeKeySignature tka.NodeKeySignature
 }{})

ipn/localapi/localapi.go

@@ -1752,10 +1752,17 @@ func (ww *multiFilePostResponseWriter) Write(p []byte) (int, error) {
 }
 
 func (ww *multiFilePostResponseWriter) Flush(w http.ResponseWriter) error {
-	maps.Copy(w.Header(), ww.Header())
-	w.WriteHeader(ww.statusCode)
-	_, err := io.Copy(w, ww.body)
-	return err
+	if ww.header != nil {
+		maps.Copy(w.Header(), ww.header)
+	}
+	if ww.statusCode > 0 {
+		w.WriteHeader(ww.statusCode)
+	}
+	if ww.body != nil {
+		_, err := io.Copy(w, ww.body)
+		return err
+	}
+	return nil
 }
 
 func (h *Handler) singleFilePut(

k8s-operator/sessionrecording/conn/conn.go

@@ -1,20 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-// Package conn contains shared interface for the hijacked
-// connection of a 'kubectl exec' session that is being recorded.
-package conn
-
-import "net"
-
-type Conn interface {
-	net.Conn
-	// Fail can be called to set connection state to failed. By default any
-	// bytes left over in write buffer are forwarded to the intended
-	// destination when the connection is  being closed except for when the
-	// connection state is failed- so set the state to failed when erroring
-	// out and failure policy is to fail closed.
-	Fail()
-}

k8s-operator/sessionrecording/fakes/fakes.go

@@ -13,6 +13,9 @@ import (
 	"net"
 	"sync"
 	"testing"
+	"time"
+
+	"math/rand"
 
 	"tailscale.com/sessionrecording"
 	"tailscale.com/tstime"
@@ -116,3 +119,20 @@ func AsciinemaResizeMsg(t *testing.T, width, height int) []byte {
 	}
 	return append(bs, '\n')
 }
+
+func RandomBytes(t *testing.T) [][]byte {
+	t.Helper()
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	n := r.Intn(4096)
+	b := make([]byte, n)
+	t.Logf("RandomBytes: generating byte slice of length %d", n)
+	_, err := r.Read(b)
+	if err != nil {
+		t.Fatalf("error generating random byte slice: %v", err)
+	}
+	if len(b) < 2 {
+		return [][]byte{b}
+	}
+	split := r.Intn(len(b) - 1)
+	return [][]byte{b[:split], b[split:]}
+}

k8s-operator/sessionrecording/hijacker.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/k8s-operator/sessionrecording/spdy"
 	"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
+	"tailscale.com/k8s-operator/sessionrecording/ws"
 	"tailscale.com/sessionrecording"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -31,11 +32,14 @@ import (
 	"tailscale.com/util/multierr"
 )
 
-const SPDYProtocol protocol = "SPDY"
+const (
+	SPDYProtocol Protocol = "SPDY"
+	WSProtocol   Protocol = "WebSocket"
+)
 
-// protocol is the streaming protocol of the hijacked session. Supported
-// protocols are SPDY.
-type protocol string
+// Protocol is the streaming protocol of the hijacked session. Supported
+// protocols are SPDY and WebSocket.
+type Protocol string
 
 var (
 	// CounterSessionRecordingsAttempted counts the number of session recording attempts.
@@ -45,22 +49,35 @@ var (
 	counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded")
 )
 
-func New(ts *tsnet.Server, req *http.Request, who *apitype.WhoIsResponse, w http.ResponseWriter, pod, ns string, proto protocol, addrs []netip.AddrPort, failOpen bool, connFunc RecorderDialFn, log *zap.SugaredLogger) *Hijacker {
+func New(opts HijackerOpts) *Hijacker {
 	return &Hijacker{
-		ts:                ts,
-		req:               req,
-		who:               who,
-		ResponseWriter:    w,
-		pod:               pod,
-		ns:                ns,
-		addrs:             addrs,
-		failOpen:          failOpen,
-		connectToRecorder: connFunc,
-		proto:             proto,
-		log:               log,
+		ts:                opts.TS,
+		req:               opts.Req,
+		who:               opts.Who,
+		ResponseWriter:    opts.W,
+		pod:               opts.Pod,
+		ns:                opts.Namespace,
+		addrs:             opts.Addrs,
+		failOpen:          opts.FailOpen,
+		proto:             opts.Proto,
+		log:               opts.Log,
+		connectToRecorder: sessionrecording.ConnectToRecorder,
 	}
 }
 
+type HijackerOpts struct {
+	TS        *tsnet.Server
+	Req       *http.Request
+	W         http.ResponseWriter
+	Who       *apitype.WhoIsResponse
+	Addrs     []netip.AddrPort
+	Log       *zap.SugaredLogger
+	Pod       string
+	Namespace string
+	FailOpen  bool
+	Proto     Protocol
+}
+
 // Hijacker implements [net/http.Hijacker] interface.
 // It must be configured with an http request for a 'kubectl exec' session that
 // needs to be recorded. It knows how to hijack the connection and configure for
@@ -76,7 +93,7 @@ type Hijacker struct {
 	addrs             []netip.AddrPort // tsrecorder addresses
 	failOpen          bool             // whether to fail open if recording fails
 	connectToRecorder RecorderDialFn
-	proto             protocol // streaming protocol
+	proto             Protocol // streaming protocol
 }
 
 // RecorderDialFn dials the specified netip.AddrPorts that should be tsrecorder
@@ -111,10 +128,14 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 		// https://docs.asciinema.org/manual/asciicast/v2/
 		asciicastv2 = 2
 	)
-	var wc io.WriteCloser
+	var (
+		wc      io.WriteCloser
+		err     error
+		errChan <-chan error
+	)
 	h.log.Infof("kubectl exec session will be recorded, recorders: %v, fail open policy: %t", h.addrs, h.failOpen)
 	// TODO (irbekrm): send client a message that session will be recorded.
-	rw, _, errChan, err := h.connectToRecorder(ctx, h.addrs, h.ts.Dial)
+	wc, _, errChan, err = h.connectToRecorder(ctx, h.addrs, h.ts.Dial)
 	if err != nil {
 		msg := fmt.Sprintf("error connecting to session recorders: %v", err)
 		if h.failOpen {
@@ -131,7 +152,6 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 
 	// TODO (irbekrm): log which recorder
 	h.log.Info("successfully connected to a session recorder")
-	wc = rw
 	cl := tstime.DefaultClock{}
 	rec := tsrecorder.New(wc, cl, cl.Now(), h.failOpen)
 	qp := h.req.URL.Query()
@@ -153,7 +173,17 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 	} else {
 		ch.SrcNodeTags = h.who.Node.Tags
 	}
-	lc := spdy.New(conn, rec, ch, h.log)
+
+	var lc net.Conn
+	switch h.proto {
+	case SPDYProtocol:
+		lc = spdy.New(conn, rec, ch, h.log)
+	case WSProtocol:
+		lc = ws.New(conn, rec, ch, h.log)
+	default:
+		return nil, fmt.Errorf("unknown protocol: %s", h.proto)
+	}
+
 	go func() {
 		var err error
 		select {
@@ -174,7 +204,6 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 		}
 		msg += "; failure mode set to 'fail closed'; closing connection"
 		h.log.Error(msg)
-		lc.Fail()
 		// TODO (irbekrm): write a message to the client
 		if err := lc.Close(); err != nil {
 			h.log.Infof("error closing recorder connections: %v", err)

k8s-operator/sessionrecording/hijacker_test.go

@@ -37,30 +37,40 @@ func Test_Hijacker(t *testing.T) {
 		failRecorderConnPostConnect bool // send error down the error channel
 		wantsConnClosed             bool
 		wantsSetupErr               bool
+		proto                       Protocol
 	}{
 		{
-			name: "setup succeeds, conn stays open",
+			name:  "setup_succeeds_conn_stays_open",
+			proto: SPDYProtocol,
 		},
 		{
-			name:                "setup fails, policy is to fail open, conn stays open",
+			name:  "setup_succeeds_conn_stays_open_ws",
+			proto: WSProtocol,
+		},
+		{
+			name:                "setup_fails_policy_is_to_fail_open_conn_stays_open",
 			failOpen:            true,
 			failRecorderConnect: true,
+			proto:               SPDYProtocol,
 		},
 		{
-			name:                "setup fails, policy is to fail closed, conn is closed",
+			name:                "setup_fails_policy_is_to_fail_closed_conn_is_closed",
 			failRecorderConnect: true,
 			wantsSetupErr:       true,
 			wantsConnClosed:     true,
+			proto:               SPDYProtocol,
 		},
 		{
-			name:                        "connection fails post-initial connect, policy is to fail open, conn stays open",
+			name:                        "connection_fails_post-initial_connect_policy_is_to_fail_open_conn_stays_open",
 			failRecorderConnPostConnect: true,
 			failOpen:                    true,
+			proto:                       SPDYProtocol,
 		},
 		{
-			name:                        "connection fails post-initial connect, policy is to fail closed, conn is closed",
+			name:                        "connection_fails_post-initial_connect,_policy_is_to_fail_closed_conn_is_closed",
 			failRecorderConnPostConnect: true,
 			wantsConnClosed:             true,
+			proto:                       SPDYProtocol,
 		},
 	}
 	for _, tt := range tests {
@@ -79,6 +89,7 @@ func Test_Hijacker(t *testing.T) {
 				log:      zl.Sugar(),
 				ts:       &tsnet.Server{},
 				req:      &http.Request{URL: &url.URL{}},
+				proto:    tt.proto,
 			}
 			ctx := context.Background()
 			_, err := h.setUpRecording(ctx, tc)

k8s-operator/sessionrecording/spdy/conn.go

@@ -19,12 +19,18 @@ import (
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
-	srconn "tailscale.com/k8s-operator/sessionrecording/conn"
 	"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
 	"tailscale.com/sessionrecording"
 )
 
-func New(nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, log *zap.SugaredLogger) srconn.Conn {
+// New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection.
+// The connection must be a hijacked connection for a 'kubectl exec' session using SPDY.
+// The hijacked connection is used to transmit SPDY streams between Kubernetes client ('kubectl') and the destination container.
+// Data read from the underlying network connection is data sent via one of the SPDY streams from the client to the container.
+// Data written to the underlying connection is data sent from the container to the client.
+// We parse the data and send everything for the STDOUT/STDERR streams to the configured tsrecorder as an asciinema recording with the provided header.
+// https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#background-remotecommand-subprotocol
+func New(nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, log *zap.SugaredLogger) net.Conn {
 	return &conn{
 		Conn: nc,
 		rec:  rec,
@@ -49,7 +55,6 @@ type conn struct {
 
 	wmu    sync.Mutex // sequences writes
 	closed bool
-	failed bool
 
 	rmu                 sync.Mutex // sequences reads
 	writeCastHeaderOnce sync.Once
@@ -172,9 +177,6 @@ func (c *conn) Close() error {
 	if c.closed {
 		return nil
 	}
-	if !c.failed && c.writeBuf.Len() > 0 {
-		c.Conn.Write(c.writeBuf.Bytes())
-	}
 	c.writeBuf.Reset()
 	c.closed = true
 	err := c.Conn.Close()
@@ -182,14 +184,8 @@ func (c *conn) Close() error {
 	return err
 }
 
-func (s *conn) Fail() {
-	s.wmu.Lock()
-	s.failed = true
-	s.wmu.Unlock()
-}
-
 // storeStreamID parses SYN_STREAM SPDY control frame and updates
-// spdyRemoteConnRecorder to store the newly created stream's ID if it is one of
+// conn to store the newly created stream's ID if it is one of
 // the stream types we care about. Storing stream_id:stream_type mapping allows
 // us to parse received data frames (that have stream IDs) differently depening
 // on which stream they belong to (i.e send data frame payload for stdout stream

k8s-operator/sessionrecording/spdy/conn_test.go

@@ -7,6 +7,7 @@ package spdy
 
 import (
 	"encoding/json"
+	"fmt"
 	"reflect"
 	"testing"
 
@@ -234,6 +235,57 @@ func Test_Reads(t *testing.T) {
 	}
 }
 
+// Test_conn_ReadRand tests reading arbitrarily generated byte slices from conn to
+// test that we don't panic when parsing input from a broken or malicious
+// client.
+func Test_conn_ReadRand(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatalf("error creating a test logger: %v", err)
+	}
+	for i := range 1000 {
+		tc := &fakes.TestConn{}
+		tc.ResetReadBuf()
+		c := &conn{
+			Conn: tc,
+			log:  zl.Sugar(),
+		}
+		bb := fakes.RandomBytes(t)
+		for j, input := range bb {
+			if err := tc.WriteReadBufBytes(input); err != nil {
+				t.Fatalf("[%d] writing bytes to test conn: %v", i, err)
+			}
+			f := func() {
+				c.Read(make([]byte, len(input)))
+			}
+			testPanic(t, f, fmt.Sprintf("[%d %d] Read panic parsing input of length %d", i, j, len(input)))
+		}
+	}
+}
+
+// Test_conn_WriteRand calls conn.Write with an arbitrary input to validate that
+// it does not panic.
+func Test_conn_WriteRand(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatalf("error creating a test logger: %v", err)
+	}
+	for i := range 100 {
+		tc := &fakes.TestConn{}
+		c := &conn{
+			Conn: tc,
+			log:  zl.Sugar(),
+		}
+		bb := fakes.RandomBytes(t)
+		for j, input := range bb {
+			f := func() {
+				c.Write(input)
+			}
+			testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d", i, j, len(input)))
+		}
+	}
+}
+
 func resizeMsgBytes(t *testing.T, width, height int) []byte {
 	t.Helper()
 	bs, err := json.Marshal(spdyResizeMsg{Width: width, Height: height})

k8s-operator/sessionrecording/spdy/frame_test.go

@@ -9,11 +9,15 @@ import (
 	"bytes"
 	"compress/zlib"
 	"encoding/binary"
+	"fmt"
 	"io"
 	"net/http"
 	"reflect"
 	"strings"
 	"testing"
+	"time"
+
+	"math/rand"
 
 	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
@@ -200,6 +204,29 @@ func Test_spdyFrame_parseHeaders(t *testing.T) {
 	}
 }
 
+// Test_spdyFrame_ParseRand calls spdyFrame.Parse with randomly generated bytes
+// to test that it doesn't panic.
+func Test_spdyFrame_ParseRand(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for i := range 100 {
+		n := r.Intn(4096)
+		b := make([]byte, n)
+		_, err := r.Read(b)
+		if err != nil {
+			t.Fatalf("error generating random byte slice: %v", err)
+		}
+		sf := &spdyFrame{}
+		f := func() {
+			sf.Parse(b, zl.Sugar())
+		}
+		testPanic(t, f, fmt.Sprintf("[%d] Parse panicked running with byte slice of length %d: %v", i, n, r))
+	}
+}
+
 // payload takes a control frame type and a map with 0 or more header keys and
 // values and returns a SPDY control frame payload with the header as SPDY zlib
 // compressed header name/value block. The payload is padded with arbitrary
@@ -291,3 +318,13 @@ func header(hs map[string]string) http.Header {
 	}
 	return h
 }
+
+func testPanic(t *testing.T, f func(), msg string) {
+	t.Helper()
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatal(msg, r)
+		}
+	}()
+	f()
+}

k8s-operator/sessionrecording/ws/conn.go

@@ -0,0 +1,301 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// package ws has functionality to parse 'kubectl exec' sessions streamed using
+// WebSocket protocol.
+package ws
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+
+	"go.uber.org/zap"
+	"k8s.io/apimachinery/pkg/util/remotecommand"
+	"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
+	"tailscale.com/sessionrecording"
+	"tailscale.com/util/multierr"
+)
+
+// New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection.
+// The connection must be a hijacked connection for a 'kubectl exec' session using WebSocket protocol and a *.channel.k8s.io subprotocol.
+// The hijacked connection is used to transmit *.channel.k8s.io streams between Kubernetes client ('kubectl') and the destination proxy controlled by Kubernetes.
+// Data read from the underlying network connection is data sent via one of the streams from the client to the container.
+// Data written to the underlying connection is data sent from the container to the client.
+// We parse the data and send everything for the STDOUT/STDERR streams to the configured tsrecorder as an asciinema recording with the provided header.
+// https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#proposal-new-remotecommand-sub-protocol-version---v5channelk8sio
+func New(c net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, log *zap.SugaredLogger) net.Conn {
+	return &conn{
+		Conn: c,
+		rec:  rec,
+		ch:   ch,
+		log:  log,
+	}
+}
+
+// conn is a wrapper around net.Conn. It reads the bytestream
+// for a 'kubectl exec' session, sends session recording data to the configured
+// recorder and forwards the raw bytes to the original destination.
+// A new conn is created per session.
+// conn only knows to how to read a 'kubectl exec' session that is streamed using WebSocket protocol.
+// https://www.rfc-editor.org/rfc/rfc6455
+type conn struct {
+	net.Conn
+	// rec knows how to send data to a tsrecorder instance.
+	rec *tsrecorder.Client
+	// ch is the asiinema CastHeader for a session.
+	ch  sessionrecording.CastHeader
+	log *zap.SugaredLogger
+
+	rmu sync.Mutex // sequences reads
+	// currentReadMsg contains parsed contents of a websocket binary data message that
+	// is currently being read from the underlying net.Conn.
+	currentReadMsg *message
+	// readBuf contains bytes for a currently parsed binary data message
+	// read from the underlying conn. If the message is masked, it is
+	// unmasked in place, so having this buffer allows us to avoid modifying
+	// the original byte array.
+	readBuf bytes.Buffer
+
+	wmu                 sync.Mutex // sequences writes
+	writeCastHeaderOnce sync.Once
+	closed              bool // connection is closed
+	// writeBuf contains bytes for a currently parsed binary data message
+	// being written to the underlying conn. If the message is masked, it is
+	// unmasked in place, so having this buffer allows us to avoid modifying
+	// the original byte array.
+	writeBuf bytes.Buffer
+	// currentWriteMsg contains parsed contents of a websocket binary data message that
+	// is currently being written to the underlying net.Conn.
+	currentWriteMsg *message
+}
+
+// Read reads bytes from the original connection and parses them as websocket
+// message fragments.
+// Bytes read from the original connection are the bytes sent from the Kubernetes client (kubectl) to the destination container via kubelet.
+
+// If the message is for the resize stream, sets the width
+// and height of the CastHeader for this connection.
+// The fragment can be incomplete.
+func (c *conn) Read(b []byte) (int, error) {
+	c.rmu.Lock()
+	defer c.rmu.Unlock()
+	n, err := c.Conn.Read(b)
+	if err != nil {
+		// It seems that we sometimes get a wrapped io.EOF, but the
+		// caller checks for io.EOF with ==.
+		if errors.Is(err, io.EOF) {
+			err = io.EOF
+		}
+		return 0, err
+	}
+	if n == 0 {
+		c.log.Debug("[unexpected] Read called for 0 length bytes")
+		return 0, nil
+	}
+
+	typ := messageType(opcode(b))
+	if (typ == noOpcode && c.readMsgIsIncomplete()) || c.readBufHasIncompleteFragment() { // subsequent fragment
+		if typ, err = c.curReadMsgType(); err != nil {
+			return 0, err
+		}
+	}
+
+	// A control message can not be fragmented and we are not interested in
+	// these messages. Just return.
+	if isControlMessage(typ) {
+		return n, nil
+	}
+
+	// The only data message type that Kubernetes supports is binary message.
+	// If we received another message type, return and let the API server close the connection.
+	// https://github.com/kubernetes/client-go/blob/release-1.30/tools/remotecommand/websocket.go#L281
+	if typ != binaryMessage {
+		c.log.Infof("[unexpected] received a data message with a type that is not binary message type %v", typ)
+		return n, nil
+	}
+
+	readMsg := &message{typ: typ} // start a new message...
+	// ... or pick up an already started one if the previous fragment was not final.
+	if c.readMsgIsIncomplete() || c.readBufHasIncompleteFragment() {
+		readMsg = c.currentReadMsg
+	}
+
+	if _, err := c.readBuf.Write(b[:n]); err != nil {
+		return 0, fmt.Errorf("[unexpected] error writing message contents to read buffer: %w", err)
+	}
+
+	ok, err := readMsg.Parse(c.readBuf.Bytes(), c.log)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing message: %v", err)
+	}
+	if !ok { // incomplete fragment
+		return n, nil
+	}
+	c.readBuf.Next(len(readMsg.raw))
+
+	if readMsg.isFinalized {
+		// Stream IDs for websocket streams are static.
+		// https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L218
+		if readMsg.streamID.Load() == remotecommand.StreamResize {
+			var err error
+			var msg tsrecorder.ResizeMsg
+			if err = json.Unmarshal(readMsg.payload, &msg); err != nil {
+				return 0, fmt.Errorf("error umarshalling resize message: %w", err)
+			}
+			c.ch.Width = msg.Width
+			c.ch.Height = msg.Height
+		}
+	}
+	c.currentReadMsg = readMsg
+	return n, nil
+}
+
+// Write parses the written bytes as WebSocket message fragment. If the message
+// is for stdout or stderr streams, it is written to the configured tsrecorder.
+// A message fragment can be incomplete.
+func (c *conn) Write(b []byte) (int, error) {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	if len(b) == 0 {
+		c.log.Debug("[unexpected] Write called with 0 bytes")
+		return 0, nil
+	}
+
+	typ := messageType(opcode(b))
+	// If we are in process of parsing a message fragment, the received
+	// bytes are not structured as a message fragment and can not be used to
+	// determine a message fragment.
+	if c.writeBufHasIncompleteFragment() { // buffer contains previous incomplete fragment
+		var err error
+		if typ, err = c.curWriteMsgType(); err != nil {
+			return 0, err
+		}
+	}
+
+	if isControlMessage(typ) {
+		return c.Conn.Write(b)
+	}
+
+	writeMsg := &message{typ: typ} // start a new message...
+	// ... or continue the existing one if it has not been finalized.
+	if c.writeMsgIsIncomplete() || c.writeBufHasIncompleteFragment() {
+		writeMsg = c.currentWriteMsg
+	}
+
+	if _, err := c.writeBuf.Write(b); err != nil {
+		c.log.Errorf("write: error writing to write buf: %v", err)
+		return 0, fmt.Errorf("[unexpected] error writing to internal write buffer: %w", err)
+	}
+
+	ok, err := writeMsg.Parse(c.writeBuf.Bytes(), c.log)
+	if err != nil {
+		c.log.Errorf("write: parsing a message errored: %v", err)
+		return 0, fmt.Errorf("write: error parsing message: %v", err)
+	}
+	c.currentWriteMsg = writeMsg
+	if !ok { // incomplete fragment
+		return len(b), nil
+	}
+	c.writeBuf.Next(len(writeMsg.raw)) // advance frame
+
+	if len(writeMsg.payload) != 0 && writeMsg.isFinalized {
+		if writeMsg.streamID.Load() == remotecommand.StreamStdOut || writeMsg.streamID.Load() == remotecommand.StreamStdErr {
+			var err error
+			c.writeCastHeaderOnce.Do(func() {
+				var j []byte
+				j, err = json.Marshal(c.ch)
+				if err != nil {
+					c.log.Errorf("error marhsalling conn: %v", err)
+					return
+				}
+				j = append(j, '\n')
+				err = c.rec.WriteCastLine(j)
+				if err != nil {
+					c.log.Errorf("received error from recorder: %v", err)
+				}
+			})
+			if err != nil {
+				return 0, fmt.Errorf("error writing CastHeader: %w", err)
+			}
+			if err := c.rec.Write(writeMsg.payload); err != nil {
+				return 0, fmt.Errorf("error writing message to recorder: %v", err)
+			}
+		}
+	}
+	_, err = c.Conn.Write(c.currentWriteMsg.raw)
+	if err != nil {
+		c.log.Errorf("write: error writing to conn: %v", err)
+	}
+	return len(b), nil
+}
+
+func (c *conn) Close() error {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	if c.closed {
+		return nil
+	}
+	c.closed = true
+	connCloseErr := c.Conn.Close()
+	recCloseErr := c.rec.Close()
+	return multierr.New(connCloseErr, recCloseErr)
+}
+
+// writeBufHasIncompleteFragment returns true if the latest data message
+// fragment written to the connection was incomplete and the following write
+// must be the remaining payload bytes of that fragment.
+func (c *conn) writeBufHasIncompleteFragment() bool {
+	return c.writeBuf.Len() != 0
+}
+
+// readBufHasIncompleteFragment returns true if the latest data message
+// fragment read from the connection was incomplete and the following read
+// must be the remaining payload bytes of that fragment.
+func (c *conn) readBufHasIncompleteFragment() bool {
+	return c.readBuf.Len() != 0
+}
+
+// writeMsgIsIncomplete returns true if the latest WebSocket message written to
+// the connection was fragmented and the next data message fragment written to
+// the connection must be a fragment of that message.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.4
+func (c *conn) writeMsgIsIncomplete() bool {
+	return c.currentWriteMsg != nil && !c.currentWriteMsg.isFinalized
+}
+
+// readMsgIsIncomplete returns true if the latest WebSocket message written to
+// the connection was fragmented and the next data message fragment written to
+// the connection must be a fragment of that message.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.4
+func (c *conn) readMsgIsIncomplete() bool {
+	return c.currentReadMsg != nil && !c.currentReadMsg.isFinalized
+}
+func (c *conn) curReadMsgType() (messageType, error) {
+	if c.currentReadMsg != nil {
+		return c.currentReadMsg.typ, nil
+	}
+	return 0, errors.New("[unexpected] attempted to determine type for nil message")
+}
+
+func (c *conn) curWriteMsgType() (messageType, error) {
+	if c.currentWriteMsg != nil {
+		return c.currentWriteMsg.typ, nil
+	}
+	return 0, errors.New("[unexpected] attempted to determine type for nil message")
+}
+
+// opcode reads the websocket message opcode that denotes the message type.
+// opcode is contained in bits [4-8] of the message.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+func opcode(b []byte) int {
+	// 0xf = 00001111; b & 00001111 zeroes out bits [0 - 3] of b
+	var mask byte = 0xf
+	return int(b[0] & mask)
+}

k8s-operator/sessionrecording/ws/conn_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package ws
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"go.uber.org/zap"
+	"k8s.io/apimachinery/pkg/util/remotecommand"
+	"tailscale.com/k8s-operator/sessionrecording/fakes"
+	"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
+	"tailscale.com/sessionrecording"
+	"tailscale.com/tstest"
+)
+
+func Test_conn_Read(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Resize stream ID + {"width": 10, "height": 20}
+	testResizeMsg := []byte{byte(remotecommand.StreamResize), 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}
+	lenResizeMsgPayload := byte(len(testResizeMsg))
+
+	tests := []struct {
+		name       string
+		inputs     [][]byte
+		wantWidth  int
+		wantHeight int
+	}{
+		{
+			name:   "single_read_control_message",
+			inputs: [][]byte{{0x88, 0x0}},
+		},
+		{
+			name:       "single_read_resize_message",
+			inputs:     [][]byte{append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...)},
+			wantWidth:  10,
+			wantHeight: 20,
+		},
+		{
+			name:       "two_reads_resize_message",
+			inputs:     [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}},
+			wantWidth:  10,
+			wantHeight: 20,
+		},
+		{
+			name:       "three_reads_resize_message_with_split_fragment",
+			inputs:     [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, {0x22, 0x3a, 0x32, 0x30, 0x7d}},
+			wantWidth:  10,
+			wantHeight: 20,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tc := &fakes.TestConn{}
+			tc.ResetReadBuf()
+			c := &conn{
+				Conn: tc,
+				log:  zl.Sugar(),
+			}
+			for i, input := range tt.inputs {
+				if err := tc.WriteReadBufBytes(input); err != nil {
+					t.Fatalf("writing bytes to test conn: %v", err)
+				}
+				_, err := c.Read(make([]byte, len(input)))
+				if err != nil {
+					t.Errorf("[%d] conn.Read() errored %v", i, err)
+					return
+				}
+			}
+			if tt.wantHeight != 0 || tt.wantWidth != 0 {
+				if tt.wantWidth != c.ch.Width {
+					t.Errorf("wants width: %v, got %v", tt.wantWidth, c.ch.Width)
+				}
+				if tt.wantHeight != c.ch.Height {
+					t.Errorf("want height: %v, got %v", tt.wantHeight, c.ch.Height)
+				}
+			}
+		})
+	}
+}
+
+func Test_conn_Write(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	tests := []struct {
+		name          string
+		inputs        [][]byte
+		wantForwarded []byte
+		wantRecorded  []byte
+		firstWrite    bool
+		width         int
+		height        int
+	}{
+		{
+			name:          "single_write_control_frame",
+			inputs:        [][]byte{{0x88, 0x0}},
+			wantForwarded: []byte{0x88, 0x0},
+		},
+		{
+			name:          "single_write_stdout_data_message",
+			inputs:        [][]byte{{0x82, 0x3, 0x1, 0x7, 0x8}},
+			wantForwarded: []byte{0x82, 0x3, 0x1, 0x7, 0x8},
+			wantRecorded:  fakes.CastLine(t, []byte{0x7, 0x8}, cl),
+		},
+		{
+			name:          "single_write_stderr_data_message",
+			inputs:        [][]byte{{0x82, 0x3, 0x2, 0x7, 0x8}},
+			wantForwarded: []byte{0x82, 0x3, 0x2, 0x7, 0x8},
+			wantRecorded:  fakes.CastLine(t, []byte{0x7, 0x8}, cl),
+		},
+		{
+			name:          "single_write_stdin_data_message",
+			inputs:        [][]byte{{0x82, 0x3, 0x0, 0x7, 0x8}},
+			wantForwarded: []byte{0x82, 0x3, 0x0, 0x7, 0x8},
+		},
+		{
+			name:          "single_write_stdout_data_message_with_cast_header",
+			inputs:        [][]byte{{0x82, 0x3, 0x1, 0x7, 0x8}},
+			wantForwarded: []byte{0x82, 0x3, 0x1, 0x7, 0x8},
+			wantRecorded:  append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x7, 0x8}, cl)...),
+			width:         10,
+			height:        20,
+			firstWrite:    true,
+		},
+		{
+			name:          "two_writes_stdout_data_message",
+			inputs:        [][]byte{{0x2, 0x3, 0x1, 0x7, 0x8}, {0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5}},
+			wantForwarded: []byte{0x2, 0x3, 0x1, 0x7, 0x8, 0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5},
+			wantRecorded:  fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl),
+		},
+		{
+			name:          "three_writes_stdout_data_message_with_split_fragment",
+			inputs:        [][]byte{{0x2, 0x3, 0x1, 0x7, 0x8}, {0x80, 0x6, 0x1, 0x1, 0x2, 0x3}, {0x4, 0x5}},
+			wantForwarded: []byte{0x2, 0x3, 0x1, 0x7, 0x8, 0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5},
+			wantRecorded:  fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tc := &fakes.TestConn{}
+			sr := &fakes.TestSessionRecorder{}
+			rec := tsrecorder.New(sr, cl, cl.Now(), true)
+			c := &conn{
+				Conn: tc,
+				log:  zl.Sugar(),
+				ch: sessionrecording.CastHeader{
+					Width:  tt.width,
+					Height: tt.height,
+				},
+				rec: rec,
+			}
+			if !tt.firstWrite {
+				// This test case does not intend to test that cast header gets written once.
+				c.writeCastHeaderOnce.Do(func() {})
+			}
+			for i, input := range tt.inputs {
+				_, err := c.Write(input)
+				if err != nil {
+					t.Fatalf("[%d] conn.Write() errored: %v", i, err)
+				}
+			}
+			// Assert that the expected bytes have been forwarded to the original destination.
+			gotForwarded := tc.WriteBufBytes()
+			if !reflect.DeepEqual(gotForwarded, tt.wantForwarded) {
+				t.Errorf("expected bytes not forwarded, wants\n%x\ngot\n%x", tt.wantForwarded, gotForwarded)
+			}
+
+			// Assert that the expected bytes have been forwarded to the session recorder.
+			gotRecorded := sr.Bytes()
+			if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) {
+				t.Errorf("expected bytes not recorded, wants\n%b\ngot\n%b", tt.wantRecorded, gotRecorded)
+			}
+		})
+	}
+}
+
+// Test_conn_ReadRand tests reading arbitrarily generated byte slices from conn to
+// test that we don't panic when parsing input from a broken or malicious
+// client.
+func Test_conn_ReadRand(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatalf("error creating a test logger: %v", err)
+	}
+	for i := range 100 {
+		tc := &fakes.TestConn{}
+		tc.ResetReadBuf()
+		c := &conn{
+			Conn: tc,
+			log:  zl.Sugar(),
+		}
+		bb := fakes.RandomBytes(t)
+		for j, input := range bb {
+			if err := tc.WriteReadBufBytes(input); err != nil {
+				t.Fatalf("[%d] writing bytes to test conn: %v", i, err)
+			}
+			f := func() {
+				c.Read(make([]byte, len(input)))
+			}
+			testPanic(t, f, fmt.Sprintf("[%d %d] Read panic parsing input of length %d first bytes: %v, current read message: %+#v", i, j, len(input), firstBytes(input), c.currentReadMsg))
+		}
+	}
+}
+
+// Test_conn_WriteRand calls conn.Write with an arbitrary input to validate that it does not
+// panic.
+func Test_conn_WriteRand(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatalf("error creating a test logger: %v", err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	sr := &fakes.TestSessionRecorder{}
+	rec := tsrecorder.New(sr, cl, cl.Now(), true)
+	for i := range 100 {
+		tc := &fakes.TestConn{}
+		c := &conn{
+			Conn: tc,
+			log:  zl.Sugar(),
+			rec:  rec,
+		}
+		bb := fakes.RandomBytes(t)
+		for j, input := range bb {
+			f := func() {
+				c.Write(input)
+			}
+			testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d first bytes %b current write message %+#v", i, j, len(input), firstBytes(input), c.currentWriteMsg))
+		}
+	}
+}
+
+func testPanic(t *testing.T, f func(), msg string) {
+	t.Helper()
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatal(msg, r)
+		}
+	}()
+	f()
+}
+
+func firstBytes(b []byte) []byte {
+	if len(b) < 10 {
+		return b
+	}
+	return b[:10]
+}

k8s-operator/sessionrecording/ws/message.go

@@ -0,0 +1,267 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package ws
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	"golang.org/x/net/websocket"
+)
+
+const (
+	noOpcode      messageType = 0 // continuation frame for fragmented messages
+	binaryMessage messageType = 2
+)
+
+// messageType is the type of a websocket data or control message as defined by opcode.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+// Known types of control messages are close, ping and pong.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.5
+// The only data message type supported by Kubernetes is binary message
+// https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L281
+type messageType int
+
+// message is a parsed Websocket Message.
+type message struct {
+	// payload is the contents of the so far parsed Websocket
+	// data Message payload, potentially from multiple fragments written by
+	// multiple invocations of Parse. As per RFC 6455 We can assume that the
+	// fragments will always arrive in order and data messages will not be
+	// interleaved.
+	payload []byte
+
+	// isFinalized is set to true if msgPayload contains full contents of
+	// the message (the final fragment has been received).
+	isFinalized bool
+
+	// streamID is the stream to which the message belongs, i.e stdin, stout
+	// etc. It is one of the stream IDs defined in
+	// https://github.com/kubernetes/apimachinery/blob/73d12d09c5be8703587b5127416eb83dc3b7e182/pkg/util/httpstream/wsstream/doc.go#L23-L36
+	streamID atomic.Uint32
+
+	// typ is the type of a WebsocketMessage as defined by its opcode
+	// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+	typ messageType
+	raw []byte
+}
+
+// Parse accepts a websocket message fragment as a byte slice and parses its contents.
+// It returns true if the fragment is complete, false if the fragment is incomplete.
+// If the fragment is incomplete, Parse will be called again with the same fragment + more bytes when those are received.
+// If the fragment is complete, it will be parsed into msg.
+// A complete fragment can be:
+// - a fragment that consists of a whole message
+// - an initial fragment for a message for which we expect more fragments
+// - a subsequent fragment for a message that we are currently parsing and whose so-far parsed contents are stored in msg.
+// Parse must not be called with bytes that don't contain fragment header (so, no less than 2 bytes).
+// 0                   1                   2                   3
+// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-------+-+-------------+-------------------------------+
+// |F|R|R|R| opcode|M| Payload len |    Extended payload length    |
+// |I|S|S|S|  (4)  |A|     (7)     |             (16/64)           |
+// |N|V|V|V|       |S|             |   (if payload len==126/127)   |
+// | |1|2|3|       |K|             |                               |
+// +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
+// |     Extended payload length continued, if payload len == 127  |
+// + - - - - - - - - - - - - - - - +-------------------------------+
+// |                               |Masking-key, if MASK set to 1  |
+// +-------------------------------+-------------------------------+
+// | Masking-key (continued)       |          Payload Data         |
+// +-------------------------------- - - - - - - - - - - - - - - - +
+// :                     Payload Data continued ...                :
+// + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
+// |                     Payload Data continued ...                |
+// +---------------------------------------------------------------+
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+//
+// Fragmentation rules:
+// An unfragmented message consists of a single frame with the FIN
+// bit set (Section 5.2) and an opcode other than 0.
+// A fragmented message consists of a single frame with the FIN bit
+// clear and an opcode other than 0, followed by zero or more frames
+// with the FIN bit clear and the opcode set to 0, and terminated by
+// a single frame with the FIN bit set and an opcode of 0.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.4
+func (msg *message) Parse(b []byte, log *zap.SugaredLogger) (bool, error) {
+	if len(b) < 2 {
+		return false, fmt.Errorf("[unexpected] Parse should not be called with less than 2 bytes, got %d bytes", len(b))
+	}
+	if msg.typ != binaryMessage {
+		return false, fmt.Errorf("[unexpected] internal error: attempted to parse a message with type %d", msg.typ)
+	}
+	isInitialFragment := len(msg.raw) == 0
+
+	msg.isFinalized = isFinalFragment(b)
+
+	maskSet := isMasked(b)
+
+	payloadLength, payloadOffset, maskOffset, err := fragmentDimensions(b, maskSet)
+	if err != nil {
+		return false, fmt.Errorf("error determining payload length: %w", err)
+	}
+	log.Debugf("parse: parsing a message fragment with payload length: %d payload offset: %d maskOffset: %d mask set: %t, is finalized: %t, is initial fragment: %t", payloadLength, payloadOffset, maskOffset, maskSet, msg.isFinalized, isInitialFragment)
+
+	if len(b) < int(payloadOffset+payloadLength) { // incomplete fragment
+		return false, nil
+	}
+	// TODO (irbekrm): perhaps only do this extra allocation if we know we
+	// will need to unmask?
+	msg.raw = make([]byte, int(payloadOffset)+int(payloadLength))
+	copy(msg.raw, b[:payloadOffset+payloadLength])
+
+	// Extract the payload.
+	msgPayload := b[payloadOffset : payloadOffset+payloadLength]
+
+	// Unmask the payload if needed.
+	// TODO (irbekrm): instead of unmasking all of the payload each time,
+	// determine if the payload is for a resize message early and skip
+	// unmasking the remaining bytes if not.
+	if maskSet {
+		m := b[maskOffset:payloadOffset]
+		var mask [4]byte
+		copy(mask[:], m)
+		maskBytes(mask, msgPayload)
+	}
+
+	// Determine what stream the message is for. Stream ID of a Kubernetes
+	// streaming session is a 32bit integer, stored in the first byte of the
+	// message payload.
+	// https://github.com/kubernetes/apimachinery/commit/73d12d09c5be8703587b5127416eb83dc3b7e182#diff-291f96e8632d04d2d20f5fb00f6b323492670570d65434e8eac90c7a442d13bdR23-R36
+	if len(msgPayload) == 0 {
+		return false, errors.New("[unexpected] received a message fragment with no stream ID")
+	}
+
+	streamID := uint32(msgPayload[0])
+	if !isInitialFragment && msg.streamID.Load() != streamID {
+		return false, fmt.Errorf("[unexpected] received message fragments with mismatched streamIDs %d and %d", msg.streamID.Load(), streamID)
+	}
+	msg.streamID.Store(streamID)
+
+	// This is normal, Kubernetes seem to send a couple data messages with
+	// no payloads at the start.
+	if len(msgPayload) < 2 {
+		return true, nil
+	}
+	msgPayload = msgPayload[1:] // remove the stream ID byte
+	msg.payload = append(msg.payload, msgPayload...)
+	return true, nil
+}
+
+// maskBytes applies mask to bytes in place.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.3
+func maskBytes(key [4]byte, b []byte) {
+	for i := range b {
+		b[i] = b[i] ^ key[i%4]
+	}
+}
+
+// isControlMessage returns true if the message type is one of the known control
+// frame message types.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.5
+func isControlMessage(t messageType) bool {
+	const (
+		closeMessage messageType = 8
+		pingMessage  messageType = 9
+		pongMessage  messageType = 10
+	)
+	return t == closeMessage || t == pingMessage || t == pongMessage
+}
+
+// isFinalFragment can be called with  websocket message fragment and returns true if
+// the fragment is the final fragment of a websocket message.
+func isFinalFragment(b []byte) bool {
+	return extractFirstBit(b[0]) != 0
+}
+
+// isMasked can be called with a websocket message fragment and returns true if
+// the payload of the message is masked. It uses the mask bit to determine if
+// the payload is masked.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.3
+func isMasked(b []byte) bool {
+	return extractFirstBit(b[1]) != 0
+}
+
+// extractFirstBit extracts first bit of a byte by zeroing out all the other
+// bits.
+func extractFirstBit(b byte) byte {
+	return b & 0x80
+}
+
+// zeroFirstBit returns the provided byte with the first bit set to 0.
+func zeroFirstBit(b byte) byte {
+	return b & 0x7f
+}
+
+// fragmentDimensions returns payload length as well as payload offset and mask offset.
+func fragmentDimensions(b []byte, maskSet bool) (payloadLength, payloadOffset, maskOffset uint64, _ error) {
+
+	// payload length can be stored either in bits [9-15] or in bytes 2, 3
+	// or in bytes 2, 3, 4, 5, 6, 7.
+	// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+	// 0                   1                   2                   3
+	//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+	// +-+-+-+-+-------+-+-------------+-------------------------------+
+	// |F|R|R|R| opcode|M| Payload len |    Extended payload length    |
+	// |I|S|S|S|  (4)  |A|     (7)     |             (16/64)           |
+	// |N|V|V|V|       |S|             |   (if payload len==126/127)   |
+	// | |1|2|3|       |K|             |                               |
+	// +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
+	// |     Extended payload length continued, if payload len == 127  |
+	// + - - - - - - - - - - - - - - - +-------------------------------+
+	// |                               |Masking-key, if MASK set to 1  |
+	// +-------------------------------+-------------------------------+
+	payloadLengthIndicator := zeroFirstBit(b[1])
+	switch {
+	case payloadLengthIndicator < 126:
+		maskOffset = 2
+		payloadLength = uint64(payloadLengthIndicator)
+	case payloadLengthIndicator == 126:
+		maskOffset = 4
+		if len(b) < int(maskOffset) {
+			return 0, 0, 0, fmt.Errorf("invalid message fragment- length indicator suggests that length is stored in bytes 2:4, but message length is only %d", len(b))
+		}
+		payloadLength = uint64(binary.BigEndian.Uint16(b[2:4]))
+	case payloadLengthIndicator == 127:
+		maskOffset = 10
+		if len(b) < int(maskOffset) {
+			return 0, 0, 0, fmt.Errorf("invalid message fragment- length indicator suggests that length is stored in bytes 2:10, but message length is only %d", len(b))
+		}
+		payloadLength = binary.BigEndian.Uint64(b[2:10])
+	default:
+		return 0, 0, 0, fmt.Errorf("unexpected payload length indicator value: %v", payloadLengthIndicator)
+	}
+
+	// Ensure that a rogue or broken client doesn't cause us attempt to
+	// allocate a huge array by setting a high payload size.
+	// websocket.DefaultMaxPayloadBytes is the maximum payload size accepted
+	// by server side of this connection, so we can safely reject messages
+	// with larger payload size.
+	if payloadLength > websocket.DefaultMaxPayloadBytes {
+		return 0, 0, 0, fmt.Errorf("[unexpected]: too large payload size: %v", payloadLength)
+	}
+
+	// Masking key can take up 0 or 4 bytes- we need to take that into
+	// account when determining payload offset.
+	// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+	// ....
+	// + - - - - - - - - - - - - - - - +-------------------------------+
+	// |                               |Masking-key, if MASK set to 1  |
+	// +-------------------------------+-------------------------------+
+	// | Masking-key (continued)       |          Payload Data         |
+	// + - - - - - - - - - - - - - - - +-------------------------------+
+	// ...
+	if maskSet {
+		payloadOffset = maskOffset + 4
+	} else {
+		payloadOffset = maskOffset
+	}
+	return
+}

k8s-operator/sessionrecording/ws/message_test.go

@@ -0,0 +1,215 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package ws
+
+import (
+	"encoding/binary"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"math/rand"
+
+	"go.uber.org/zap"
+	"golang.org/x/net/websocket"
+)
+
+func Test_msg_Parse(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatalf("error creating a test logger: %v", err)
+	}
+	testMask := [4]byte{1, 2, 3, 4}
+	bs126, bs126Len := bytesSlice2ByteLen(t)
+	bs127, bs127Len := byteSlice8ByteLen(t)
+	tests := []struct {
+		name            string
+		b               []byte
+		initialPayload  []byte
+		wantPayload     []byte
+		wantIsFinalized bool
+		wantStreamID    uint32
+		wantErr         bool
+	}{
+		{
+			name:            "single_fragment_stdout_stream_no_payload_no_mask",
+			b:               []byte{0x82, 0x1, 0x1},
+			wantPayload:     nil,
+			wantIsFinalized: true,
+			wantStreamID:    1,
+		},
+		{
+			name:            "single_fragment_stderr_steam_no_payload_has_mask",
+			b:               append([]byte{0x82, 0x81, 0x1, 0x2, 0x3, 0x4}, maskedBytes(testMask, []byte{0x2})...),
+			wantPayload:     nil,
+			wantIsFinalized: true,
+			wantStreamID:    2,
+		},
+		{
+			name:            "single_fragment_stdout_stream_no_mask_has_payload",
+			b:               []byte{0x82, 0x3, 0x1, 0x7, 0x8},
+			wantPayload:     []byte{0x7, 0x8},
+			wantIsFinalized: true,
+			wantStreamID:    1,
+		},
+		{
+			name:            "single_fragment_stdout_stream_has_mask_has_payload",
+			b:               append([]byte{0x82, 0x83, 0x1, 0x2, 0x3, 0x4}, maskedBytes(testMask, []byte{0x1, 0x7, 0x8})...),
+			wantPayload:     []byte{0x7, 0x8},
+			wantIsFinalized: true,
+			wantStreamID:    1,
+		},
+		{
+			name:         "initial_fragment_stdout_stream_no_mask_has_payload",
+			b:            []byte{0x2, 0x3, 0x1, 0x7, 0x8},
+			wantPayload:  []byte{0x7, 0x8},
+			wantStreamID: 1,
+		},
+		{
+			name:         "initial_fragment_stdout_stream_has_mask_has_payload",
+			b:            append([]byte{0x2, 0x83, 0x1, 0x2, 0x3, 0x4}, maskedBytes(testMask, []byte{0x1, 0x7, 0x8})...),
+			wantPayload:  []byte{0x7, 0x8},
+			wantStreamID: 1,
+		},
+		{
+			name:           "subsequent_fragment_stdout_stream_no_mask_has_payload",
+			b:              []byte{0x0, 0x3, 0x1, 0x7, 0x8},
+			initialPayload: []byte{0x1, 0x2, 0x3},
+			wantPayload:    []byte{0x1, 0x2, 0x3, 0x7, 0x8},
+			wantStreamID:   1,
+		},
+		{
+			name:           "subsequent_fragment_stdout_stream_has_mask_has_payload",
+			b:              append([]byte{0x0, 0x83, 0x1, 0x2, 0x3, 0x4}, maskedBytes(testMask, []byte{0x1, 0x7, 0x8})...),
+			initialPayload: []byte{0x1, 0x2, 0x3},
+			wantPayload:    []byte{0x1, 0x2, 0x3, 0x7, 0x8},
+			wantStreamID:   1,
+		},
+		{
+			name:            "final_fragment_stdout_stream_no_mask_has_payload",
+			b:               []byte{0x80, 0x3, 0x1, 0x7, 0x8},
+			initialPayload:  []byte{0x1, 0x2, 0x3},
+			wantIsFinalized: true,
+			wantPayload:     []byte{0x1, 0x2, 0x3, 0x7, 0x8},
+			wantStreamID:    1,
+		},
+		{
+			name:            "final_fragment_stdout_stream_has_mask_has_payload",
+			b:               append([]byte{0x80, 0x83, 0x1, 0x2, 0x3, 0x4}, maskedBytes(testMask, []byte{0x1, 0x7, 0x8})...),
+			initialPayload:  []byte{0x1, 0x2, 0x3},
+			wantIsFinalized: true,
+			wantPayload:     []byte{0x1, 0x2, 0x3, 0x7, 0x8},
+			wantStreamID:    1,
+		},
+		{
+			name:            "single_large_fragment_no_mask_length_hint_126",
+			b:               append(append([]byte{0x80, 0x7e}, bs126Len...), append([]byte{0x1}, bs126...)...),
+			wantIsFinalized: true,
+			wantPayload:     bs126,
+			wantStreamID:    1,
+		},
+		{
+			name:            "single_large_fragment_no_mask_length_hint_127",
+			b:               append(append([]byte{0x80, 0x7f}, bs127Len...), append([]byte{0x1}, bs127...)...),
+			wantIsFinalized: true,
+			wantPayload:     bs127,
+			wantStreamID:    1,
+		},
+		{
+			name:    "zero_length_bytes",
+			b:       []byte{},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			msg := &message{
+				typ:     binaryMessage,
+				payload: tt.initialPayload,
+			}
+			if _, err := msg.Parse(tt.b, zl.Sugar()); (err != nil) != tt.wantErr {
+				t.Errorf("msg.Parse() = %v, wantsErr: %t", err, tt.wantErr)
+			}
+			if msg.isFinalized != tt.wantIsFinalized {
+				t.Errorf("wants message to be finalized: %t, got: %t", tt.wantIsFinalized, msg.isFinalized)
+			}
+			if msg.streamID.Load() != tt.wantStreamID {
+				t.Errorf("wants stream ID: %d, got: %d", tt.wantStreamID, msg.streamID.Load())
+			}
+			if !reflect.DeepEqual(msg.payload, tt.wantPayload) {
+				t.Errorf("unexpected message payload after Parse, wants %b got %b", tt.wantPayload, msg.payload)
+			}
+		})
+	}
+}
+
+// Test_msg_Parse_Rand calls Parse with a randomly generated input to verify
+// that it doesn't panic.
+func Test_msg_Parse_Rand(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatalf("error creating a test logger: %v", err)
+	}
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for i := range 100 {
+		n := r.Intn(4096)
+		b := make([]byte, n)
+		_, err := r.Read(b)
+		if err != nil {
+			t.Fatalf("error generating random byte slice: %v", err)
+		}
+		msg := message{typ: binaryMessage}
+		f := func() {
+			msg.Parse(b, zl.Sugar())
+		}
+		testPanic(t, f, fmt.Sprintf("[%d] Parse panicked running with byte slice of length %d: %v", i, n, r))
+	}
+}
+
+// byteSlice2ByteLen generates a number that represents websocket message fragment length and is stored in an 8 byte slice.
+// Returns the byte slice with the length as well as a slice of arbitrary bytes of the given length.
+// This is used to generate test input representing websocket message with payload length hint 126.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+func bytesSlice2ByteLen(t *testing.T) ([]byte, []byte) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	var n uint16
+	n = uint16(rand.Intn(65535 - 1)) // space for and additional 1 byte stream ID
+	b := make([]byte, n)
+	_, err := r.Read(b)
+	if err != nil {
+		t.Fatalf("error generating random byte slice: %v ", err)
+	}
+	bb := make([]byte, 2)
+	binary.BigEndian.PutUint16(bb, n+1) // + stream ID
+	return b, bb
+}
+
+// byteSlice8ByteLen generates a number that represents websocket message fragment length and is stored in an 8 byte slice.
+// Returns the byte slice with the length as well as a slice of arbitrary bytes of the given length.
+// This is used to generate test input representing websocket message with payload length hint 127.
+// https://www.rfc-editor.org/rfc/rfc6455#section-5.2
+func byteSlice8ByteLen(t *testing.T) ([]byte, []byte) {
+	nanos := time.Now().UnixNano()
+	t.Logf("Creating random source with seed %v", nanos)
+	r := rand.New(rand.NewSource(nanos))
+	var n uint64
+	n = uint64(rand.Intn(websocket.DefaultMaxPayloadBytes - 1)) // space for and additional 1 byte stream ID
+	t.Logf("byteSlice8ByteLen: generating message payload of length %d", n)
+	b := make([]byte, n)
+	_, err := r.Read(b)
+	if err != nil {
+		t.Fatalf("error generating random byte slice: %v ", err)
+	}
+	bb := make([]byte, 8)
+	binary.BigEndian.PutUint64(bb, n+1) // + stream ID
+	return b, bb
+}
+
+func maskedBytes(mask [4]byte, b []byte) []byte {
+	maskBytes(mask, b)
+	return b
+}

licenses/android.md

@@ -60,8 +60,8 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/tailscale/netlink](https://pkg.go.dev/github.com/tailscale/netlink) ([Apache-2.0](https://github.com/tailscale/netlink/blob/cabfb018fe85/LICENSE))
  - [github.com/tailscale/peercred](https://pkg.go.dev/github.com/tailscale/peercred) ([BSD-3-Clause](https://github.com/tailscale/peercred/blob/b535050b2aa4/LICENSE))
  - [github.com/tailscale/tailscale-android/libtailscale](https://pkg.go.dev/github.com/tailscale/tailscale-android/libtailscale) ([BSD-3-Clause](https://github.com/tailscale/tailscale-android/blob/HEAD/LICENSE))
- - [github.com/tailscale/wireguard-go](https://pkg.go.dev/github.com/tailscale/wireguard-go) ([MIT](https://github.com/tailscale/wireguard-go/blob/2f5d148bcfe1/LICENSE))
- - [github.com/tailscale/xnet/webdav](https://pkg.go.dev/github.com/tailscale/xnet/webdav) ([BSD-3-Clause](https://github.com/tailscale/xnet/blob/62b9a7c569f9/LICENSE))
+ - [github.com/tailscale/wireguard-go](https://pkg.go.dev/github.com/tailscale/wireguard-go) ([MIT](https://github.com/tailscale/wireguard-go/blob/71393c576b98/LICENSE))
+ - [github.com/tailscale/xnet/webdav](https://pkg.go.dev/github.com/tailscale/xnet/webdav) ([BSD-3-Clause](https://github.com/tailscale/xnet/blob/8497ac4dab2e/LICENSE))
  - [github.com/tcnksm/go-httpstat](https://pkg.go.dev/github.com/tcnksm/go-httpstat) ([MIT](https://github.com/tcnksm/go-httpstat/blob/v0.2.0/LICENSE))
  - [github.com/u-root/uio](https://pkg.go.dev/github.com/u-root/uio) ([BSD-3-Clause](https://github.com/u-root/uio/blob/a3c409a6018e/LICENSE))
  - [github.com/vishvananda/netlink/nl](https://pkg.go.dev/github.com/vishvananda/netlink/nl) ([Apache-2.0](https://github.com/vishvananda/netlink/blob/v1.2.1-beta.2/LICENSE))
@@ -71,18 +71,18 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [go4.org/mem](https://pkg.go.dev/go4.org/mem) ([Apache-2.0](https://github.com/go4org/mem/blob/4f986261bf13/LICENSE))
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [go4.org/unsafe/assume-no-moving-gc](https://pkg.go.dev/go4.org/unsafe/assume-no-moving-gc) ([BSD-3-Clause](https://github.com/go4org/unsafe-assume-no-moving-gc/blob/e7c30c78aeb2/LICENSE))
- - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.24.0:LICENSE))
+ - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.25.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/1b970713:LICENSE))
  - [golang.org/x/mobile](https://pkg.go.dev/golang.org/x/mobile) ([BSD-3-Clause](https://cs.opensource.google/go/x/mobile/+/c58ccf4b:LICENSE))
- - [golang.org/x/mod/semver](https://pkg.go.dev/golang.org/x/mod/semver) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.18.0:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.26.0:LICENSE))
+ - [golang.org/x/mod/semver](https://pkg.go.dev/golang.org/x/mod/semver) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.19.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.27.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.7.0:LICENSE))
- - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.21.0:LICENSE))
- - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.21.0:LICENSE))
+ - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.22.0:LICENSE))
+ - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.22.0:LICENSE))
  - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.16.0:LICENSE))
  - [golang.org/x/time/rate](https://pkg.go.dev/golang.org/x/time/rate) ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE))
- - [golang.org/x/tools](https://pkg.go.dev/golang.org/x/tools) ([BSD-3-Clause](https://cs.opensource.google/go/x/tools/+/v0.22.0:LICENSE))
- - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/ee1e1f6070e3/LICENSE))
+ - [golang.org/x/tools](https://pkg.go.dev/golang.org/x/tools) ([BSD-3-Clause](https://cs.opensource.google/go/x/tools/+/v0.23.0:LICENSE))
+ - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/64c016c92987/LICENSE))
  - [inet.af/netaddr](https://pkg.go.dev/inet.af/netaddr) ([BSD-3-Clause](Unknown))
- - [nhooyr.io/websocket](https://pkg.go.dev/nhooyr.io/websocket) ([ISC](https://github.com/nhooyr/websocket/blob/v1.8.10/LICENSE.txt))
+ - [nhooyr.io/websocket](https://pkg.go.dev/nhooyr.io/websocket) ([ISC](https://github.com/nhooyr/websocket-old/blob/v1.8.10/LICENSE.txt))
  - [tailscale.com](https://pkg.go.dev/tailscale.com) ([BSD-3-Clause](https://github.com/tailscale/tailscale/blob/HEAD/LICENSE))