cli: implement tailscale dns stream

Updates tailscale/tailscale#13326

This PR adds another subcommand to `tailscale dns`, to stream queries and answers returned by the DNS forwarder as they are handled.

Useful for debugging purposes, and is equivalent to setting the `TS_DEBUG_DNS_FORWARD_SEND` envknob and filtering the logs for relevant entries. This also adds a new envknob, `TS_DEBUG_DNS_INCLUDE_NAMES`, which includes the actual hostnames in the log lines (with a huge privacy warning!). This makes it easier to diagnose issues with DNS resolution.

.github/workflows/golangci-lint.yml

@@ -31,10 +31,10 @@ jobs:
           cache: false
 
       - name: golangci-lint
-        # Note: this is the 'v3' tag as of 2023-08-14
-        uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299
+        # Note: this is the 'v6.1.0' tag as of 2024-08-21
+        uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86
         with:
-          version: v1.56
+          version: v1.60
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/update-flake.yml

@@ -35,7 +35,7 @@ jobs:
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 #v7.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 #v7.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.gitignore

@@ -43,3 +43,9 @@ client/web/build/assets
 
 /gocross
 /dist
+
+# Ignore xcode userstate and workspace data
+*.xcuserstate
+*.xcworkspacedata
+/tstest/tailmac/bin
+/tstest/tailmac/build

Dockerfile

@@ -1,6 +1,13 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
+# Note that this Dockerfile is currently NOT used to build any of the published
+# Tailscale container images and may have drifted from the image build mechanism
+# we use.
+# Tailscale images are currently built using https://github.com/tailscale/mkctr,
+# and the build script can be found in ./build_docker.sh.
+#
+#
 # This Dockerfile includes all the tailscale binaries.
 #
 # To build the Dockerfile:
@@ -20,7 +27,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.22-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
@@ -35,7 +42,7 @@ RUN go install \
     gvisor.dev/gvisor/pkg/tcpip/stack \
     golang.org/x/crypto/ssh \
     golang.org/x/crypto/acme \
-    nhooyr.io/websocket \
+    github.com/coder/websocket \
     github.com/mdlayher/netlink
 
 COPY . .

Makefile

@@ -117,7 +117,8 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
+	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
 
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"

README.md

@@ -37,7 +37,7 @@ not open source.
 
 ## Building
 
-We always require the latest Go release, currently Go 1.22. (While we build
+We always require the latest Go release, currently Go 1.23. (While we build
 releases with our [Go fork](https://github.com/tailscale/go/), its use is not
 required.)
 

VERSION.txt

@@ -1 +1 @@
-1.71.0
+1.75.0

api.md

@@ -1,104 +1,2 @@
 > [!IMPORTANT]
 > The Tailscale API documentation has moved to https://tailscale.com/api
-
-# Tailscale API
-
-The Tailscale API documentation is located in **[tailscale/publicapi](./publicapi/readme.md#tailscale-api)**.
-
-# APIs
-
-**[Overview](./publicapi/readme.md)**
-
-**[Device](./publicapi/device.md#device)**
-
-<a href="device-delete"></a>
-<a href="expire-device-key"></a>
-<a href="device-routes-get">
-<a href="device-routes-post"></a>
-<a href="#device-authorized-post"></a>
-<a href="device-tags-post"></a>
-<a href="device-key-post"></a>
-<a href="tailnet-acl-get"></a>
-
-- Get a device: [`GET /api/v2/device/{deviceid}`](./publicapi/device.md#get-device)
-- Delete a device: [`DELETE /api/v2/device/{deviceID}`](./publicapi/device.md#delete-device)
-- Expire device key: [`POST /api/v2/device/{deviceID}/expire`](./publicapi/device.md#expire-device-key)
-- [**Routes**](./publicapi/device.md#routes)
-  - Get device routes: [`GET /api/v2/device/{deviceID}/routes`](./publicapi/device.md#get-device-routes)
-  - Set device routes: [`POST /api/v2/device/{deviceID}/routes`](./publicapi/device.md#set-device-routes)
-- [**Authorize**](./publicapi/device.md#authorize)
-  - Authorize a device: [`POST /api/v2/device/{deviceID}/authorized`](./publicapi/device.md#authorize-device)
-- [**Tags**](./publicapi/device.md#tags)
-  - Update tags: [`POST /api/v2/device/{deviceID}/tags`](./publicapi/device.md#update-device-tags)
-- [**Keys**](./publicapi/device.md#keys)
-  - Update device key: [`POST /api/v2/device/{deviceID}/key`](./publicapi/device.md#update-device-key)
-- [**IP Addresses**](./publicapi/device.md#ip-addresses)
-  - Set device IPv4 address: [`POST /api/v2/device/{deviceID}/ip`](./publicapi/device.md#set-device-ipv4-address)
-- [**Device posture attributes**](./publicapi/device.md#device-posture-attributes)
-  - Get device posture attributes: [`GET /api/v2/device/{deviceID}/attributes`](./publicapi/device.md#get-device-posture-attributes)
-  - Set custom device posture attributes: [`POST /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#set-device-posture-attributes)
-  - Delete custom device posture attributes: [`DELETE /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#delete-custom-device-posture-attributes)
-- [**Device invites**](./publicapi/device.md#invites-to-a-device)
-  - List device invites: [`GET /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#list-device-invites)
-  - Create device invites: [`POST /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#create-device-invites)
-
-**[Tailnet](./publicapi/tailnet.md#tailnet)**
-
-<a href="tailnet-acl-post"></a>
-<a href="tailnet-acl-preview-post"></a>
-<a href="tailnet-acl-validate-post"></a>
-<a href="tailnet-devices"></a>
-<a href="tailnet-keys-get"></a>
-<a href="tailnet-keys-post"></a>
-<a href="tailnet-keys-key-get"></a>
-<a href="tailnet-keys-key-delete"></a>
-<a href="tailnet-dns"></a>
-<a href="tailnet-dns-nameservers-get"></a>
-<a href="tailnet-dns-nameservers-post"></a>
-<a href="tailnet-dns-preferences-get"></a>
-<a href="tailnet-dns-preferences-post"></a>
-<a href="tailnet-dns-searchpaths-get"></a>
-<a href="tailnet-dns-searchpaths-post"></a>
-
-- [**Policy File**](./publicapi/tailnet.md#policy-file)
-  - Get policy file: [`GET /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#get-policy-file)
-  - Update policy file: [`POST /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#update-policy-file)
-  - Preview rule matches: [`POST /api/v2/tailnet/{tailnet}/acl/preview`](./publicapi/tailnet.md#preview-policy-file-rule-matches)
-  - Validate and test policy file: [`POST /api/v2/tailnet/{tailnet}/acl/validate`](./publicapi/tailnet.md#validate-and-test-policy-file)
-- [**Devices**](./publicapi/tailnet.md#devices)
-  - List tailnet devices: [`GET /api/v2/tailnet/{tailnet}/devices`](./publicapi/tailnet.md#list-tailnet-devices)
-- [**Keys**](./publicapi/tailnet.md#tailnet-keys)
-  - List tailnet keys: [`GET /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#list-tailnet-keys)
-  - Create an auth key: [`POST /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#create-auth-key)
-  - Get a key: [`GET /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#get-key)
-  - Delete a key: [`DELETE /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#delete-key)
-- [**DNS**](./publicapi/tailnet.md#dns)
-  - [**Nameservers**](./publicapi/tailnet.md#nameservers)
-    - Get nameservers: [`GET /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#get-nameservers)
-    - Set nameservers: [`POST /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#set-nameservers)
-  - [**Preferences**](./publicapi/tailnet.md#preferences)
-    - Get DNS preferences: [`GET /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#get-dns-preferences)
-    - Set DNS preferences: [`POST /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#set-dns-preferences)
-  - [**Search Paths**](./publicapi/tailnet.md#search-paths)
-    - Get search paths: [`GET /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#get-search-paths)
-    - Set search paths: [`POST /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#set-search-paths)
-  - [**Split DNS**](./publicapi/tailnet.md#split-dns)
-    - Get split DNS: [`GET /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#get-split-dns)
-    - Update split DNS: [`PATCH /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#update-split-dns)
-    - Set split DNS: [`PUT /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#set-split-dns)
-- [**User invites**](./publicapi/tailnet.md#tailnet-user-invites)
-  - List user invites: [`GET /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#list-user-invites)
-  - Create user invites: [`POST /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#create-user-invites)
-
-**[User invites](./publicapi/userinvites.md#user-invites)**
-
-- Get user invite: [`GET /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#get-user-invite)
-- Delete user invite: [`DELETE /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#delete-user-invite)
-- Resend user invite (by email): [`POST /api/v2/user-invites/{userInviteId}/resend`](#resend-user-invite)
-
-**[Device invites](./publicapi/deviceinvites.md#device-invites)**
-
-- Get device invite: [`GET /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#get-device-invite)
-- Delete device invite: [`DELETE /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#delete-device-invite)
-- Resend device invite (by email): [`POST /api/v2/device-invites/{deviceInviteId}/resend`](./publicapi/deviceinvites.md#resend-device-invite)
-- Accept device invite [`POST /api/v2/device-invites/-/accept`](#accept-device-invite)

build_docker.sh

@@ -1,21 +1,11 @@
 #!/usr/bin/env sh
-
 #
-# Runs `go build` with flags configured for docker distribution. All
-# it does differently from `go build` is burn git commit and version
-# information into the binaries inside docker, so that we can track down user
-# issues.
-#
-############################################################################
-#
-# WARNING: Tailscale is not yet officially supported in container
-# environments, such as Docker and Kubernetes. Though it should work, we
-# don't regularly test it, and we know there are some feature limitations.
-#
-# See current bugs tagged "containers":
-#    https://github.com/tailscale/tailscale/labels/containers
-#
-############################################################################
+# This script builds Tailscale container images using
+# github.com/tailscale/mkctr.
+# By default the images will be tagged with the current version and git
+# hash of this repository as produced by ./cmd/mkversion.
+# This is the image build mechanim used to build the official Tailscale
+# container images.
 
 set -eu
 

client/tailscale/acl.go

@@ -19,6 +19,7 @@ import (
 // Only one of Src/Dst or Users/Ports may be specified.
 type ACLRow struct {
 	Action string   `json:"action,omitempty"` // valid values: "accept"
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Users  []string `json:"users,omitempty"`  // old name for src
 	Ports  []string `json:"ports,omitempty"`  // old name for dst
 	Src    []string `json:"src,omitempty"`
@@ -31,6 +32,7 @@ type ACLRow struct {
 type ACLTest struct {
 	Src    string   `json:"src,omitempty"`    // source
 	User   string   `json:"user,omitempty"`   // old name for source
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Accept []string `json:"accept,omitempty"` // expected destination ip:port that user can access
 	Deny   []string `json:"deny,omitempty"`   // expected destination ip:port that user cannot access
 
@@ -286,6 +288,9 @@ type UserRuleMatch struct {
 	Users      []string `json:"users"`
 	Ports      []string `json:"ports"`
 	LineNumber int      `json:"lineNumber"`
+	// Via is the list of targets through which Users can access Ports.
+	// See https://tailscale.com/kb/1378/via for more information.
+	Via []string `json:"via,omitempty"`
 
 	// Postures is a list of posture policies that are
 	// associated with this match. The rules can be looked

client/tailscale/apitype/apitype.go

@@ -4,7 +4,10 @@
 // Package apitype contains types for the Tailscale LocalAPI and control plane API.
 package apitype
 
-import "tailscale.com/tailcfg"
+import (
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/dnstype"
+)
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
@@ -57,3 +60,19 @@ type ExitNodeSuggestionResponse struct {
 	Name     string
 	Location tailcfg.LocationView `json:",omitempty"`
 }
+
+// DNSOSConfig mimics dns.OSConfig without forcing us to import the entire dns package
+// into the CLI.
+type DNSOSConfig struct {
+	Nameservers   []string
+	SearchDomains []string
+	MatchDomains  []string
+}
+
+// DNSQueryResponse is the response to a DNS query request sent via LocalAPI.
+type DNSQueryResponse struct {
+	// Bytes is the raw DNS response bytes.
+	Bytes []byte
+	// Resolvers is the list of resolvers that the forwarder deemed able to resolve the query.
+	Resolvers []*dnstype.Resolver
+}

client/tailscale/localclient.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
+	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
 )
@@ -69,6 +70,14 @@ type LocalClient struct {
 	// connecting to the GUI client variants.
 	UseSocketOnly bool
 
+	// OmitAuth, if true, omits sending the local Tailscale daemon any
+	// authentication token that might be required by the platform.
+	//
+	// As of 2024-08-12, only macOS uses an authentication token. OmitAuth is
+	// meant for when Dial is set and the LocalAPI is being proxied to a
+	// different operating system, such as in integration tests.
+	OmitAuth bool
+
 	// tsClient does HTTP requests to the local Tailscale daemon.
 	// It's lazily initialized on first use.
 	tsClient     *http.Client
@@ -124,8 +133,10 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 			},
 		}
 	})
-	if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
-		req.SetBasicAuth("", token)
+	if !lc.OmitAuth {
+		if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
+			req.SetBasicAuth("", token)
+		}
 	}
 	return lc.tsClient.Do(req)
 }
@@ -343,6 +354,12 @@ func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
+// UserMetrics returns the user metrics in
+// the Prometheus text exposition format.
+func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
+	return lc.get200(ctx, "/localapi/v0/usermetrics")
+}
+
 // IncrementCounter increments the value of a Tailscale daemon's counter
 // metric by the given delta. If the metric has yet to exist, a new counter
 // metric is created and initialized to delta.
@@ -797,6 +814,35 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
+// GetDNSOSConfig returns the system DNS configuration for the current device.
+// That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
+func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
+	if err != nil {
+		return nil, err
+	}
+	var osCfg apitype.DNSOSConfig
+	if err := json.Unmarshal(body, &osCfg); err != nil {
+		return nil, fmt.Errorf("invalid dns.OSConfig: %w", err)
+	}
+	return &osCfg, nil
+}
+
+// QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
+// It returns the raw DNS response bytes and the resolvers that were used to answer the query
+// (often just one, but can be more if we raced multiple resolvers).
+func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
+	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
+	if err != nil {
+		return nil, nil, err
+	}
+	var res apitype.DNSQueryResponse
+	if err := json.Unmarshal(body, &res); err != nil {
+		return nil, nil, fmt.Errorf("invalid query response: %w", err)
+	}
+	return res.Bytes, res.Resolvers, nil
+}
+
 // StartLoginInteractive starts an interactive login.
 func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)
@@ -1424,6 +1470,13 @@ func (lc *LocalClient) DebugDERPRegion(ctx context.Context, regionIDOrCode strin
 	return decodeJSON[*ipnstate.DebugDERPRegionReport](body)
 }
 
+// DebugEnvknob sets a envknob for debugging purposes.
+func (lc *LocalClient) DebugEnvknob(ctx context.Context, key, value string) error {
+	v := url.Values{"key": {key}, "value": {value}}
+	_, err := lc.send(ctx, "POST", "/localapi/v0/debug-envknob?"+v.Encode(), 200, nil)
+	return err
+}
+
 // DebugPacketFilterRules returns the packet filter rules for the current device.
 func (lc *LocalClient) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterRule, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug-packet-filter-rules", 200, nil)

client/tailscale/required_version.go

@@ -1,10 +1,10 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !go1.21
+//go:build !go1.23
 
 package tailscale
 
 func init() {
-	you_need_Go_1_21_to_compile_Tailscale()
+	you_need_Go_1_23_to_compile_Tailscale()
 }

client/web/web.go

@@ -283,6 +283,12 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if r.URL.Path == "/metrics" {
+		r.URL.Path = "/api/local/v0/usermetrics"
+		s.proxyRequestToLocalAPI(w, r)
+		return
+	}
+
 	if strings.HasPrefix(r.URL.Path, "/api/") {
 		switch {
 		case r.URL.Path == "/api/auth" && r.Method == httpm.GET:

client/web/yarn.lock

@@ -5382,9 +5382,9 @@ wrappy@1:
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
 ws@^8.14.2:
-  version "8.14.2"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f"
-  integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
+  integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
 
 xml-name-validator@^5.0.0:
   version "5.0.0"

cmd/cloner/cloner.go

@@ -47,7 +47,7 @@ func main() {
 	it := codegen.NewImportTracker(pkg.Types)
 	buf := new(bytes.Buffer)
 	for _, typeName := range typeNames {
-		typ, ok := namedTypes[typeName]
+		typ, ok := namedTypes[typeName].(*types.Named)
 		if !ok {
 			log.Fatalf("could not find type %s", typeName)
 		}
@@ -115,7 +115,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		if !codegen.ContainsPointers(ft) || codegen.HasNoClone(t.Tag(i)) {
 			continue
 		}
-		if named, _ := ft.(*types.Named); named != nil {
+		if named, _ := codegen.NamedTypeOf(ft); named != nil {
 			if codegen.IsViewType(ft) {
 				writef("dst.%s = src.%s", fname, fname)
 				continue
@@ -161,7 +161,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		case *types.Pointer:
 			base := ft.Elem()
 			hasPtrs := codegen.ContainsPointers(base)
-			if named, _ := base.(*types.Named); named != nil && hasPtrs {
+			if named, _ := codegen.NamedTypeOf(base); named != nil && hasPtrs {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}

cmd/containerboot/forwarding.go

@@ -0,0 +1,262 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"tailscale.com/util/linuxfw"
+)
+
+// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
+	var (
+		v4Forwarding, v6Forwarding bool
+	)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	// Currently we only proxy traffic to the IPv4 address of the tailnet
+	// target.
+	if tailnetTargetFQDN != "" {
+		v4Forwarding = true
+	}
+	if routes != nil && *routes != "" {
+		for _, route := range strings.Split(*routes, ",") {
+			cidr, err := netip.ParsePrefix(route)
+			if err != nil {
+				return fmt.Errorf("invalid subnet route: %v", err)
+			}
+			if cidr.Addr().Is4() {
+				v4Forwarding = true
+			} else {
+				v6Forwarding = true
+			}
+		}
+	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
+
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
+	var paths []string
+	if v4Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
+	}
+	if v6Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
+	}
+
+	// In some common configurations (e.g. default docker,
+	// kubernetes), the container environment denies write access to
+	// most sysctls, including IP forwarding controls. Check the
+	// sysctl values before trying to change them, so that we
+	// gracefully do nothing if the container's already been set up
+	// properly by e.g. a k8s initContainer.
+	for _, path := range paths {
+		bs, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %q: %w", path, err)
+		}
+		if v := strings.TrimSpace(string(bs)); v != "1" {
+			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
+				return fmt.Errorf("enabling %q: %w", path, err)
+			}
+		}
+	}
+	return nil
+}
+
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	return nil
+}
+
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(_ context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	proxyHasIPv4Address := false
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.AddDNATRule(local, dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRuleForDNSTarget(_ context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}

cmd/containerboot/healthz.go

@@ -0,0 +1,51 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"log"
+	"net"
+	"net/http"
+	"sync"
+)
+
+// healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type healthz struct {
+	sync.Mutex
+	hasAddrs bool
+}
+
+func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+	if h.hasAddrs {
+		w.Write([]byte("ok"))
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
+	}
+}
+
+// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
+// provided address. A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func runHealthz(addr string, h *healthz) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", h)
+	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
+	hs := &http.Server{Handler: mux}
+
+	go func() {
+		if err := hs.Serve(lis); err != nil {
+			log.Fatalf("failed running health endpoint: %v", err)
+		}
+	}()
+}

cmd/containerboot/kube.go

@@ -8,21 +8,21 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 	"tailscale.com/tailcfg"
 )
 
 // storeDeviceID writes deviceID to 'device_id' data field of the named
 // Kubernetes Secret.
 func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
 			"device_id": []byte(deviceID),
 		},
@@ -42,7 +42,7 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 		return err
 	}
 
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
 			"device_fqdn": []byte(fqdn),
 			"device_ips":  deviceIPs,
@@ -55,14 +55,14 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 // secret. No-op if there is no authkey in the secret.
 func deleteAuthKey(ctx context.Context, secretName string) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
-	m := []kube.JSONPatch{
+	m := []kubeclient.JSONPatch{
 		{
 			Op:   "remove",
 			Path: "/data/authkey",
 		},
 	}
 	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
-		if s, ok := err.(*kube.Status); ok && s.Code == http.StatusUnprocessableEntity {
+		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
 			return nil
@@ -72,66 +72,16 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kube.Client
-
-// setupKube is responsible for doing any necessary configuration and checks to
-// ensure that tailscale state storage and authentication mechanism will work on
-// Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
-	if cfg.KubeSecret == "" {
-		return nil
-	}
-	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
-	if err != nil {
-		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-	}
-	cfg.KubernetesCanPatch = canPatch
-
-	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
-	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
-		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
-			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
-			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
-			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
-	} else if err != nil && !kube.IsNotFoundErr(err) {
-		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
-	}
-
-	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-		if s == nil {
-			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
-			return nil
-		}
-		keyBytes, _ := s.Data["authkey"]
-		key := string(keyBytes)
-
-		if key != "" {
-			// This behavior of pulling authkeys from kube secrets was added
-			// at the same time as the patch permission, so we can enforce
-			// that we must be able to patch out the authkey after
-			// authenticating if you want to use this feature. This avoids
-			// us having to deal with the case where we might leave behind
-			// an unnecessary reusable authkey in a secret, like a rake in
-			// the grass.
-			if !cfg.KubernetesCanPatch {
-				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-			}
-			cfg.AuthKey = key
-		} else {
-			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-		}
-	}
-	return nil
-}
+var kc kubeclient.Client
 
 func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.
-		kube.SetRootPathForTesting(root)
+		kubeclient.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kube.New()
+	kc, err = kubeclient.New()
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/kube_test.go

@@ -11,7 +11,8 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 )
 
 func TestSetupKube(t *testing.T) {
@@ -20,7 +21,7 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kube.Client
+		kc      kubeclient.Client
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
@@ -28,11 +29,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, nil
 				},
 			},
@@ -47,12 +48,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -66,12 +67,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -86,12 +87,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 403}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 403}
 				},
 			},
 			wantCfg: &settings{
@@ -110,7 +111,7 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
@@ -126,12 +127,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 		},
@@ -144,12 +145,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{}, nil
 				},
 			},
 		},
@@ -158,12 +159,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{
@@ -176,12 +177,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{

cmd/containerboot/main.go

@@ -52,6 +52,12 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
+//   - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
+//     served at /healthz at the provided address, which should be in form [<address>]:<port>.
+//     If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
+//     The health endpoint will return 200 OK if this node has at least one tailnet IP address,
+//     otherwise returns 503.
+//     NB: the health criteria might change in the future.
 //   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
 //     directory that containers tailscaled config in file. The config file needs to be
 //     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
@@ -86,9 +92,7 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -97,24 +101,19 @@ import (
 	"net"
 	"net/netip"
 	"os"
-	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
-	"reflect"
 	"slices"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"golang.org/x/sys/unix"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
-	"tailscale.com/ipn/conffile"
 	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -158,6 +157,7 @@ func main() {
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
 		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -349,6 +349,9 @@ authLoop:
 
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
+
+		h             = &healthz{} // http server for the healthz endpoint
+		healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
 	)
 	if cfg.ServeConfigPath != "" {
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
@@ -565,6 +568,13 @@ runLoop:
 						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
+
+				if cfg.HealthCheckAddrPort != "" {
+					h.Lock()
+					h.hasAddrs = len(addrs) != 0
+					h.Unlock()
+					healthzRunner()
+				}
 			}
 			if !startupTasksDone {
 				// For containerboot instances that act as TCP
@@ -632,221 +642,6 @@ runLoop:
 	wg.Wait()
 }
 
-// watchServeConfigChanges watches path for changes, and when it sees one, reads
-// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
-// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
-// is written to when the certDomain changes, causing the serve config to be
-// re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
-	if certDomainAtomic == nil {
-		panic("cd must not be nil")
-	}
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
-			log.Fatalf("failed to add fsnotify watch: %v", err)
-		}
-		eventChan = w.Events
-	}
-
-	var certDomain string
-	var prevServeConfig *ipn.ServeConfig
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-cdChanged:
-			certDomain = *certDomainAtomic.Load()
-		case <-tickChan:
-		case <-eventChan:
-			// We can't do any reasonable filtering on the event because of how
-			// k8s handles these mounts. So just re-read the file and apply it
-			// if it's changed.
-		}
-		if certDomain == "" {
-			continue
-		}
-		sc, err := readServeConfig(path, certDomain)
-		if err != nil {
-			log.Fatalf("failed to read serve config: %v", err)
-		}
-		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
-			continue
-		}
-		log.Printf("Applying serve config")
-		if err := lc.SetServeConfig(ctx, sc); err != nil {
-			log.Fatalf("failed to set serve config: %v", err)
-		}
-		prevServeConfig = sc
-	}
-}
-
-// readServeConfig reads the ipn.ServeConfig from path, replacing
-// ${TS_CERT_DOMAIN} with certDomain.
-func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
-	if path == "" {
-		return nil, nil
-	}
-	j, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
-	var sc ipn.ServeConfig
-	if err := json.Unmarshal(j, &sc); err != nil {
-		return nil, err
-	}
-	return &sc, nil
-}
-
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
-	args := tailscaledArgs(cfg)
-	// tailscaled runs without context, since it needs to persist
-	// beyond the startup timeout in ctx.
-	cmd := exec.Command("tailscaled", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-	log.Printf("Starting tailscaled")
-	if err := cmd.Start(); err != nil {
-		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
-	}
-
-	// Wait for the socket file to appear, otherwise API ops will racily fail.
-	log.Printf("Waiting for tailscaled socket")
-	for {
-		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
-		}
-		_, err := os.Stat(cfg.Socket)
-		if errors.Is(err, fs.ErrNotExist) {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
-		}
-		break
-	}
-
-	tsClient := &tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
-	return tsClient, cmd.Process, nil
-}
-
-// tailscaledArgs uses cfg to construct the argv for tailscaled.
-func tailscaledArgs(cfg *settings) []string {
-	args := []string{"--socket=" + cfg.Socket}
-	switch {
-	case cfg.InKubernetes && cfg.KubeSecret != "":
-		args = append(args, "--state=kube:"+cfg.KubeSecret)
-		if cfg.StateDir == "" {
-			cfg.StateDir = "/tmp"
-		}
-		fallthrough
-	case cfg.StateDir != "":
-		args = append(args, "--statedir="+cfg.StateDir)
-	default:
-		args = append(args, "--state=mem:", "--statedir=/tmp")
-	}
-
-	if cfg.UserspaceMode {
-		args = append(args, "--tun=userspace-networking")
-	} else if err := ensureTunFile(cfg.Root); err != nil {
-		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
-	}
-
-	if cfg.SOCKSProxyAddr != "" {
-		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
-	}
-	if cfg.HTTPProxyAddr != "" {
-		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
-	}
-	if cfg.TailscaledConfigFilePath != "" {
-		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
-	}
-	if cfg.DaemonExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
-	}
-	return args
-}
-
-// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
-// if TS_AUTH_ONCE is set, only the first time containerboot starts.
-func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	if cfg.AuthKey != "" {
-		args = append(args, "--authkey="+cfg.AuthKey)
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
-	}
-	return nil
-}
-
-// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
-// options that are passed in via environment variables. This is run after the
-// node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}
-
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
 func ensureTunFile(root string) error {
@@ -866,344 +661,6 @@ func ensureTunFile(root string) error {
 	return nil
 }
 
-// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
-	var (
-		v4Forwarding, v6Forwarding bool
-	)
-	if clusterProxyTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid cluster destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	if tailnetTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid tailnet destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	// Currently we only proxy traffic to the IPv4 address of the tailnet
-	// target.
-	if tailnetTargetFQDN != "" {
-		v4Forwarding = true
-	}
-	if routes != nil && *routes != "" {
-		for _, route := range strings.Split(*routes, ",") {
-			cidr, err := netip.ParsePrefix(route)
-			if err != nil {
-				return fmt.Errorf("invalid subnet route: %v", err)
-			}
-			if cidr.Addr().Is4() {
-				v4Forwarding = true
-			} else {
-				v6Forwarding = true
-			}
-		}
-	}
-	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
-}
-
-func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
-	var paths []string
-	if v4Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
-	}
-	if v6Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
-	}
-
-	// In some common configurations (e.g. default docker,
-	// kubernetes), the container environment denies write access to
-	// most sysctls, including IP forwarding controls. Check the
-	// sysctl values before trying to change them, so that we
-	// gracefully do nothing if the container's already been set up
-	// properly by e.g. a k8s initContainer.
-	for _, path := range paths {
-		bs, err := os.ReadFile(path)
-		if err != nil {
-			return fmt.Errorf("reading %q: %w", path, err)
-		}
-		if v := strings.TrimSpace(string(bs)); v != "1" {
-			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
-				return fmt.Errorf("enabling %q: %w", path, err)
-			}
-		}
-	}
-	return nil
-}
-
-func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	return nil
-}
-
-// installTSForwardingRuleForDestination accepts a destination address and a
-// list of node's tailnet addresses, sets up rules to forward traffic for
-// destination to the tailnet IP matching the destination IP family.
-// Destination can be Pod IP of this node.
-func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstFilter)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
-	}
-	if err := nfr.AddDNATRule(dst, local); err != nil {
-		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	proxyHasIPv4Address := false
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() {
-			proxyHasIPv4Address = true
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if proxyHasIPv4Address && dst.Is6() {
-		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.AddDNATRule(local, dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRuleForDNSTarget(ctx context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	var (
-		tsv4       netip.Addr
-		tsv6       netip.Addr
-		v4Backends []netip.Addr
-		v6Backends []netip.Addr
-	)
-	for _, pfx := range tsIPs {
-		if pfx.IsSingleIP() && pfx.Addr().Is4() {
-			tsv4 = pfx.Addr()
-			continue
-		}
-		if pfx.IsSingleIP() && pfx.Addr().Is6() {
-			tsv6 = pfx.Addr()
-			continue
-		}
-	}
-	// TODO: log if more than one backend address is found and firewall is
-	// in nftables mode that only the first IP will be used.
-	for _, ip := range backendAddrs {
-		if ip.To4() != nil {
-			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
-		}
-		if ip.To16() != nil {
-			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
-		}
-	}
-
-	// Enable IP forwarding here as opposed to at the start of containerboot
-	// as the IPv4/IPv6 requirements might have changed.
-	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
-	// enabled by an init container, so in practice enabling forwarding here
-	// is only needed if this proxy has been configured by manually setting
-	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
-	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
-		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
-	}
-
-	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
-		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
-			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
-		}
-		// The backend might advertize MSS higher than that of the
-		// tailscale interfaces. Clamp MSS of packets going out via
-		// tailscale0 interface to its MTU to prevent broken connections
-		// in environments where path MTU discovery is not working.
-		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
-		}
-		return nil
-	}
-
-	if len(v4Backends) != 0 {
-		if !tsv4.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
-		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
-			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
-		}
-	}
-	if len(v6Backends) != 0 && !tsv6.IsValid() {
-		if !tsv6.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
-		} else if !nfr.HasIPV6NAT() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
-		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
-			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
-		}
-	}
-	return nil
-}
-
-// settings is all the configuration for containerboot.
-type settings struct {
-	AuthKey  string
-	Hostname string
-	Routes   *string
-	// ProxyTargetIP is the destination IP to which all incoming
-	// Tailscale traffic should be proxied. If empty, no proxying
-	// is done. This is typically a locally reachable IP.
-	ProxyTargetIP string
-	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
-	// incoming Tailscale traffic should be proxied.
-	ProxyTargetDNSName string
-	// TailnetTargetIP is the destination IP to which all incoming
-	// non-Tailscale traffic should be proxied. This is typically a
-	// Tailscale IP.
-	TailnetTargetIP string
-	// TailnetTargetFQDN is an MagicDNS name to which all incoming
-	// non-Tailscale traffic should be proxied. This must be a full Tailnet
-	// node FQDN.
-	TailnetTargetFQDN             string
-	ServeConfigPath               string
-	DaemonExtraArgs               string
-	ExtraArgs                     string
-	InKubernetes                  bool
-	UserspaceMode                 bool
-	StateDir                      string
-	AcceptDNS                     *bool
-	KubeSecret                    string
-	SOCKSProxyAddr                string
-	HTTPProxyAddr                 string
-	Socket                        string
-	AuthOnce                      bool
-	Root                          string
-	KubernetesCanPatch            bool
-	TailscaledConfigFilePath      string
-	EnableForwardingOptimizations bool
-	// If set to true and, if this containerboot instance is a Kubernetes
-	// ingress proxy, set up rules to forward incoming cluster traffic to be
-	// forwarded to the ingress target in cluster.
-	AllowProxyingClusterTrafficViaIngress bool
-	// PodIP is the IP of the Pod if running in Kubernetes. This is used
-	// when setting up rules to proxy cluster traffic to cluster ingress
-	// target.
-	PodIP string
-}
-
-func (s *settings) validate() error {
-	if s.TailscaledConfigFilePath != "" {
-		dir, file := path.Split(s.TailscaledConfigFilePath)
-		if _, err := os.Stat(dir); err != nil {
-			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
-		}
-		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
-		}
-		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
-		}
-	}
-	if s.ProxyTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
-	}
-	if s.TailnetTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
-		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
-	}
-	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
-	}
-	if s.EnableForwardingOptimizations && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
-	}
-	return nil
-}
-
 func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	// TODO (irbekrm): look at using recursive.Resolver instead to resolve
 	// the DNS names as well as retrieve TTLs. It looks though that this
@@ -1226,57 +683,6 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	return append(ip4s, ip6s...), nil
 }
 
-// defaultEnv returns the value of the given envvar name, or defVal if
-// unset.
-func defaultEnv(name, defVal string) string {
-	if v, ok := os.LookupEnv(name); ok {
-		return v
-	}
-	return defVal
-}
-
-// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being set to empty string vs unset.
-func defaultEnvStringPointer(name string) *string {
-	if v, ok := os.LookupEnv(name); ok {
-		return &v
-	}
-	return nil
-}
-
-// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being explicitly set to false vs unset.
-func defaultEnvBoolPointer(name string) *bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return nil
-	}
-	return &ret
-}
-
-func defaultEnvs(names []string, defVal string) string {
-	for _, name := range names {
-		if v, ok := os.LookupEnv(name); ok {
-			return v
-		}
-	}
-	return defVal
-}
-
-// defaultBool returns the boolean value of the given envvar name, or
-// defVal if unset or not a bool.
-func defaultBool(name string, defVal bool) bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return defVal
-	}
-	return ret
-}
-
 // contextWithExitSignalWatch watches for SIGTERM/SIGINT signals. It returns a
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
@@ -1299,43 +705,6 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 	return ctx, f
 }
 
-// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
-// in two steps and login should only happen once.
-// Step 1: run 'tailscaled'
-// Step 2):
-// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
-// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
-func isTwoStepConfigAuthOnce(cfg *settings) bool {
-	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
-// in two steps and we should log in every time it starts.
-// Step 1: run 'tailscaled'
-// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
-func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
-	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isOneStepConfig returns true if the Tailscale node should always be ran and
-// configured in a single step by running 'tailscaled <config opts>'
-func isOneStepConfig(cfg *settings) bool {
-	return cfg.TailscaledConfigFilePath != ""
-}
-
-// isL3Proxy returns true if the Tailscale node needs to be configured to act
-// as an L3 proxy, proxying to an endpoint provided via one of the config env
-// vars.
-func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
-}
-
-// hasKubeStateStore returns true if the state must be stored in a Kubernetes
-// Secret.
-func hasKubeStateStore(cfg *settings) bool {
-	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
-}
-
 // tailscaledConfigFilePath returns the path to the tailscaled config file that
 // should be used for the current capability version. It is determined by the
 // TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a

cmd/containerboot/serve.go

@@ -0,0 +1,96 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+)
+
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+	if certDomainAtomic == nil {
+		panic("cd must not be nil")
+	}
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(path)); err != nil {
+			log.Fatalf("failed to add fsnotify watch: %v", err)
+		}
+		eventChan = w.Events
+	}
+
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-eventChan:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		if certDomain == "" {
+			continue
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("failed to read serve config: %v", err)
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		log.Printf("Applying serve config")
+		if err := lc.SetServeConfig(ctx, sc); err != nil {
+			log.Fatalf("failed to set serve config: %v", err)
+		}
+		prevServeConfig = sc
+	}
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}

cmd/containerboot/settings.go

@@ -0,0 +1,259 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"path"
+	"strconv"
+
+	"tailscale.com/ipn/conffile"
+	"tailscale.com/kube/kubeclient"
+)
+
+// settings is all the configuration for containerboot.
+type settings struct {
+	AuthKey  string
+	Hostname string
+	Routes   *string
+	// ProxyTargetIP is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. This is typically a
+	// Tailscale IP.
+	TailnetTargetIP string
+	// TailnetTargetFQDN is an MagicDNS name to which all incoming
+	// non-Tailscale traffic should be proxied. This must be a full Tailnet
+	// node FQDN.
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	PodIP               string
+	HealthCheckAddrPort string
+}
+
+func (s *settings) validate() error {
+	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
+		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
+		}
+	}
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
+	if s.TailnetTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
+		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
+	}
+	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
+	if s.HealthCheckAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+		}
+	}
+	return nil
+}
+
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kubeclient.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kubeclient.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
+
+// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
+// in two steps and login should only happen once.
+// Step 1: run 'tailscaled'
+// Step 2):
+// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
+// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
+func isTwoStepConfigAuthOnce(cfg *settings) bool {
+	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
+// in two steps and we should log in every time it starts.
+// Step 1: run 'tailscaled'
+// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
+func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
+	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isOneStepConfig returns true if the Tailscale node should always be ran and
+// configured in a single step by running 'tailscaled <config opts>'
+func isOneStepConfig(cfg *settings) bool {
+	return cfg.TailscaledConfigFilePath != ""
+}
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
+// defaultEnv returns the value of the given envvar name, or defVal if
+// unset.
+func defaultEnv(name, defVal string) string {
+	if v, ok := os.LookupEnv(name); ok {
+		return v
+	}
+	return defVal
+}
+
+// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being set to empty string vs unset.
+func defaultEnvStringPointer(name string) *string {
+	if v, ok := os.LookupEnv(name); ok {
+		return &v
+	}
+	return nil
+}
+
+// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being explicitly set to false vs unset.
+func defaultEnvBoolPointer(name string) *bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return nil
+	}
+	return &ret
+}
+
+func defaultEnvs(names []string, defVal string) string {
+	for _, name := range names {
+		if v, ok := os.LookupEnv(name); ok {
+			return v
+		}
+	}
+	return defVal
+}
+
+// defaultBool returns the boolean value of the given envvar name, or
+// defVal if unset or not a bool.
+func defaultBool(name string, defVal bool) bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return defVal
+	}
+	return ret
+}

cmd/containerboot/tailscaled.go

@@ -0,0 +1,162 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+
+	"tailscale.com/client/tailscale"
+)
+
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+	args := tailscaledArgs(cfg)
+	// tailscaled runs without context, since it needs to persist
+	// beyond the startup timeout in ctx.
+	cmd := exec.Command("tailscaled", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	log.Printf("Starting tailscaled")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
+	}
+
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
+	log.Printf("Waiting for tailscaled socket")
+	for {
+		if ctx.Err() != nil {
+			log.Fatalf("Timed out waiting for tailscaled socket")
+		}
+		_, err := os.Stat(cfg.Socket)
+		if errors.Is(err, fs.ErrNotExist) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		} else if err != nil {
+			log.Fatalf("Waiting for tailscaled socket: %v", err)
+		}
+		break
+	}
+
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process, nil
+}
+
+// tailscaledArgs uses cfg to construct the argv for tailscaled.
+func tailscaledArgs(cfg *settings) []string {
+	args := []string{"--socket=" + cfg.Socket}
+	switch {
+	case cfg.InKubernetes && cfg.KubeSecret != "":
+		args = append(args, "--state=kube:"+cfg.KubeSecret)
+		if cfg.StateDir == "" {
+			cfg.StateDir = "/tmp"
+		}
+		fallthrough
+	case cfg.StateDir != "":
+		args = append(args, "--statedir="+cfg.StateDir)
+	default:
+		args = append(args, "--state=mem:", "--statedir=/tmp")
+	}
+
+	if cfg.UserspaceMode {
+		args = append(args, "--tun=userspace-networking")
+	} else if err := ensureTunFile(cfg.Root); err != nil {
+		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
+	}
+
+	if cfg.SOCKSProxyAddr != "" {
+		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
+	}
+	if cfg.HTTPProxyAddr != "" {
+		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
+	}
+	if cfg.TailscaledConfigFilePath != "" {
+		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
+	}
+	if cfg.DaemonExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
+	}
+	return args
+}
+
+// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
+// if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleUp(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "up"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale up'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale up failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state and only if TS_AUTH_ONCE is set.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	log.Printf("Running 'tailscale set'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale set failed: %v", err)
+	}
+	return nil
+}

cmd/derper/depaware.txt

@@ -7,10 +7,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/cmd/derper+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -48,7 +52,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
@@ -82,10 +86,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoiface                from google.golang.org/protobuf/internal/impl+
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
-        nhooyr.io/websocket                                          from tailscale.com/cmd/derper+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -99,6 +99,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
@@ -127,7 +128,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -146,9 +147,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -159,6 +162,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
@@ -171,15 +177,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from tailscale.com/tka
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -251,6 +259,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/ipn+
@@ -266,7 +275,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -295,3 +304,4 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/derper/websocket.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"strings"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/derp"
 	"tailscale.com/net/wsconn"
 )

cmd/derpprobe/derpprobe.go

@@ -7,8 +7,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"html"
-	"io"
 	"log"
 	"net/http"
 	"sort"
@@ -70,8 +68,13 @@ func main() {
 	}
 
 	mux := http.NewServeMux()
-	tsweb.Debugger(mux)
-	mux.HandleFunc("/", http.HandlerFunc(serveFunc(p)))
+	d := tsweb.Debugger(mux)
+	d.Handle("probe-run", "Run a probe", tsweb.StdHandler(tsweb.ReturnHandlerFunc(p.RunHandler), tsweb.HandlerOptions{Logf: log.Printf}))
+	mux.Handle("/", tsweb.StdHandler(p.StatusHandler(
+		prober.WithTitle("DERP Prober"),
+		prober.WithPageLink("Prober metrics", "/debug/varz"),
+		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
+	), tsweb.HandlerOptions{Logf: log.Printf}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }
@@ -105,26 +108,3 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 	sort.Strings(o.good)
 	return
 }
-
-func serveFunc(p *prober.Prober) func(w http.ResponseWriter, r *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		st := getOverallStatus(p)
-		summary := "All good"
-		if (float64(len(st.bad)) / float64(len(st.bad)+len(st.good))) > 0.25 {
-			// Returning a 500 allows monitoring this server externally and configuring
-			// an alert on HTTP response code.
-			w.WriteHeader(500)
-			summary = fmt.Sprintf("%d problems", len(st.bad))
-		}
-
-		io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
-		fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
-		for _, s := range st.bad {
-			fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
-		}
-		for _, s := range st.good {
-			fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
-		}
-		io.WriteString(w, "</ul></body></html>\n")
-	}
-}

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/k8s-operator/connector.go

@@ -26,6 +26,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -61,11 +62,11 @@ type ConnectorReconciler struct {
 
 var (
 	// gaugeConnectorResources tracks the overall number of Connectors currently managed by this operator instance.
-	gaugeConnectorResources = clientmetric.NewGauge("k8s_connector_resources")
+	gaugeConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorResourceCount)
 	// gaugeConnectorSubnetRouterResources tracks the number of Connectors managed by this operator instance that are subnet routers.
-	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
+	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
-	gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
+	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {

cmd/k8s-operator/connector_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/util/mak"
 )
@@ -74,6 +75,7 @@ func TestConnector(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -169,6 +171,7 @@ func TestConnector(t *testing.T) {
 		parentType:   "connector",
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -254,6 +257,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)

cmd/k8s-operator/depaware.txt

@@ -5,7 +5,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
-  LD    github.com/anmitsu/go-shlex                                  from tailscale.com/tempfork/gliderlabs/ssh
    L    github.com/aws/aws-sdk-go-v2/aws                             from github.com/aws/aws-sdk-go-v2/aws/defaults+
    L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/aws/defaults                    from github.com/aws/aws-sdk-go-v2/service/ssm+
@@ -81,8 +80,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
-  LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
    W 💣 github.com/dblohm7/wingoes/com                               from tailscale.com/util/osdiag+
@@ -98,7 +100,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -113,7 +115,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/go-openapi/jsonreference                          from k8s.io/kube-openapi/pkg/internal+
         github.com/go-openapi/jsonreference/internal                 from github.com/go-openapi/jsonreference
         github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
-   L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns+
+   L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
      💣 github.com/gogo/protobuf/proto                               from k8s.io/api/admission/v1+
         github.com/gogo/protobuf/sortkeys                            from k8s.io/api/admission/v1+
         github.com/golang/groupcache/lru                             from k8s.io/client-go/tools/record+
@@ -141,7 +143,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
+   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
         github.com/imdario/mergo                                     from k8s.io/client-go/tools/clientcmd
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
@@ -161,7 +163,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
         github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
-  LD    github.com/kr/fs                                             from github.com/pkg/sftp
         github.com/mailru/easyjson/buffer                            from github.com/mailru/easyjson/jwriter
      💣 github.com/mailru/easyjson/jlexer                            from github.com/go-openapi/swag
         github.com/mailru/easyjson/jwriter                           from github.com/go-openapi/swag
@@ -170,7 +171,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink+
         github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
         github.com/modern-go/concurrent                              from github.com/json-iterator/go
@@ -183,8 +184,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/pierrec/lz4/v4/internal/lz4stream                 from github.com/pierrec/lz4/v4
    L    github.com/pierrec/lz4/v4/internal/xxh32                     from github.com/pierrec/lz4/v4/internal/lz4stream
         github.com/pkg/errors                                        from github.com/evanphx/json-patch/v5+
-  LD    github.com/pkg/sftp                                          from tailscale.com/ssh/tailssh
-  LD    github.com/pkg/sftp/internal/encoding/ssh/filexfer           from github.com/pkg/sftp
    D    github.com/prometheus-community/pro-bing                     from tailscale.com/wgengine/netstack
      💣 github.com/prometheus/client_golang/prometheus               from github.com/prometheus/client_golang/prometheus/collectors+
         github.com/prometheus/client_golang/prometheus/collectors    from sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics
@@ -207,7 +206,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
         github.com/tailscale/golang-x-crypto/acme                    from tailscale.com/ipn/ipnlocal
   LD    github.com/tailscale/golang-x-crypto/internal/poly1305       from github.com/tailscale/golang-x-crypto/ssh
-  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal+
+  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal
   LD    github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh
         github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2+
         github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
@@ -217,6 +216,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tailscale/hujson                                  from tailscale.com/ipn/conffile
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/net/routetable+
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/peercred                                from tailscale.com/ipn/ipnauth
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
      💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
@@ -230,10 +230,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device
      💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
-  LD    github.com/u-root/u-root/pkg/termios                         from tailscale.com/ssh/tailssh
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
         go.uber.org/multierr                                         from go.uber.org/zap+
@@ -307,7 +305,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
         gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
-        gvisor.dev/gvisor/pkg/tcpip/link/channel                     from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
         gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
@@ -317,6 +314,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack/gro
         gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
         gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
@@ -423,6 +421,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/util/naming                          from k8s.io/apimachinery/pkg/runtime+
         k8s.io/apimachinery/pkg/util/net                             from k8s.io/apimachinery/pkg/watch+
         k8s.io/apimachinery/pkg/util/rand                            from k8s.io/apiserver/pkg/storage/names
+        k8s.io/apimachinery/pkg/util/remotecommand                   from tailscale.com/k8s-operator/sessionrecording/ws
         k8s.io/apimachinery/pkg/util/runtime                         from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
         k8s.io/apimachinery/pkg/util/sets                            from k8s.io/apimachinery/pkg/api/meta+
         k8s.io/apimachinery/pkg/util/strategicpatch                  from k8s.io/client-go/tools/record+
@@ -600,10 +599,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
         k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
         k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
         sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
         sigs.k8s.io/controller-runtime/pkg/cache/internal            from sigs.k8s.io/controller-runtime/pkg/cache
@@ -660,7 +655,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
         tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
-  LD    tailscale.com/cmd/tailscaled/childproc                       from tailscale.com/ssh/tailssh
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
@@ -692,7 +686,13 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/k8s-operator                                   from tailscale.com/cmd/k8s-operator
         tailscale.com/k8s-operator/apis                              from tailscale.com/k8s-operator/apis/v1alpha1
         tailscale.com/k8s-operator/apis/v1alpha1                     from tailscale.com/cmd/k8s-operator+
-        tailscale.com/kube                                           from tailscale.com/cmd/k8s-operator+
+        tailscale.com/k8s-operator/sessionrecording                  from tailscale.com/cmd/k8s-operator
+        tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
+        tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
+        tailscale.com/k8s-operator/sessionrecording/ws               from tailscale.com/k8s-operator/sessionrecording
+        tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
+        tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
+        tailscale.com/kube/kubetypes                                 from tailscale.com/cmd/k8s-operator+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -701,6 +701,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns+
@@ -743,19 +744,19 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
-     💣 tailscale.com/ssh/tailssh                                    from tailscale.com/cmd/k8s-operator
+        tailscale.com/sessionrecording                               from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
-  LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
-        tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator
+        tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
@@ -797,7 +798,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/osdiag                                    from tailscale.com/ipn/localapi
    W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
         tailscale.com/util/osshare                                   from tailscale.com/ipn/ipnlocal
-        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal
         tailscale.com/util/progresstracking                          from tailscale.com/ipn/localapi
         tailscale.com/util/race                                      from tailscale.com/net/dns/resolver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -807,11 +808,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
         tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/usermetric                                from tailscale.com/health+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
@@ -828,6 +832,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/tsnet
+        tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -837,7 +842,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
-  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf+
+  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf
         golang.org/x/crypto/chacha20                                 from github.com/tailscale/golang-x-crypto/ssh+
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
@@ -846,9 +851,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-  LD    golang.org/x/crypto/ssh                                      from github.com/pkg/sftp+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
@@ -865,6 +870,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
+        golang.org/x/net/websocket                                   from tailscale.com/k8s-operator/sessionrecording/ws
         golang.org/x/oauth2                                          from golang.org/x/oauth2/clientcredentials+
         golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
         golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+
@@ -949,11 +955,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
+        iter                                                         from go/ast+
         log                                                          from expvar+
         log/internal                                                 from log+
         log/slog                                                     from github.com/go-logr/logr+
         log/slog/internal                                            from log/slog
-  LD    log/syslog                                                   from tailscale.com/ssh/tailssh
         maps                                                         from sigs.k8s.io/controller-runtime/pkg/predicate+
         math                                                         from archive/tar+
         math/big                                                     from crypto/dsa+
@@ -987,7 +993,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from encoding/base32+
-        sort                                                         from archive/tar+
+        sort                                                         from compress/flate+
         strconv                                                      from archive/tar+
         strings                                                      from archive/tar+
         sync                                                         from archive/tar+
@@ -1000,3 +1006,4 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -77,6 +77,10 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            {{- if .Values.proxyConfig.defaultProxyClass }}
+            - name: PROXY_DEFAULT_CLASS
+              value: {{ .Values.proxyConfig.defaultProxyClass }}
+            {{- end }}
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -14,10 +14,10 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["events", "services", "services/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingresses", "ingresses/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
@@ -27,6 +27,9 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["dnsconfigs", "dnsconfigs/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["recorders", "recorders/status"]
+  verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -49,13 +52,16 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
   verbs: ["get", "list", "watch"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings"]
+  verbs: ["get", "create", "patch", "update", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -15,7 +15,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -78,6 +78,9 @@ proxyConfig:
   # Note that if you pass multiple tags to this field via `--set` flag to helm upgrade/install commands you must escape the comma (for example, "tag:k8s-proxies\,tag:prod"). See https://github.com/helm/helm/issues/1556
   defaultTags: "tag:k8s"
   firewallMode: auto
+  # If defined, this proxy class will be used as the default proxy class for
+  # service and ingress resources that do not have a proxy class defined.
+  defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -89,14 +89,14 @@ spec:
                   type: object
                   properties:
                     image:
-                      description: Nameserver image.
+                      description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
                       type: object
                       properties:
                         repo:
                           description: Repo defaults to tailscale/k8s-nameserver.
                           type: string
                         tag:
-                          description: Tag defaults to operator's own tag.
+                          description: Tag defaults to unstable.
                           type: string
             status:
               description: |-

cmd/k8s-operator/deploy/examples/recorder.yaml

@@ -0,0 +1,6 @@
+apiVersion: tailscale.com/v1alpha1
+kind: Recorder
+metadata:
+  name: recorder
+spec:
+  enableUI: true

cmd/k8s-operator/dnsrecords.go

@@ -24,6 +24,7 @@ import (
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
 )
 
 const (
@@ -167,36 +168,49 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 		}
 	}
 
-	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
-	// headless Service.
+	// Get the Pod IP addresses for the proxy from the EndpointSlices for
+	// the headless Service. The Service can have multiple EndpointSlices
+	// associated with it, for example in dual-stack clusters.
 	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
-	if err != nil {
-		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
+	var eps = new(discoveryv1.EndpointSliceList)
+	if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error listing EndpointSlices for the proxy's headless Service: %w", err)
 	}
-	if eps == nil {
+	if len(eps.Items) == 0 {
 		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
 		return nil
 	}
-	// An EndpointSlice for a Service can have a list of endpoints that each
+	// Each EndpointSlice for a Service can have a list of endpoints that each
 	// can have multiple addresses - these are the IP addresses of any Pods
 	// selected by that Service. Pick all the IPv4 addresses.
-	ips := make([]string, 0)
-	for _, ep := range eps.Endpoints {
-		for _, ip := range ep.Addresses {
-			if !net.IsIPv4String(ip) {
-				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
-			} else {
-				ips = append(ips, ip)
+	// It is also possible that multiple EndpointSlices have overlapping addresses.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints
+	ips := make(set.Set[string], 0)
+	for _, slice := range eps.Items {
+		if slice.AddressType != discoveryv1.AddressTypeIPv4 {
+			logger.Infof("EndpointSlice is for AddressType %s, currently only IPv4 address type is supported", slice.AddressType)
+			continue
+		}
+		for _, ep := range slice.Endpoints {
+			if !epIsReady(&ep) {
+				logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String())
+				continue
+			}
+			for _, ip := range ep.Addresses {
+				if !net.IsIPv4String(ip) {
+					logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+				} else {
+					ips.Add(ip)
+				}
 			}
 		}
 	}
-	if len(ips) == 0 {
+	if ips.Len() == 0 {
 		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
 		return nil
 	}
 	updateFunc := func(rec *operatorutils.Records) {
-		mak.Set(&rec.IP4, fqdn, ips)
+		mak.Set(&rec.IP4, fqdn, ips.Slice())
 	}
 	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
 		return fmt.Errorf("error updating DNS records: %w", err)
@@ -204,6 +218,17 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 	return nil
 }
 
+// epIsReady reports whether the endpoint is currently in a state to receive new
+// traffic. As per kube docs, only explicitly set 'false' for 'Ready' or
+// 'Serving' conditions or explicitly set 'true' for 'Terminating' condition
+// means that the Endpoint is NOT ready.
+// https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/apis/discovery/types.go#L109-L131
+func epIsReady(ep *discoveryv1.Endpoint) bool {
+	return (ep.Conditions.Ready == nil || *ep.Conditions.Ready) &&
+		(ep.Conditions.Serving == nil || *ep.Conditions.Serving) &&
+		(ep.Conditions.Terminating == nil || !*ep.Conditions.Terminating)
+}
+
 // maybeCleanup ensures that the DNS record for the proxy has been removed from
 // dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
 // has been removed from the Service. If the record is not found in the

cmd/k8s-operator/dnsrecords_test.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -87,13 +88,16 @@ func TestDNSRecordsReconciler(t *testing.T) {
 		},
 	}
 	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
-	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4)
+	epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6)
+
 	mustCreate(t, fc, egressSvcFQDN)
 	mustCreate(t, fc, headlessForEgressSvcFQDN)
 	mustCreate(t, fc, ep)
+	mustCreate(t, fc, epv6)
 	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
 	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
-	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} // IPv6 endpoint is currently ignored
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
@@ -106,7 +110,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 3. DNS record is updated if the IP address of the proxy Pod changes.
-	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4)
 	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
 		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
 	})
@@ -116,7 +120,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 
 	// 4. DNS record is created for an ingress proxy configured via Ingress
 	headlessForIngress := headlessSvcForParent(ing, "ingress")
-	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4)
 	mustCreate(t, fc, headlessForIngress)
 	mustCreate(t, fc, ep)
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
@@ -140,6 +144,17 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
 	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
 	expectHostsRecords(t, fc, wantHosts)
+
+	// 7. A not-ready Endpoint is removed from DNS config.
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Conditions.Ready = ptr.To(false)
+		ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{
+			Addresses: []string{"1.2.3.4"},
+		})
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
+	wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"}
+	expectHostsRecords(t, fc, wantHosts)
 }
 
 func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
@@ -162,15 +177,21 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 	}
 }
 
-func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
+func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.AddressType) *discoveryv1.EndpointSlice {
 	return &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      svc.Name,
+			Name:      fmt.Sprintf("%s-%s", svc.Name, string(fam)),
 			Namespace: svc.Namespace,
 			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
 		},
+		AddressType: fam,
 		Endpoints: []discoveryv1.Endpoint{{
 			Addresses: []string{ip},
+			Conditions: discoveryv1.EndpointConditions{
+				Ready:       ptr.To(true),
+				Serving:     ptr.To(true),
+				Terminating: ptr.To(false),
+			},
 		}},
 	}
 }

cmd/k8s-operator/generate/main.go

@@ -24,10 +24,12 @@ const (
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
+	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
+	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -111,7 +113,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass, DNSConfig, Recorder) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -137,28 +139,32 @@ func generate(baseDir string) error {
 		}
 		return nil
 	}
-	if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
+	for _, crd := range []struct {
+		crdPath, templatePath string
+	}{
+		{connectorCRDPath, connectorCRDHelmTemplatePath},
+		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
+		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
+		{recorderCRDPath, recorderCRDHelmTemplatePath},
+	} {
+		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
+			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
+		}
 	}
 	return nil
 }
 
 func cleanup(baseDir string) error {
 	log.Print("Cleaning up CRD from Helm templates")
-	if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up Connector CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
+	for _, path := range []string{
+		connectorCRDHelmTemplatePath,
+		proxyClassCRDHelmTemplatePath,
+		dnsConfigCRDHelmTemplatePath,
+		recorderCRDHelmTemplatePath,
+	} {
+		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("error cleaning up %s: %w", path, err)
+		}
 	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -59,6 +59,9 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -77,4 +80,7 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/ingress.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -46,12 +47,14 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
+
+	proxyDefaultClass string
 }
 
 var (
 	// gaugeIngressResources tracks the number of ingress resources that we're
 	// currently managing.
-	gaugeIngressResources = clientmetric.NewGauge("k8s_ingress_resources")
+	gaugeIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressResourceCount)
 )
 
 func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
@@ -133,7 +136,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing)
+	proxyClass := proxyClassForObject(ing, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/ingress_test.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
@@ -93,6 +94,7 @@ func TestTailscaleIngress(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -224,6 +226,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},

cmd/k8s-operator/nameserver.go

@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/yaml"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -62,9 +63,7 @@ type NameserverReconciler struct {
 	managedNameservers set.Slice[types.UID] // one or none
 }
 
-var (
-	gaugeNameserverResources = clientmetric.NewGauge("k8s_nameserver_resources")
-)
+var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
 
 func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := a.logger.With("dnsConfig", req.Name)

cmd/k8s-operator/nameserver_test.go

@@ -33,7 +33,7 @@ func TestNameserverReconciler(t *testing.T) {
 		},
 		Spec: tsapi.DNSConfigSpec{
 			Nameserver: &tsapi.Nameserver{
-				Image: &tsapi.Image{
+				Image: &tsapi.NameserverImage{
 					Repo: "test",
 					Tag:  "v0.0.1",
 				},

cmd/k8s-operator/operator.go

@@ -22,6 +22,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -39,6 +40,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/kubestore"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
 	"tailscale.com/types/logger"
@@ -51,7 +53,8 @@ import (
 // Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
 //go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
 
-// TODO (irbekrm): generate CRD docs from the yamls
+// Generate CRD API docs.
+//go:generate go run github.com/elastic/crd-ref-docs --renderer=markdown --source-path=../../k8s-operator/apis/ --config=../../k8s-operator/api-docs-config.yaml --output-path=../../k8s-operator/api.md
 
 func main() {
 	// Required to use our client API. We're fine with the instability since the
@@ -65,6 +68,7 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
@@ -85,9 +89,9 @@ func main() {
 	// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
 	mode := parseAPIProxyMode()
 	if mode == apiserverProxyModeDisabled {
-		hostinfo.SetApp("k8s-operator")
+		hostinfo.SetApp(kubetypes.AppOperator)
 	} else {
-		hostinfo.SetApp("k8s-operator-proxy")
+		hostinfo.SetApp(kubetypes.AppAPIServerProxy)
 	}
 
 	s, tsClient := initTSNet(zlog)
@@ -105,6 +109,7 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
+		proxyDefaultClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -237,6 +242,8 @@ func runReconcilers(opts reconcilerOpts) {
 				&appsv1.StatefulSet{}:        nsFilter,
 				&appsv1.Deployment{}:         nsFilter,
 				&discoveryv1.EndpointSlice{}: nsFilter,
+				&rbacv1.Role{}:               nsFilter,
+				&rbacv1.RoleBinding{}:        nsFilter,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
@@ -278,6 +285,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
+			proxyDefaultClass:     opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -296,10 +304,11 @@ func runReconcilers(opts reconcilerOpts) {
 		Watches(&corev1.Service{}, svcHandlerForIngress).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress).
 		Complete(&IngressReconciler{
-			ssr:      ssr,
-			recorder: eventRecorder,
-			Client:   mgr.GetClient(),
-			logger:   opts.log.Named("ingress-reconciler"),
+			ssr:               ssr,
+			recorder:          eventRecorder,
+			Client:            mgr.GetClient(),
+			logger:            opts.log.Named("ingress-reconciler"),
+			proxyDefaultClass: opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -383,6 +392,28 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create DNS records reconciler: %v", err)
 	}
+
+	// Recorder reconciler.
+	recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
+	err = builder.ControllerManagedBy(mgr).
+		For(&tsapi.Recorder{}).
+		Watches(&appsv1.StatefulSet{}, recorderFilter).
+		Watches(&corev1.ServiceAccount{}, recorderFilter).
+		Watches(&corev1.Secret{}, recorderFilter).
+		Watches(&rbacv1.Role{}, recorderFilter).
+		Watches(&rbacv1.RoleBinding{}, recorderFilter).
+		Complete(&RecorderReconciler{
+			recorder:    eventRecorder,
+			tsNamespace: opts.tailscaleNamespace,
+			Client:      mgr.GetClient(),
+			l:           opts.log.Named("recorder-reconciler"),
+			clock:       tstime.DefaultClock{},
+			tsClient:    opts.tsClient,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create Recorder reconciler: %v", err)
+	}
+
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
@@ -423,6 +454,10 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
+	// proxyDefaultClass is the name of the ProxyClass to use as the default
+	// class for proxies that do not have a ProxyClass set.
+	// this is defined by an operator env variable.
+	proxyDefaultClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
@@ -515,6 +550,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
+	Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error
 }
 

cmd/k8s-operator/operator_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime"
@@ -123,6 +124,7 @@ func TestLoadBalancerClass(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
@@ -260,6 +262,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		parentType:        "svc",
 		tailnetTargetFQDN: tailnetTargetFQDN,
 		hostname:          "default-test",
+		app:               kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -371,6 +374,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		parentType:      "svc",
 		tailnetTargetIP: tailnetTargetIP,
 		hostname:        "default-test",
+		app:             kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -479,6 +483,7 @@ func TestAnnotations(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -584,6 +589,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -713,6 +719,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -852,6 +859,7 @@ func TestCustomHostname(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "reindeer-flotilla",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -964,6 +972,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 		hostname:          "tailscale-critical",
 		priorityClassName: "custom-priority-class-name",
 		clusterTargetIP:   "10.20.30.40",
+		app:               kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@@ -1032,6 +1041,7 @@ func TestProxyClassForService(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
@@ -1125,6 +1135,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -1181,6 +1192,7 @@ func TestProxyFirewallMode(t *testing.T) {
 		hostname:        "default-test",
 		firewallMode:    "nftables",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 }
@@ -1235,6 +1247,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
 		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 
@@ -1529,6 +1542,7 @@ func Test_externalNameService(t *testing.T) {
 		parentType:       "svc",
 		hostname:         "default-test",
 		clusterTargetDNS: "foo.com",
+		app:              kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)

cmd/k8s-operator/proxy.go

@@ -22,8 +22,8 @@ import (
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
-	tskube "tailscale.com/kube"
-	"tailscale.com/ssh/tailssh"
+	ksr "tailscale.com/k8s-operator/sessionrecording"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -31,17 +31,10 @@ import (
 	"tailscale.com/util/set"
 )
 
-var whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
-
 var (
 	// counterNumRequestsproxies counts the number of API server requests proxied via this proxy.
 	counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
-
-	// counterSessionRecordingsAttempted counts the number of session recording attempts.
-	counterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy__session_recordings_attempted")
-
-	// counterSessionRecordingsUploaded counts the number of successfully uploaded session recordings.
-	counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded")
+	whoIsKey                  = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
 )
 
 type apiServerProxyMode int
@@ -173,7 +166,8 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", ap.serveDefault)
-	mux.HandleFunc("/api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExec)
+	mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
+	mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
 
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
@@ -214,9 +208,25 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
 	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
-// serveExec serves 'kubectl exec' requests, optionally configuring the kubectl
-// exec sessions to be recorded.
-func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
+// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.SPDYProtocol)
+}
+
+// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.WSProtocol)
+}
+
+func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
+	const (
+		podNameKey       = "pod"
+		namespaceNameKey = "namespace"
+		upgradeHeaderKey = "Upgrade"
+	)
+
 	who, err := ap.whoIs(r)
 	if err != nil {
 		ap.authError(w, err)
@@ -232,15 +242,17 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 		return
 	}
-	counterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
+	ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
 	if !failOpen && len(addrs) == 0 {
 		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
 		ap.log.Error(msg)
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-	if r.Method != "POST" || r.Header.Get("Upgrade") != "SPDY/3.1" {
-		msg := "'kubectl exec' session recording is configured, but the request is not over SPDY. Session recording is currently only supported for SPDY based clients"
+
+	wantsHeader := upgradeHeaderForProto[proto]
+	if h := r.Header.Get(upgradeHeaderKey); h != wantsHeader {
+		msg := fmt.Sprintf("[unexpected] unable to verify that streaming protocol is %s, wants Upgrade header %q, got: %q", proto, wantsHeader, h)
 		if failOpen {
 			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
 			ap.log.Warn(msg)
@@ -252,20 +264,22 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-	spdyH := &spdyHijacker{
-		ts:                ap.ts,
-		req:               r,
-		who:               who,
-		ResponseWriter:    w,
-		log:               ap.log,
-		pod:               r.PathValue("pod"),
-		ns:                r.PathValue("namespace"),
-		addrs:             addrs,
-		failOpen:          failOpen,
-		connectToRecorder: tailssh.ConnectToRecorder,
-	}
 
-	ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	opts := ksr.HijackerOpts{
+		Req:       r,
+		W:         w,
+		Proto:     proto,
+		TS:        ap.ts,
+		Who:       who,
+		Addrs:     addrs,
+		FailOpen:  failOpen,
+		Pod:       r.PathValue(podNameKey),
+		Namespace: r.PathValue(namespaceNameKey),
+		Log:       ap.log,
+	}
+	h := ksr.New(opts)
+
+	ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
 func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
@@ -300,9 +314,11 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
 		log.Printf("failed to add impersonation headers: " + err.Error())
 	}
 }
+
 func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
 	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
 }
+
 func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
 	ap.log.Errorf("failed to authenticate caller: %v", err)
 	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
@@ -323,10 +339,10 @@ const (
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)
@@ -376,7 +392,7 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 		return false, nil, errors.New("[unexpected] cannot determine caller")
 	}
 	failOpen = true
-	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if err != nil {
 		return failOpen, nil, fmt.Errorf("failed to unmarshal Kubernetes capability: %w", err)
 	}
@@ -398,3 +414,8 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 	}
 	return failOpen, recorderAddresses, nil
 }
+
+var upgradeHeaderForProto = map[ksr.Protocol]string{
+	ksr.SPDYProtocol: "SPDY/3.1",
+	ksr.WSProtocol:   "websocket",
+}

cmd/k8s-operator/recorder.go

@@ -1,88 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"tailscale.com/tstime"
-)
-
-// recorder knows how to send the provided bytes to the configured tsrecorder
-// instance in asciinema format.
-type recorder struct {
-	start time.Time
-	clock tstime.Clock
-
-	// failOpen specifies whether the session should be allowed to
-	// continue if writing to the recording fails.
-	failOpen bool
-
-	// backOff is set to true if  we've failed open and should stop
-	// attempting to write to tsrecorder.
-	backOff bool
-
-	mu   sync.Mutex     // guards writes to conn
-	conn io.WriteCloser // connection to a tsrecorder instance
-}
-
-// Write appends timestamp to the provided bytes and sends them to the
-// configured tsrecorder.
-func (rec *recorder) Write(p []byte) (err error) {
-	if len(p) == 0 {
-		return nil
-	}
-	if rec.backOff {
-		return nil
-	}
-	j, err := json.Marshal([]any{
-		rec.clock.Now().Sub(rec.start).Seconds(),
-		"o",
-		string(p),
-	})
-	if err != nil {
-		return fmt.Errorf("error marhalling payload: %w", err)
-	}
-	j = append(j, '\n')
-	if err := rec.writeCastLine(j); err != nil {
-		if !rec.failOpen {
-			return fmt.Errorf("error writing payload to recorder: %w", err)
-		}
-		rec.backOff = true
-	}
-	return nil
-}
-
-func (rec *recorder) Close() error {
-	rec.mu.Lock()
-	defer rec.mu.Unlock()
-	if rec.conn == nil {
-		return nil
-	}
-	err := rec.conn.Close()
-	rec.conn = nil
-	return err
-}
-
-// writeCastLine sends bytes to the tsrecorder. The bytes should be in
-// asciinema format.
-func (rec *recorder) writeCastLine(j []byte) error {
-	rec.mu.Lock()
-	defer rec.mu.Unlock()
-	if rec.conn == nil {
-		return errors.New("recorder closed")
-	}
-	_, err := rec.conn.Write(j)
-	if err != nil {
-		return fmt.Errorf("recorder write error: %w", err)
-	}
-	return nil
-}

cmd/k8s-operator/spdy-remote-conn-recorder.go

@@ -1,194 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"net"
-	"net/http"
-	"sync"
-	"sync/atomic"
-
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-)
-
-// spdyRemoteConnRecorder is a wrapper around net.Conn. It reads the bytestream
-// for a 'kubectl exec' session, sends session recording data to the configured
-// recorder and forwards the raw bytes to the original destination.
-type spdyRemoteConnRecorder struct {
-	net.Conn
-	// rec knows how to send data written to it to a tsrecorder instance.
-	rec *recorder
-	ch  CastHeader
-
-	stdoutStreamID atomic.Uint32
-	stderrStreamID atomic.Uint32
-	resizeStreamID atomic.Uint32
-
-	wmu    sync.Mutex // sequences writes
-	closed bool
-	failed bool
-
-	rmu                 sync.Mutex // sequences reads
-	writeCastHeaderOnce sync.Once
-
-	zlibReqReader zlibReader
-	// writeBuf is used to store data written to the connection that has not
-	// yet been parsed as SPDY frames.
-	writeBuf bytes.Buffer
-	// readBuf is used to store data read from the connection that has not
-	// yet been parsed as SPDY frames.
-	readBuf bytes.Buffer
-	log     *zap.SugaredLogger
-}
-
-// Read reads bytes from the original connection and parses them as SPDY frames.
-// If the frame is a data frame for resize stream, sends resize message to the
-// recorder. If the frame is a SYN_STREAM control frame that starts stdout,
-// stderr or resize stream, store the stream ID.
-func (c *spdyRemoteConnRecorder) Read(b []byte) (int, error) {
-	c.rmu.Lock()
-	defer c.rmu.Unlock()
-	n, err := c.Conn.Read(b)
-	if err != nil {
-		return n, fmt.Errorf("error reading from connection: %w", err)
-	}
-	c.readBuf.Write(b[:n])
-
-	var sf spdyFrame
-	ok, err := sf.Parse(c.readBuf.Bytes(), c.log)
-	if err != nil {
-		return 0, fmt.Errorf("error parsing data read from connection: %w", err)
-	}
-	if !ok {
-		// The parsed data in the buffer will be processed together with
-		// the new data on the next call to Read.
-		return n, nil
-	}
-	c.readBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame
-
-	if !sf.Ctrl { // data frame
-		switch sf.StreamID {
-		case c.resizeStreamID.Load():
-			var err error
-			var msg spdyResizeMsg
-			if err = json.Unmarshal(sf.Payload, &msg); err != nil {
-				return 0, fmt.Errorf("error umarshalling resize msg: %w", err)
-			}
-			c.ch.Width = msg.Width
-			c.ch.Height = msg.Height
-		}
-		return n, nil
-	}
-	// We always want to parse the headers, even if we don't care about the
-	// frame, as we need to advance the zlib reader otherwise we will get
-	// garbage.
-	header, err := sf.parseHeaders(&c.zlibReqReader, c.log)
-	if err != nil {
-		return 0, fmt.Errorf("error parsing frame headers: %w", err)
-	}
-	if sf.Type == SYN_STREAM {
-		c.storeStreamID(sf, header)
-	}
-	return n, nil
-}
-
-// Write forwards the raw data of the latest parsed SPDY frame to the original
-// destination. If the frame is an SPDY data frame, it also sends the payload to
-// the connected session recorder.
-func (c *spdyRemoteConnRecorder) Write(b []byte) (int, error) {
-	c.wmu.Lock()
-	defer c.wmu.Unlock()
-	c.writeBuf.Write(b)
-
-	var sf spdyFrame
-	ok, err := sf.Parse(c.writeBuf.Bytes(), c.log)
-	if err != nil {
-		return 0, fmt.Errorf("error parsing data: %w", err)
-	}
-	if !ok {
-		// The parsed data in the buffer will be processed together with
-		// the new data on the next call to Write.
-		return len(b), nil
-	}
-	c.writeBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame
-
-	// If this is a stdout or stderr data frame, send its payload to the
-	// session recorder.
-	if !sf.Ctrl {
-		switch sf.StreamID {
-		case c.stdoutStreamID.Load(), c.stderrStreamID.Load():
-			var err error
-			c.writeCastHeaderOnce.Do(func() {
-				var j []byte
-				j, err = json.Marshal(c.ch)
-				if err != nil {
-					return
-				}
-				j = append(j, '\n')
-				err = c.rec.writeCastLine(j)
-				if err != nil {
-					c.log.Errorf("received error from recorder: %v", err)
-				}
-			})
-			if err != nil {
-				return 0, fmt.Errorf("error writing CastHeader: %w", err)
-			}
-			if err := c.rec.Write(sf.Payload); err != nil {
-				return 0, fmt.Errorf("error sending payload to session recorder: %w", err)
-			}
-		}
-	}
-	// Forward the whole frame to the original destination.
-	_, err = c.Conn.Write(sf.Raw) // send to net.Conn
-	return len(b), err
-}
-
-func (c *spdyRemoteConnRecorder) Close() error {
-	c.wmu.Lock()
-	defer c.wmu.Unlock()
-	if c.closed {
-		return nil
-	}
-	if !c.failed && c.writeBuf.Len() > 0 {
-		c.Conn.Write(c.writeBuf.Bytes())
-	}
-	c.writeBuf.Reset()
-	c.closed = true
-	err := c.Conn.Close()
-	c.rec.Close()
-	return err
-}
-
-// parseSynStream parses SYN_STREAM SPDY control frame and updates
-// spdyRemoteConnRecorder to store the newly created stream's ID if it is one of
-// the stream types we care about. Storing stream_id:stream_type mapping allows
-// us to parse received data frames (that have stream IDs) differently depening
-// on which stream they belong to (i.e send data frame payload for stdout stream
-// to session recorder).
-func (c *spdyRemoteConnRecorder) storeStreamID(sf spdyFrame, header http.Header) {
-	const (
-		streamTypeHeaderKey = "Streamtype"
-	)
-	id := binary.BigEndian.Uint32(sf.Payload[0:4])
-	switch header.Get(streamTypeHeaderKey) {
-	case corev1.StreamTypeStdout:
-		c.stdoutStreamID.Store(id)
-	case corev1.StreamTypeStderr:
-		c.stderrStreamID.Store(id)
-	case corev1.StreamTypeResize:
-		c.resizeStreamID.Store(id)
-	}
-}
-
-type spdyResizeMsg struct {
-	Width  int `json:"width"`
-	Height int `json:"height"`
-}

cmd/k8s-operator/sts.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
@@ -342,7 +343,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		if len(tags) == 0 {
 			tags = a.defaultTags
 		}
-		authKey, err = a.newAuthKey(ctx, tags)
+		authKey, err = newAuthKey(ctx, a.tsClient, tags)
 		if err != nil {
 			return "", "", nil, err
 		}
@@ -418,6 +419,11 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	if sec == nil {
 		return "", "", nil, nil
 	}
+
+	return deviceInfo(sec)
+}
+
+func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	id = tailcfg.StableNodeID(sec.Data["device_id"])
 	if id == "" {
 		return "", "", nil, nil
@@ -441,7 +447,7 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	return id, hostname, ips, nil
 }
 
-func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {
+func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) {
 	caps := tailscale.KeyCapabilities{
 		Devices: tailscale.KeyDeviceCapabilities{
 			Create: tailscale.KeyDeviceCreateCapabilities{
@@ -452,7 +458,7 @@ func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string)
 		},
 	}
 
-	key, _, err := a.tsClient.CreateKey(ctx, caps)
+	key, _, err := tsClient.CreateKey(ctx, caps)
 	if err != nil {
 		return "", err
 	}
@@ -598,6 +604,18 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	}
+	app, err := appInfoForProxy(sts)
+	if err != nil {
+		// No need to error out if now or in future we end up in a
+		// situation where app info cannot be determined for one of the
+		// many proxy configurations that the operator can produce.
+		logger.Error("[unexpected] unable to determine proxy type")
+	} else {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_INTERNAL_APP",
+			Value: app,
+		})
+	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	if sts.ProxyClassName != "" {
 		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
@@ -611,6 +629,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS)
 }
 
+func appInfoForProxy(cfg *tailscaleSTSConfig) (string, error) {
+	if cfg.ClusterTargetDNSName != "" || cfg.ClusterTargetIP != "" {
+		return kubetypes.AppIngressProxy, nil
+	}
+	if cfg.TailnetTargetFQDN != "" || cfg.TailnetTargetIP != "" {
+		return kubetypes.AppEgressProxy, nil
+	}
+	if cfg.ServeConfig != nil {
+		return kubetypes.AppIngressResource, nil
+	}
+	if cfg.Connector != nil {
+		return kubetypes.AppConnector, nil
+	}
+	return "", errors.New("unable to determine proxy type")
+}
+
 // mergeStatefulSetLabelsOrAnnots returns a map that contains all keys/values
 // present in 'custom' map as well as those keys/values from the current map
 // whose keys are present in the 'managed' map. The reason why this merge is

cmd/k8s-operator/svc.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
@@ -62,15 +63,17 @@ type ServiceReconciler struct {
 	tsNamespace string
 
 	clock tstime.Clock
+
+	proxyDefaultClass string
 }
 
 var (
 	// gaugeEgressProxies tracks the number of egress proxies that we're
 	// currently managing.
-	gaugeEgressProxies = clientmetric.NewGauge("k8s_egress_proxies")
+	gaugeEgressProxies = clientmetric.NewGauge(kubetypes.MetricEgressProxyCount)
 	// gaugeIngressProxies tracks the number of ingress proxies that we're
 	// currently managing.
-	gaugeIngressProxies = clientmetric.NewGauge("k8s_ingress_proxies")
+	gaugeIngressProxies = clientmetric.NewGauge(kubetypes.MetricIngressProxyCount)
 )
 
 func childResourceLabels(name, ns, typ string) map[string]string {
@@ -208,7 +211,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc)
+	proxyClass := proxyClassForObject(svc, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -325,7 +328,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	if err != nil {
 		msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
 		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
-		return fmt.Errorf(msg)
+		return errors.New(msg)
 	}
 	for _, ip := range tsIPs {
 		addr, err := netip.ParseAddr(ip)
@@ -404,8 +407,14 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-func proxyClassForObject(o client.Object) string {
-	return o.GetLabels()[LabelProxyClass]
+// proxyClassForObject returns the proxy class for the given object. If the
+// object does not have a proxy class label, it returns the default proxy class
+func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+	proxyClass, exists := o.GetLabels()[LabelProxyClass]
+	if !exists {
+		proxyClass = proxyDefaultClass
+	}
+	return proxyClass
 }
 
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {

cmd/k8s-operator/testutils_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/netip"
+	"reflect"
 	"strings"
 	"sync"
 	"testing"
@@ -51,6 +52,7 @@ type configOpts struct {
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
 	proxyClass                                     string // configuration from the named ProxyClass should be applied to proxy resources
+	app                                            string
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -142,6 +144,10 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
 		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
 	}
+	tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
+		Name:  "TS_INTERNAL_APP",
+		Value: opts.app,
+	})
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -224,6 +230,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
+			{Name: "TS_INTERNAL_APP", Value: opts.app},
 		},
 		ImagePullPolicy: "Always",
 		VolumeMounts: []corev1.VolumeMount{
@@ -481,7 +488,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
-		t.Fatalf("unexpected object (-got +want):\n%s", diff)
+		t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
 	}
 }
 
@@ -492,7 +499,7 @@ func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns
 		Name:      name,
 		Namespace: ns,
 	}, obj); !apierrors.IsNotFound(err) {
-		t.Fatalf("object %s/%s unexpectedly present, wanted missing", ns, name)
+		t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
 	}
 }
 
@@ -586,6 +593,17 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili
 	return "secret-authkey", k, nil
 }
 
+func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) {
+	return &tailscale.Device{
+		DeviceID: deviceID,
+		Hostname: "test-device",
+		Addresses: []string{
+			"1.2.3.4",
+			"::1",
+		},
+	}, nil
+}
+
 func (c *fakeTSClient) DeleteDevice(ctx context.Context, deviceID string) error {
 	c.Lock()
 	defer c.Unlock()

cmd/k8s-operator/tsrecorder.go

@@ -0,0 +1,375 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"sync"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/tailscale"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonRecorderCreationFailed = "RecorderCreationFailed"
+	reasonRecorderCreated        = "RecorderCreated"
+	reasonRecorderInvalid        = "RecorderInvalid"
+
+	currentProfileKey = "_current-profile"
+)
+
+var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount)
+
+// RecorderReconciler syncs Recorder statefulsets with their definition in
+// Recorder CRs.
+type RecorderReconciler struct {
+	client.Client
+	l           *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+	tsClient    tsClient
+
+	mu        sync.Mutex           // protects following
+	recorders set.Slice[types.UID] // for recorders gauge
+}
+
+func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger {
+	return r.l.With("Recorder", name)
+}
+
+func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
+	logger := r.logger(req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	tsr := new(tsapi.Recorder)
+	err = r.Get(ctx, req.NamespacedName, tsr)
+	if apierrors.IsNotFound(err) {
+		logger.Debugf("Recorder not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err)
+	}
+	if markedForDeletion(tsr) {
+		logger.Debugf("Recorder is being deleted, cleaning up resources")
+		ix := xslices.Index(tsr.Finalizers, FinalizerName)
+		if ix < 0 {
+			logger.Debugf("no finalizer, nothing to do")
+			return reconcile.Result{}, nil
+		}
+
+		if done, err := r.maybeCleanup(ctx, tsr); err != nil {
+			return reconcile.Result{}, err
+		} else if !done {
+			logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
+			return reconcile.Result{RequeueAfter: shortRequeue}, nil
+		}
+
+		tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
+		if err := r.Update(ctx, tsr); err != nil {
+			return reconcile.Result{}, err
+		}
+		return reconcile.Result{}, nil
+	}
+
+	oldTSRStatus := tsr.Status.DeepCopy()
+	setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
+		if !apiequality.Semantic.DeepEqual(oldTSRStatus, tsr.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
+		}
+		return reconcile.Result{}, err
+	}
+
+	if !slices.Contains(tsr.Finalizers, FinalizerName) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to log that the high level, multi-reconcile
+		// operation is underway.
+		logger.Infof("ensuring Recorder is set up")
+		tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
+		if err := r.Update(ctx, tsr); err != nil {
+			logger.Errorf("error adding finalizer: %w", err)
+			return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
+		}
+	}
+
+	if err := r.validate(tsr); err != nil {
+		logger.Errorf("error validating Recorder spec: %w", err)
+		message := fmt.Sprintf("Recorder is invalid: %s", err)
+		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message)
+		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
+	}
+
+	if err = r.maybeProvision(ctx, tsr); err != nil {
+		logger.Errorf("error creating Recorder resources: %w", err)
+		message := fmt.Sprintf("failed creating Recorder: %s", err)
+		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
+		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, message)
+	}
+
+	logger.Info("Recorder resources synced")
+	return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated)
+}
+
+func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error {
+	logger := r.logger(tsr.Name)
+
+	r.mu.Lock()
+	r.recorders.Add(tsr.UID)
+	gaugeRecorderResources.Set(int64(r.recorders.Len()))
+	r.mu.Unlock()
+
+	if err := r.ensureAuthSecretCreated(ctx, tsr); err != nil {
+		return fmt.Errorf("error creating secrets: %w", err)
+	}
+	// State secret is precreated so we can use the Recorder CR as its owner ref.
+	sec := tsrStateSecret(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
+		s.ObjectMeta.Labels = sec.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error creating state Secret: %w", err)
+	}
+	sa := tsrServiceAccount(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
+		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error creating ServiceAccount: %w", err)
+	}
+	role := tsrRole(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
+		r.ObjectMeta.Labels = role.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
+		r.Rules = role.Rules
+	}); err != nil {
+		return fmt.Errorf("error creating Role: %w", err)
+	}
+	roleBinding := tsrRoleBinding(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
+		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
+		r.RoleRef = roleBinding.RoleRef
+		r.Subjects = roleBinding.Subjects
+	}); err != nil {
+		return fmt.Errorf("error creating RoleBinding: %w", err)
+	}
+	ss := tsrStatefulSet(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
+		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
+		s.Spec = ss.Spec
+	}); err != nil {
+		return fmt.Errorf("error creating StatefulSet: %w", err)
+	}
+
+	var devices []tsapi.TailnetDevice
+
+	device, ok, err := r.getDeviceInfo(ctx, tsr.Name)
+	if err != nil {
+		return fmt.Errorf("failed to get device info: %w", err)
+	}
+	if !ok {
+		logger.Debugf("no Tailscale hostname known yet, waiting for Recorder pod to finish auth")
+		return nil
+	}
+
+	devices = append(devices, device)
+
+	tsr.Status.Devices = devices
+
+	return nil
+}
+
+// maybeCleanup just deletes the device from the tailnet. All the kubernetes
+// resources linked to a Recorder will get cleaned up via owner references
+// (which we can use because they are all in the same namespace).
+func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
+	logger := r.logger(tsr.Name)
+
+	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
+	if err != nil {
+		return false, err
+	}
+	if !ok {
+		logger.Debugf("state Secret %s-0 not found or does not contain node ID, continuing cleanup", tsr.Name)
+		r.mu.Lock()
+		r.recorders.Remove(tsr.UID)
+		gaugeRecorderResources.Set(int64(r.recorders.Len()))
+		r.mu.Unlock()
+		return true, nil
+	}
+
+	logger.Debugf("deleting device %s from control", string(id))
+	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
+			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
+		} else {
+			return false, fmt.Errorf("error deleting device: %w", err)
+		}
+	} else {
+		logger.Debugf("device %s deleted from control", string(id))
+	}
+
+	// Unlike most log entries in the reconcile loop, this will get printed
+	// exactly once at the very end of cleanup, because the final step of
+	// cleanup removes the tailscale finalizer, which will make all future
+	// reconciles exit early.
+	logger.Infof("cleaned up Recorder resources")
+	r.mu.Lock()
+	r.recorders.Remove(tsr.UID)
+	gaugeRecorderResources.Set(int64(r.recorders.Len()))
+	r.mu.Unlock()
+	return true, nil
+}
+
+func (r *RecorderReconciler) ensureAuthSecretCreated(ctx context.Context, tsr *tsapi.Recorder) error {
+	logger := r.logger(tsr.Name)
+	key := types.NamespacedName{
+		Namespace: r.tsNamespace,
+		Name:      tsr.Name,
+	}
+	if err := r.Get(ctx, key, &corev1.Secret{}); err == nil {
+		// No updates, already created the auth key.
+		logger.Debugf("auth Secret %s already exists", key.Name)
+		return nil
+	} else if !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// Create the auth key Secret which is going to be used by the StatefulSet
+	// to authenticate with Tailscale.
+	logger.Debugf("creating authkey for new Recorder")
+	tags := tsr.Spec.Tags
+	if len(tags) == 0 {
+		tags = tsapi.Tags{"tag:k8s"}
+	}
+	authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify())
+	if err != nil {
+		return err
+	}
+
+	logger.Debug("creating a new Secret for the Recorder")
+	if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *RecorderReconciler) validate(tsr *tsapi.Recorder) error {
+	if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil {
+		return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible")
+	}
+
+	return nil
+}
+
+// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// is expected to always be non-empty if the node ID is, but not required.
+func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.tsNamespace,
+			Name:      fmt.Sprintf("%s-0", tsrName),
+		},
+	}
+	if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return "", "", false, nil
+		}
+
+		return "", "", false, err
+	}
+
+	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
+	currentProfile, ok := secret.Data[currentProfileKey]
+	if !ok {
+		return "", "", false, nil
+	}
+	profileBytes, ok := secret.Data[string(currentProfile)]
+	if !ok {
+		return "", "", false, nil
+	}
+	var profile profile
+	if err := json.Unmarshal(profileBytes, &profile); err != nil {
+		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
+	}
+
+	ok = profile.Config.NodeID != ""
+	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
+}
+
+func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.TailnetDevice, ok bool, err error) {
+	nodeID, dnsName, ok, err := r.getNodeMetadata(ctx, tsrName)
+	if !ok || err != nil {
+		return tsapi.TailnetDevice{}, false, err
+	}
+
+	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
+	// need the API. Should we instead update the profile to include addresses?
+	device, err := r.tsClient.Device(ctx, string(nodeID), nil)
+	if err != nil {
+		return tsapi.TailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
+	}
+
+	d = tsapi.TailnetDevice{
+		Hostname:   device.Hostname,
+		TailnetIPs: device.Addresses,
+	}
+	if dnsName != "" {
+		d.URL = fmt.Sprintf("https://%s", dnsName)
+	}
+
+	return d, true, nil
+}
+
+type profile struct {
+	Config struct {
+		NodeID      string `json:"NodeID"`
+		UserProfile struct {
+			LoginName string `json:"LoginName"`
+		} `json:"UserProfile"`
+	} `json:"Config"`
+}
+
+func markedForDeletion(tsr *tsapi.Recorder) bool {
+	return !tsr.DeletionTimestamp.IsZero()
+}

cmd/k8s-operator/tsrecorder_specs.go

@@ -0,0 +1,278 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+	"tailscale.com/version"
+)
+
+func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet {
+	return &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Labels),
+			OwnerReferences: tsrOwnerReference(tsr),
+			Annotations:     tsr.Spec.StatefulSet.Annotations,
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: ptr.To[int32](1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels),
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tsr.Name,
+					Namespace:   namespace,
+					Labels:      labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels),
+					Annotations: tsr.Spec.StatefulSet.Pod.Annotations,
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: tsr.Name,
+					Affinity:           tsr.Spec.StatefulSet.Pod.Affinity,
+					SecurityContext:    tsr.Spec.StatefulSet.Pod.SecurityContext,
+					ImagePullSecrets:   tsr.Spec.StatefulSet.Pod.ImagePullSecrets,
+					NodeSelector:       tsr.Spec.StatefulSet.Pod.NodeSelector,
+					Tolerations:        tsr.Spec.StatefulSet.Pod.Tolerations,
+					Containers: []corev1.Container{
+						{
+							Name: "recorder",
+							Image: func() string {
+								image := tsr.Spec.StatefulSet.Pod.Container.Image
+								if image == "" {
+									image = fmt.Sprintf("tailscale/tsrecorder:%s", selfVersionImageTag())
+								}
+
+								return image
+							}(),
+							ImagePullPolicy: tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy,
+							Resources:       tsr.Spec.StatefulSet.Pod.Container.Resources,
+							SecurityContext: tsr.Spec.StatefulSet.Pod.Container.SecurityContext,
+							Env:             env(tsr),
+							EnvFrom: func() []corev1.EnvFromSource {
+								if tsr.Spec.Storage.S3 == nil || tsr.Spec.Storage.S3.Credentials.Secret.Name == "" {
+									return nil
+								}
+
+								return []corev1.EnvFromSource{{
+									SecretRef: &corev1.SecretEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: tsr.Spec.Storage.S3.Credentials.Secret.Name,
+										},
+									},
+								}}
+							}(),
+							Command: []string{"/tsrecorder"},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "data",
+									MountPath: "/data",
+									ReadOnly:  false,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: "data",
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func tsrServiceAccount(tsr *tsapi.Recorder, namespace string) *corev1.ServiceAccount {
+	return &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+	}
+}
+
+func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"get",
+					"patch",
+					"update",
+				},
+				ResourceNames: []string{
+					tsr.Name,                      // Contains the auth key.
+					fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
+				},
+			},
+		},
+	}
+}
+
+func tsrRoleBinding(tsr *tsapi.Recorder, namespace string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      tsr.Name,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: tsr.Name,
+		},
+	}
+}
+
+func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:       namespace,
+			Name:            tsr.Name,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		StringData: map[string]string{
+			"authkey": authKey,
+		},
+	}
+}
+
+func tsrStateSecret(tsr *tsapi.Recorder, namespace string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            fmt.Sprintf("%s-0", tsr.Name),
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+	}
+}
+
+func env(tsr *tsapi.Recorder) []corev1.EnvVar {
+	envs := []corev1.EnvVar{
+		{
+			Name: "TS_AUTHKEY",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: tsr.Name,
+					},
+					Key: "authkey",
+				},
+			},
+		},
+		{
+			Name: "POD_NAME",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					// Secret is named after the pod.
+					FieldPath: "metadata.name",
+				},
+			},
+		},
+		{
+			Name:  "TS_STATE",
+			Value: "kube:$(POD_NAME)",
+		},
+		{
+			Name:  "TSRECORDER_HOSTNAME",
+			Value: "$(POD_NAME)",
+		},
+	}
+
+	for _, env := range tsr.Spec.StatefulSet.Pod.Container.Env {
+		envs = append(envs, corev1.EnvVar{
+			Name:  string(env.Name),
+			Value: env.Value,
+		})
+	}
+
+	if tsr.Spec.Storage.S3 != nil {
+		envs = append(envs,
+			corev1.EnvVar{
+				Name:  "TSRECORDER_DST",
+				Value: fmt.Sprintf("s3://%s", tsr.Spec.Storage.S3.Endpoint),
+			},
+			corev1.EnvVar{
+				Name:  "TSRECORDER_BUCKET",
+				Value: tsr.Spec.Storage.S3.Bucket,
+			},
+		)
+	} else {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TSRECORDER_DST",
+			Value: "/data/recordings",
+		})
+	}
+
+	if tsr.Spec.EnableUI {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TSRECORDER_UI",
+			Value: "true",
+		})
+	}
+
+	return envs
+}
+
+func labels(app, instance string, customLabels map[string]string) map[string]string {
+	l := make(map[string]string, len(customLabels)+3)
+	for k, v := range customLabels {
+		l[k] = v
+	}
+
+	// ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
+	l["app.kubernetes.io/name"] = app
+	l["app.kubernetes.io/instance"] = instance
+	l["app.kubernetes.io/managed-by"] = "tailscale-operator"
+
+	return l
+}
+
+func tsrOwnerReference(owner metav1.Object) []metav1.OwnerReference {
+	return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("Recorder"))}
+}
+
+// selfVersionImageTag returns the container image tag of the running operator
+// build.
+func selfVersionImageTag() string {
+	meta := version.GetMeta()
+	var versionPrefix string
+	if meta.UnstableBranch {
+		versionPrefix = "unstable-"
+	}
+	return fmt.Sprintf("%sv%s", versionPrefix, meta.MajorMinorPatch)
+}

cmd/k8s-operator/tsrecorder_specs_test.go

@@ -0,0 +1,143 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+)
+
+func TestRecorderSpecs(t *testing.T) {
+	t.Run("ensure spec fields are passed through correctly", func(t *testing.T) {
+		tsr := &tsapi.Recorder{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test",
+			},
+			Spec: tsapi.RecorderSpec{
+				StatefulSet: tsapi.RecorderStatefulSet{
+					Labels: map[string]string{
+						"ss-label-key": "ss-label-value",
+					},
+					Annotations: map[string]string{
+						"ss-annotation-key": "ss-annotation-value",
+					},
+					Pod: tsapi.RecorderPod{
+						Labels: map[string]string{
+							"pod-label-key": "pod-label-value",
+						},
+						Annotations: map[string]string{
+							"pod-annotation-key": "pod-annotation-value",
+						},
+						Affinity: &corev1.Affinity{
+							PodAffinity: &corev1.PodAffinity{
+								RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{
+									LabelSelector: &metav1.LabelSelector{
+										MatchLabels: map[string]string{
+											"match-label": "match-value",
+										},
+									}},
+								},
+							},
+						},
+						SecurityContext: &corev1.PodSecurityContext{
+							RunAsUser: ptr.To[int64](1000),
+						},
+						ImagePullSecrets: []corev1.LocalObjectReference{{
+							Name: "img-pull",
+						}},
+						NodeSelector: map[string]string{
+							"some-node": "selector",
+						},
+						Tolerations: []corev1.Toleration{{
+							Key:               "key",
+							Value:             "value",
+							TolerationSeconds: ptr.To[int64](60),
+						}},
+						Container: tsapi.RecorderContainer{
+							Env: []tsapi.Env{{
+								Name:  "some_env",
+								Value: "env_value",
+							}},
+							Image:           "custom-image",
+							ImagePullPolicy: corev1.PullAlways,
+							SecurityContext: &corev1.SecurityContext{
+								Capabilities: &corev1.Capabilities{
+									Add: []corev1.Capability{
+										"NET_ADMIN",
+									},
+								},
+							},
+							Resources: corev1.ResourceRequirements{
+								Limits: corev1.ResourceList{
+									corev1.ResourceCPU: resource.MustParse("100m"),
+								},
+								Requests: corev1.ResourceList{
+									corev1.ResourceCPU: resource.MustParse("50m"),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		ss := tsrStatefulSet(tsr, tsNamespace)
+
+		// StatefulSet-level.
+		if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Annotations, tsr.Spec.StatefulSet.Pod.Annotations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+
+		// Pod-level.
+		if diff := cmp.Diff(ss.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.SecurityContext, tsr.Spec.StatefulSet.Pod.SecurityContext); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.ImagePullSecrets, tsr.Spec.StatefulSet.Pod.ImagePullSecrets); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.NodeSelector, tsr.Spec.StatefulSet.Pod.NodeSelector); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Tolerations, tsr.Spec.StatefulSet.Pod.Tolerations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+
+		// Container-level.
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].SecurityContext, tsr.Spec.StatefulSet.Pod.Container.SecurityContext); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+	})
+}

cmd/k8s-operator/tsrecorder_test.go

@@ -0,0 +1,162 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+)
+
+const tsNamespace = "tailscale"
+
+func TestRecorder(t *testing.T) {
+	tsr := &tsapi.Recorder{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Finalizers: []string{"tailscale.com/finalizer"},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(tsr).
+		WithStatusSubresource(tsr).
+		Build()
+	tsClient := &fakeTSClient{}
+	zl, _ := zap.NewDevelopment()
+	fr := record.NewFakeRecorder(1)
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	reconciler := &RecorderReconciler{
+		tsNamespace: tsNamespace,
+		Client:      fc,
+		tsClient:    tsClient,
+		recorder:    fr,
+		l:           zl.Sugar(),
+		clock:       cl,
+	}
+
+	t.Run("invalid spec gives an error condition", func(t *testing.T) {
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar())
+		expectEqual(t, fc, tsr, nil)
+		if expected := 0; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		expectRecorderResources(t, fc, tsr, false)
+
+		expectedEvent := "Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
+		expectEvents(t, fr, []string{expectedEvent})
+	})
+
+	t.Run("observe Ready=true status condition for a valid spec", func(t *testing.T) {
+		tsr.Spec.EnableUI = true
+		mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
+			t.Spec = tsr.Spec
+		})
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar())
+		expectEqual(t, fc, tsr, nil)
+		if expected := 1; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		expectRecorderResources(t, fc, tsr, true)
+	})
+
+	t.Run("populate node info in state secret, and see it appear in status", func(t *testing.T) {
+		bytes, err := json.Marshal(map[string]any{
+			"Config": map[string]any{
+				"NodeID": "nodeid-123",
+				"UserProfile": map[string]any{
+					"LoginName": "test-0.example.ts.net",
+				},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		const key = "profile-abc"
+		mustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) {
+			s.Data = map[string][]byte{
+				currentProfileKey: []byte(key),
+				key:               bytes,
+			}
+		})
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+		tsr.Status.Devices = []tsapi.TailnetDevice{
+			{
+				Hostname:   "test-device",
+				TailnetIPs: []string{"1.2.3.4", "::1"},
+				URL:        "https://test-0.example.ts.net",
+			},
+		}
+		expectEqual(t, fc, tsr, nil)
+	})
+
+	t.Run("delete the Recorder and observe cleanup", func(t *testing.T) {
+		if err := fc.Delete(context.Background(), tsr); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		expectMissing[tsapi.Recorder](t, fc, "", tsr.Name)
+		if expected := 0; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-123"}); diff != "" {
+			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
+		}
+		// The fake client does not clean up objects whose owner has been
+		// deleted, so we can't test for the owned resources getting deleted.
+	})
+}
+
+func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) {
+	t.Helper()
+
+	auth := tsrAuthSecret(tsr, tsNamespace, "secret-authkey")
+	state := tsrStateSecret(tsr, tsNamespace)
+	role := tsrRole(tsr, tsNamespace)
+	roleBinding := tsrRoleBinding(tsr, tsNamespace)
+	serviceAccount := tsrServiceAccount(tsr, tsNamespace)
+	statefulSet := tsrStatefulSet(tsr, tsNamespace)
+
+	if shouldExist {
+		expectEqual(t, fc, auth, nil)
+		expectEqual(t, fc, state, nil)
+		expectEqual(t, fc, role, nil)
+		expectEqual(t, fc, roleBinding, nil)
+		expectEqual(t, fc, serviceAccount, nil)
+		expectEqual(t, fc, statefulSet, nil)
+	} else {
+		expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name)
+		expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name)
+		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
+		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
+		expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
+		expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
+	}
+}

cmd/natc/natc.go

@@ -456,6 +456,11 @@ func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool {
 }
 
 func proxyTCPConn(c net.Conn, dest string) {
+	if c.RemoteAddr() == nil {
+		log.Printf("proxyTCPConn: nil RemoteAddr")
+		c.Close()
+		return
+	}
 	addrPortStr := c.LocalAddr().String()
 	_, port, err := net.SplitHostPort(addrPortStr)
 	if err != nil {
@@ -489,6 +494,9 @@ type perPeerState struct {
 func (ps *perPeerState) domainForIP(ip netip.Addr) (_ string, ok bool) {
 	ps.mu.Lock()
 	defer ps.mu.Unlock()
+	if ps.addrToDomain == nil {
+		return "", false
+	}
 	return ps.addrToDomain.Lookup(ip)
 }
 

cmd/stund/depaware.txt

@@ -50,6 +50,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/envknob                                        from tailscale.com/tsweb+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/net/stunserver+
         tailscale.com/net/netaddr                                    from tailscale.com/net/tsaddr
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
@@ -81,15 +82,16 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/version/distro                                 from tailscale.com/envknob
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/net/dns/dnsmessage                              from net
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
+        golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
@@ -153,6 +155,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from google.golang.org/protobuf/internal/impl
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/tailcfg+
@@ -168,7 +171,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -195,3 +198,4 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/stunstamp/stunstamp.go

@@ -1,13 +1,14 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-// The stunstamp binary measures STUN round-trip latency with DERPs.
+// The stunstamp binary measures round-trip latency with DERPs.
 package main
 
 import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -22,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"runtime"
 	"slices"
 	"strconv"
 	"strings"
@@ -31,8 +33,10 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/prompb"
+	"github.com/tcnksm/go-httpstat"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/stun"
+	"tailscale.com/net/tcpinfo"
 	"tailscale.com/tailcfg"
 )
 
@@ -42,11 +46,23 @@ var (
 	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
 	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
 	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
-	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+	flagSTUNDstPorts   = flag.String("stun-dst-ports", "", "comma-separated list of STUN destination ports to monitor")
+	flagHTTPSDstPorts  = flag.String("https-dst-ports", "", "comma-separated list of HTTPS destination ports to monitor")
+	flagTCPDstPorts    = flag.String("tcp-dst-ports", "", "comma-separated list of TCP destination ports to monitor")
+	flagICMP           = flag.Bool("icmp", false, "probe ICMP")
 )
 
 const (
-	minInterval       = time.Second
+	// maxTxJitter is the upper bounds for jitter introduced across probes
+	maxTXJitter = time.Millisecond * 400
+	// minInterval is the minimum allowed probe interval/step
+	minInterval = time.Second * 10
+	// txRxTimeout is the timeout value used for kernel timestamping loopback,
+	// and packet receive operations
+	txRxTimeout = time.Second * 2
+	// maxBufferDuration is the maximum duration (maxBufferDuration /
+	// *flagInterval steps worth) of buffered data that can be held in memory
+	// before data loss occurs around prometheus unavailability.
 	maxBufferDuration = time.Hour
 )
 
@@ -89,12 +105,22 @@ func (t timestampSource) String() string {
 	}
 }
 
+type protocol string
+
+const (
+	protocolSTUN  protocol = "stun"
+	protocolICMP  protocol = "icmp"
+	protocolHTTPS protocol = "https"
+	protocolTCP   protocol = "tcp"
+)
+
 // resultKey contains the stable dimensions and their values for a given
 // timeseries, i.e. not time and not rtt/timeout.
 type resultKey struct {
 	meta            nodeMeta
 	timestampSource timestampSource
 	connStability   connStability
+	protocol        protocol
 	dstPort         int
 }
 
@@ -104,19 +130,218 @@ type result struct {
 	rtt *time.Duration // nil signifies failure, e.g. timeout
 }
 
-func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+type lportsPool struct {
+	sync.Mutex
+	ports []int
+}
+
+func (l *lportsPool) get() int {
+	l.Lock()
+	defer l.Unlock()
+	ret := l.ports[0]
+	l.ports = append(l.ports[:0], l.ports[1:]...)
+	return ret
+}
+
+func (l *lportsPool) put(i int) {
+	l.Lock()
+	defer l.Unlock()
+	l.ports = append(l.ports, int(i))
+}
+
+var (
+	lports *lportsPool
+)
+
+const (
+	lportPoolSize = 16000
+	lportBase     = 2048
+)
+
+func init() {
+	lports = &lportsPool{
+		ports: make([]int, 0, lportPoolSize),
+	}
+	for i := lportBase; i < lportBase+lportPoolSize; i++ {
+		lports.ports = append(lports.ports, i)
+	}
+}
+
+// lportForTCPConn satisfies io.ReadWriteCloser, but is really just used to pass
+// around a persistent laddr for stableConn purposes. The underlying TCP
+// connection is not created until measurement time as in some cases we need to
+// measure dial time.
+type lportForTCPConn int
+
+func (l *lportForTCPConn) Close() error {
+	if *l == 0 {
+		return nil
+	}
+	lports.put(int(*l))
+	return nil
+}
+
+func (l *lportForTCPConn) Write([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func (l *lportForTCPConn) Read([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func addrInUse(err error, lport *lportForTCPConn) bool {
+	if errors.Is(err, syscall.EADDRINUSE) {
+		old := int(*lport)
+		// abandon port, don't return it to pool
+		*lport = lportForTCPConn(lports.get()) // get a new port
+		log.Printf("EADDRINUSE: %v old: %d new: %d", err, old, *lport)
+		return true
+	}
+	return false
+}
+
+func tcpDial(ctx context.Context, lport *lportForTCPConn, dst netip.AddrPort) (net.Conn, error) {
+	for {
+		var opErr error
+		dialer := &net.Dialer{
+			LocalAddr: &net.TCPAddr{
+				Port: int(*lport),
+			},
+			Control: func(network, address string, c syscall.RawConn) error {
+				return c.Control(func(fd uintptr) {
+					// we may restart faster than TIME_WAIT can clear
+					opErr = setSOReuseAddr(fd)
+				})
+			},
+		}
+		if opErr != nil {
+			panic(opErr)
+		}
+		tcpConn, err := dialer.DialContext(ctx, "tcp", dst.String())
+		if err != nil {
+			if addrInUse(err, lport) {
+				continue
+			}
+			return nil, err
+		}
+		return tcpConn, nil
+	}
+}
+
+type tempError struct {
+	error
+}
+
+func (t tempError) Temporary() bool {
+	return true
+}
+
+func measureTCPRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	// Set a dial timeout < 1s (TCP_TIMEOUT_INIT on Linux) as a means to avoid
+	// SYN retries, which can contribute to tcpi->rtt below. This simply limits
+	// retries from the initiator, but SYN+ACK on the reverse path can also
+	// time out and be retransmitted.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*750)
+	defer cancel()
+	tcpConn, err := tcpDial(ctx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	// This is an unreliable method to measure TCP RTT. The Linux kernel
+	// describes it as such in tcp_rtt_estimator(). We take some care in how we
+	// hold tcp_info->rtt here, e.g. clamping dial timeout, but if we are to
+	// actually use this elsewhere as an input to some decision it warrants a
+	// deeper study and consideration for alternative methods. Its usefulness
+	// here is as a point of comparison against the other methods.
+	rtt, err = tcpinfo.RTT(tcpConn)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	return rtt, nil
+}
+
+func measureHTTPSRTT(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	var httpResult httpstat.Result
+	// 5s mirrors net/netcheck.overallProbeTimeout used in net/netcheck.Client.measureHTTPSLatency.
+	reqCtx, cancel := context.WithTimeout(httpstat.WithHTTPStat(context.Background(), &httpResult), time.Second*5)
+	defer cancel()
+	reqURL := "https://" + dst.String() + "/derp/latency-check"
+	req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil)
+	if err != nil {
+		return 0, err
+	}
+	client := &http.Client{}
+	// 1.5s mirrors derp/derphttp.dialnodeTimeout used in derp/derphttp.DialNode().
+	dialCtx, dialCancel := context.WithTimeout(reqCtx, time.Millisecond*1500)
+	defer dialCancel()
+	tcpConn, err := tcpDial(dialCtx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	tlsConn := tls.Client(tcpConn, &tls.Config{
+		ServerName: hostname,
+	})
+	// Mirror client/netcheck behavior, which handshakes before handing the
+	// tlsConn over to the http.Client via http.Transport
+	err = tlsConn.Handshake()
+	if err != nil {
+		return 0, tempError{err}
+	}
+	tlsConnCh := make(chan net.Conn, 1)
+	tlsConnCh <- tlsConn
+	tr := &http.Transport{
+		DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
+			select {
+			case tlsConn := <-tlsConnCh:
+				return tlsConn, nil
+			default:
+				return nil, errors.New("unexpected second call of DialTLSContext")
+			}
+		},
+	}
+	client.Transport = tr
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	if resp.StatusCode/100 != 2 {
+		return 0, tempError{fmt.Errorf("unexpected status code: %d", resp.StatusCode)}
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(io.Discard, io.LimitReader(resp.Body, 8<<10))
+	if err != nil {
+		return 0, tempError{err}
+	}
+	httpResult.End(time.Now())
+	return httpResult.ServerProcessing, nil
+}
+
+func measureSTUNRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	uconn, ok := conn.(*net.UDPConn)
 	if !ok {
 		return 0, fmt.Errorf("unexpected conn type: %T", conn)
 	}
-	err = uconn.SetReadDeadline(time.Now().Add(time.Second * 2))
+	err = uconn.SetReadDeadline(time.Now().Add(txRxTimeout))
 	if err != nil {
 		return 0, fmt.Errorf("error setting read deadline: %w", err)
 	}
 	txID := stun.NewTxID()
 	req := stun.Request(txID)
 	txAt := time.Now()
-	_, err = uconn.WriteToUDP(req, dst)
+	_, err = uconn.WriteToUDP(req, &net.UDPAddr{
+		IP:   dst.Addr().AsSlice(),
+		Port: int(dst.Port()),
+	})
 	if err != nil {
 		return 0, fmt.Errorf("error writing to udp socket: %w", err)
 	}
@@ -153,29 +378,7 @@ type nodeMeta struct {
 	addr       netip.Addr
 }
 
-type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
-
-// probe measures STUN round trip time for the node described by meta over
-// conn against dstPort. It may return a nil duration and nil error if the
-// STUN request timed out. A non-nil error indicates an unrecoverable or
-// non-temporary error.
-func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
-	ua := &net.UDPAddr{
-		IP:   net.IP(meta.addr.AsSlice()),
-		Port: dstPort,
-	}
-
-	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
-	rtt, err := fn(conn, ua)
-	if err != nil {
-		if isTemporaryOrTimeoutErr(err) {
-			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
-			return nil, nil
-		}
-		return nil, err
-	}
-	return &rtt, nil
-}
+type measureFn func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error)
 
 // nodeMetaFromDERPMap parses the provided DERP map in order to update nodeMeta
 // in the provided nodeMetaByAddr. It returns a slice of nodeMeta containing
@@ -237,43 +440,144 @@ func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]node
 	return stale, nil
 }
 
-func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
-	conns := [2]io.ReadWriteCloser{}
-	byDstPort, ok := stableConns[addr]
-	if ok {
-		conns, ok = byDstPort[dstPort]
-		if ok {
-			return conns, nil
-		}
-	}
-	if supportsKernelTS() {
-		kconn, err := getConnKernelTimestamp()
-		if err != nil {
-			return conns, err
-		}
-		conns[timestampSourceKernel] = kconn
-	}
-	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
-	if err != nil {
-		if supportsKernelTS() {
-			conns[timestampSourceKernel].Close()
-		}
-		return conns, err
-	}
-	conns[timestampSourceUserspace] = uconn
-	if byDstPort == nil {
-		byDstPort = make(map[int][2]io.ReadWriteCloser)
-	}
-	byDstPort[dstPort] = conns
-	stableConns[addr] = byDstPort
-	return conns, nil
+type connAndMeasureFn struct {
+	conn io.ReadWriteCloser
+	fn   measureFn
 }
 
-// probeNodes measures the round-trip time for STUN binding requests against the
-// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
-// UDP sockets that should be recycled across runs. It returns the results or
-// an error if one occurs.
-func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+// newConnAndMeasureFn returns a connAndMeasureFn or an error. It may return
+// nil for both if some combination of the supplied timestampSource, protocol,
+// or connStability is unsupported.
+func newConnAndMeasureFn(forDst netip.Addr, source timestampSource, protocol protocol, stable connStability) (*connAndMeasureFn, error) {
+	info := getProtocolSupportInfo(protocol)
+	if !info.stableConn && bool(stable) {
+		return nil, nil
+	}
+	if !info.userspaceTS && source == timestampSourceUserspace {
+		return nil, nil
+	}
+	if !info.kernelTS && source == timestampSourceKernel {
+		return nil, nil
+	}
+	switch protocol {
+	case protocolSTUN:
+		if source == timestampSourceKernel {
+			conn, err := getUDPConnKernelTimestamp()
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTTKernel,
+			}, nil
+		} else {
+			conn, err := net.ListenUDP("udp", &net.UDPAddr{})
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTT,
+			}, nil
+		}
+	case protocolICMP:
+		conn, err := getICMPConn(forDst, source)
+		if err != nil {
+			return nil, err
+		}
+		return &connAndMeasureFn{
+			conn: conn,
+			fn:   mkICMPMeasureFn(source),
+		}, nil
+	case protocolHTTPS:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureHTTPSRTT,
+		}, nil
+	case protocolTCP:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureTCPRTT,
+		}, nil
+	}
+	return nil, errors.New("unknown protocol")
+}
+
+type stableConnKey struct {
+	node     netip.Addr
+	protocol protocol
+	port     int
+}
+
+type protocolSupportInfo struct {
+	kernelTS    bool
+	userspaceTS bool
+	stableConn  bool
+}
+
+func getConns(
+	stableConns map[stableConnKey][2]*connAndMeasureFn,
+	addr netip.Addr,
+	protocol protocol,
+	dstPort int,
+) (stable, unstable [2]*connAndMeasureFn, err error) {
+	key := stableConnKey{addr, protocol, dstPort}
+	defer func() {
+		if err != nil {
+			for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+				c := stable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+				c = unstable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+			}
+		}
+	}()
+
+	var ok bool
+	stable, ok = stableConns[key]
+	if !ok {
+		for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+			var cf *connAndMeasureFn
+			cf, err = newConnAndMeasureFn(addr, source, protocol, stableConn)
+			if err != nil {
+				return
+			}
+			stable[source] = cf
+		}
+		stableConns[key] = stable
+	}
+
+	for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+		var cf *connAndMeasureFn
+		cf, err = newConnAndMeasureFn(addr, source, protocol, unstableConn)
+		if err != nil {
+			return
+		}
+		unstable[source] = cf
+	}
+	return stable, unstable, nil
+}
+
+// probeNodes measures the round-trip time for the protocols and ports described
+// by portsByProtocol against the DERP nodes described by nodeMetaByAddr.
+// stableConns are used to recycle connections across calls to probeNodes.
+// probeNodes is also responsible for trimming stableConns based on node
+// lifetime in nodeMetaByAddr. It returns the results or an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[stableConnKey][2]*connAndMeasureFn, portsByProtocol map[protocol][]int) ([]result, error) {
 	wg := sync.WaitGroup{}
 	results := make([]result, 0)
 	resultsCh := make(chan result)
@@ -283,49 +587,36 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 	at := time.Now()
 	addrsToProbe := make(map[netip.Addr]bool)
 
-	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+	doProbe := func(cf *connAndMeasureFn, meta nodeMeta, source timestampSource, stable connStability, protocol protocol, dstPort int) {
 		defer wg.Done()
 		r := result{
 			key: resultKey{
 				meta:            meta,
 				timestampSource: source,
+				connStability:   stable,
 				dstPort:         dstPort,
+				protocol:        protocol,
 			},
 			at: at,
 		}
-		if conn == nil {
-			var err error
-			if source == timestampSourceKernel {
-				conn, err = getConnKernelTimestamp()
+		time.Sleep(rand.N(maxTXJitter)) // jitter across tx
+		addrPort := netip.AddrPortFrom(meta.addr, uint16(dstPort))
+		rtt, err := cf.fn(cf.conn, meta.hostname, addrPort)
+		if err != nil {
+			if isTemporaryOrTimeoutErr(err) {
+				r.rtt = nil
+				log.Printf("%s: temp error measuring RTT to %s(%s): %v", protocol, meta.hostname, addrPort, err)
 			} else {
-				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
-			}
-			if err != nil {
 				select {
 				case <-doneCh:
 					return
-				case errCh <- err:
+				case errCh <- fmt.Errorf("%s: %v", protocol, err):
 					return
 				}
 			}
-			defer conn.Close()
 		} else {
-			r.key.connStability = stableConn
+			r.rtt = &rtt
 		}
-		fn := measureRTT
-		if source == timestampSourceKernel {
-			fn = measureRTTKernel
-		}
-		rtt, err := probe(meta, conn, fn, dstPort)
-		if err != nil {
-			select {
-			case <-doneCh:
-				return
-			case errCh <- err:
-				return
-			}
-		}
-		r.rtt = rtt
 		select {
 		case <-doneCh:
 		case resultsCh <- r:
@@ -334,37 +625,42 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 
 	for _, meta := range nodeMetaByAddr {
 		addrsToProbe[meta.addr] = true
-		for _, port := range dstPorts {
-			stable, err := getStableConns(stableConns, meta.addr, port)
-			if err != nil {
-				close(doneCh)
-				wg.Wait()
-				return nil, err
-			}
+		for p, ports := range portsByProtocol {
+			for _, port := range ports {
+				stable, unstable, err := getConns(stableConns, meta.addr, p, port)
+				if err != nil {
+					close(doneCh)
+					wg.Wait()
+					return nil, err
+				}
 
-			wg.Add(2)
-			numProbes += 2
-			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
-			go doProbe(nil, meta, timestampSourceUserspace, port)
-			if supportsKernelTS() {
-				wg.Add(2)
-				numProbes += 2
-				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
-				go doProbe(nil, meta, timestampSourceKernel, port)
+				for i, cf := range stable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), stableConn, p, port)
+					}
+				}
+
+				for i, cf := range unstable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), unstableConn, p, port)
+					}
+				}
 			}
 		}
 	}
 
 	// cleanup conns we no longer need
-	for k, byDstPort := range stableConns {
-		if !addrsToProbe[k] {
-			for _, conns := range byDstPort {
-				if conns[timestampSourceKernel] != nil {
-					conns[timestampSourceKernel].Close()
-				}
-				conns[timestampSourceUserspace].Close()
-				delete(stableConns, k)
+	for k, cf := range stableConns {
+		if !addrsToProbe[k.node] {
+			if cf[timestampSourceKernel] != nil {
+				cf[timestampSourceKernel].conn.Close()
 			}
+			cf[timestampSourceUserspace].conn.Close()
+			delete(stableConns, k)
 		}
 	}
 
@@ -391,11 +687,11 @@ const (
 )
 
 const (
-	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
-	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+	rttMetricName      = "stunstamp_derp_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_timeouts_total"
 )
 
-func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, protocol protocol, dstPort int) []prompb.Label {
 	addressFamily := "ipv4"
 	if meta.addr.Is6() {
 		addressFamily = "ipv6"
@@ -425,6 +721,10 @@ func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source
 		Name:  "hostname",
 		Value: meta.hostname,
 	})
+	labels = append(labels, prompb.Label{
+		Name:  "protocol",
+		Value: string(protocol),
+	})
 	labels = append(labels, prompb.Label{
 		Name:  "dst_port",
 		Value: strconv.Itoa(dstPort),
@@ -453,53 +753,35 @@ const (
 	staleNaN uint64 = 0x7ff0000000000002
 )
 
-func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, portsByProtocol map[protocol][]int) []prompb.TimeSeries {
 	staleMarkers := make([]prompb.TimeSeries, 0)
 	now := time.Now()
-	for _, s := range stale {
-		for _, dstPort := range dstPorts {
-			samples := []prompb.Sample{
-				{
-					Timestamp: now.UnixMilli(),
-					Value:     math.Float64frombits(staleNaN),
-				},
-			}
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			if supportsKernelTS() {
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
+
+	for p, ports := range portsByProtocol {
+		for _, port := range ports {
+			for _, s := range stale {
+				samples := []prompb.Sample{
+					{
+						Timestamp: now.UnixMilli(),
+						Value:     math.Float64frombits(staleNaN),
+					},
+				}
+				// We send stale markers for all combinations in the interest
+				// of simplicity.
+				for _, name := range []string{rttMetricName, timeoutsMetricName} {
+					for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+						for _, stable := range []connStability{unstableConn, stableConn} {
+							staleMarkers = append(staleMarkers, prompb.TimeSeries{
+								Labels:  timeSeriesLabels(name, s, instance, source, stable, p, port),
+								Samples: samples,
+							})
+						}
+					}
+				}
 			}
 		}
 	}
+
 	return staleMarkers
 }
 
@@ -513,7 +795,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 	for _, r := range results {
 		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
 		seenKeys[r.key] = true
-		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		rttSamples := make([]prompb.Sample, 1)
 		rttSamples[0].Timestamp = r.at.UnixMilli()
 		if r.rtt != nil {
@@ -528,7 +810,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 		}
 		all = append(all, rttTS)
 		timeouts[r.key] = timeoutsCount
-		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		timeoutsSamples := make([]prompb.Sample, 1)
 		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
 		timeoutsSamples[0].Value = float64(timeoutsCount)
@@ -620,22 +902,59 @@ func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSer
 	}
 }
 
-func main() {
-	flag.Parse()
-	if len(*flagDstPorts) == 0 {
-		log.Fatal("dst-ports flag is unset")
+func getPortsFromFlag(f string) ([]int, error) {
+	if len(f) == 0 {
+		return nil, nil
 	}
-	dstPortsSplit := strings.Split(*flagDstPorts, ",")
-	slices.Sort(dstPortsSplit)
-	dstPortsSplit = slices.Compact(dstPortsSplit)
-	dstPorts := make([]int, 0, len(dstPortsSplit))
-	for _, d := range dstPortsSplit {
-		i, err := strconv.ParseUint(d, 10, 16)
+	split := strings.Split(f, ",")
+	slices.Sort(split)
+	split = slices.Compact(split)
+	ports := make([]int, 0)
+	for _, portStr := range split {
+		port, err := strconv.ParseUint(portStr, 10, 16)
 		if err != nil {
-			log.Fatal("invalid dst-ports")
+			return nil, err
 		}
-		dstPorts = append(dstPorts, int(i))
+		ports = append(ports, int(port))
 	}
+	return ports, nil
+}
+
+func main() {
+	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
+		log.Fatal("unsupported platform")
+	}
+	flag.Parse()
+
+	portsByProtocol := make(map[protocol][]int)
+	stunPorts, err := getPortsFromFlag(*flagSTUNDstPorts)
+	if err != nil {
+		log.Fatalf("invalid stun-dst-ports flag value: %v", err)
+	}
+	if len(stunPorts) > 0 {
+		portsByProtocol[protocolSTUN] = stunPorts
+	}
+	httpsPorts, err := getPortsFromFlag(*flagHTTPSDstPorts)
+	if err != nil {
+		log.Fatalf("invalid https-dst-ports flag value: %v", err)
+	}
+	if len(httpsPorts) > 0 {
+		portsByProtocol[protocolHTTPS] = httpsPorts
+	}
+	tcpPorts, err := getPortsFromFlag(*flagTCPDstPorts)
+	if err != nil {
+		log.Fatalf("invalid tcp-dst-ports flag value: %v", err)
+	}
+	if len(tcpPorts) > 0 {
+		portsByProtocol[protocolTCP] = tcpPorts
+	}
+	if *flagICMP {
+		portsByProtocol[protocolICMP] = []int{0}
+	}
+	if len(portsByProtocol) == 0 {
+		log.Fatal("nothing to probe")
+	}
+
 	if len(*flagDERPMap) < 1 {
 		log.Fatal("derp-map flag is unset")
 	}
@@ -645,7 +964,7 @@ func main() {
 	if len(*flagRemoteWriteURL) < 1 {
 		log.Fatal("rw-url flag is unset")
 	}
-	_, err := url.Parse(*flagRemoteWriteURL)
+	_, err = url.Parse(*flagRemoteWriteURL)
 	if err != nil {
 		log.Fatalf("invalid rw-url flag value: %v", err)
 	}
@@ -707,7 +1026,7 @@ func main() {
 		for _, v := range nodeMetaByAddr {
 			staleMeta = append(staleMeta, v)
 		}
-		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 		if len(staleMarkers) > 0 {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 			rwc.write(ctx, staleMarkers)
@@ -723,8 +1042,8 @@ func main() {
 	// in a higher probability of the packets traversing the same underlay path.
 	// Comparison of stable and unstable 5-tuple results can shed light on
 	// differences between paths where hashing (multipathing/load balancing)
-	// comes into play.
-	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+	// comes into play. The inner 2 element array index is timestampSource.
+	stableConns := make(map[stableConnKey][2]*connAndMeasureFn)
 
 	// timeouts holds counts of timeout events. Values are persisted for the
 	// lifetime of the related node in the DERP map.
@@ -738,7 +1057,7 @@ func main() {
 	for {
 		select {
 		case <-probeTicker.C:
-			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			results, err := probeNodes(nodeMetaByAddr, stableConns, portsByProtocol)
 			if err != nil {
 				log.Printf("unrecoverable error while probing: %v", err)
 				shutdown()
@@ -761,7 +1080,7 @@ func main() {
 				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
 				continue
 			}
-			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 			if len(staleMarkers) < 1 {
 				continue
 			}
@@ -780,7 +1099,7 @@ func main() {
 				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 				defer cancel()
 				updatedDM, err := getDERPMap(ctx, *flagDERPMap)
-				if err != nil {
+				if err == nil {
 					dmCh <- updatedDM
 				}
 			}()

cmd/stunstamp/stunstamp_default.go

@@ -8,18 +8,58 @@ package main
 import (
 	"errors"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unimplemented")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	return 0, errors.New("unimplemented")
 }
 
-func supportsKernelTS() bool {
-	return false
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	case protocolICMP:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: false,
+			stableConn:  false,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func getICMPConn(forDst netip.Addr, source timestampSource) (io.ReadWriteCloser, error) {
+	return nil, errors.New("platform unsupported")
+}
+
+func mkICMPMeasureFn(source timestampSource) measureFn {
+	return func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+		return 0, errors.New("platform unsupported")
+	}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	return nil
 }

cmd/stunstamp/stunstamp_linux.go

@@ -10,21 +10,27 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
+	"math"
+	"math/rand/v2"
+	"net/netip"
+	"syscall"
 	"time"
 
 	"github.com/mdlayher/socket"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/stun"
 )
 
 const (
-	flags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
+	timestampingFlags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
 		unix.SOF_TIMESTAMPING_RX_SOFTWARE | // rx timestamp generation in the kernel
 		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
 	if err != nil {
 		return nil, err
@@ -34,7 +40,7 @@ func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, flags)
+	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, timestampingFlags)
 	if err != nil {
 		return nil, err
 	}
@@ -56,24 +62,144 @@ func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
 	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func mkICMPMeasureFn(source timestampSource) measureFn {
+	return func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+		return measureICMPRTT(source, conn, hostname, dst)
+	}
+}
+
+func measureICMPRTT(source timestampSource, conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	sconn, ok := conn.(*socket.Conn)
+	if !ok {
+		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
+	}
+	txBody := &icmp.Echo{
+		// The kernel overrides this and routes appropriately so there is no
+		// point in setting or verifying.
+		ID: 0,
+		// Make this sufficiently random so that we do not account a late
+		// arriving reply in a future probe window.
+		Seq: int(rand.Int32N(math.MaxUint16)),
+		// Fingerprint ourselves.
+		Data: []byte("stunstamp"),
+	}
+	txMsg := icmp.Message{
+		Body: txBody,
+	}
+	var to unix.Sockaddr
+	if dst.Addr().Is4() {
+		txMsg.Type = ipv4.ICMPTypeEcho
+		to = &unix.SockaddrInet4{}
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
+	} else {
+		txMsg.Type = ipv6.ICMPTypeEchoRequest
+		to = &unix.SockaddrInet6{}
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
+	}
+	txBuf, err := txMsg.Marshal(nil)
+	if err != nil {
+		return 0, err
+	}
+	txAt := time.Now()
+	err = sconn.Sendto(context.Background(), txBuf, 0, to)
+	if err != nil {
+		return 0, fmt.Errorf("sendto error: %v", err)
+	}
+
+	if source == timestampSourceKernel {
+		txCtx, txCancel := context.WithTimeout(context.Background(), txRxTimeout)
+		defer txCancel()
+
+		buf := make([]byte, 1024)
+		oob := make([]byte, 1024)
+
+		for {
+			n, oobn, _, _, err := sconn.Recvmsg(txCtx, buf, oob, unix.MSG_ERRQUEUE)
+			if err != nil {
+				return 0, fmt.Errorf("recvmsg (MSG_ERRQUEUE) error: %v", err) // don't wrap
+			}
+
+			buf = buf[:n]
+			// Spin until we find the message we sent. We get the full packet
+			// looped including eth header so match against the tail.
+			if n < len(txBuf) {
+				continue
+			}
+			txLoopedMsg, err := icmp.ParseMessage(txMsg.Type.Protocol(), buf[len(buf)-len(txBuf):])
+			if err != nil {
+				continue
+			}
+			txLoopedBody, ok := txLoopedMsg.Body.(*icmp.Echo)
+			if !ok || txLoopedBody.Seq != txBody.Seq || txLoopedMsg.Code != txMsg.Code ||
+				txLoopedMsg.Type != txLoopedMsg.Type || !bytes.Equal(txLoopedBody.Data, txBody.Data) {
+				continue
+			}
+			txAt, err = parseTimestampFromCmsgs(oob[:oobn])
+			if err != nil {
+				return 0, fmt.Errorf("failed to get tx timestamp: %v", err) // don't wrap
+			}
+			break
+		}
+	}
+
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), txRxTimeout)
+	defer rxCancel()
+
+	rxBuf := make([]byte, 1024)
+	oob := make([]byte, 1024)
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(rxCtx, rxBuf, oob, 0)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg error: %w", err)
+		}
+		rxAt := time.Now()
+		rxMsg, err := icmp.ParseMessage(txMsg.Type.Protocol(), rxBuf[:n])
+		if err != nil {
+			continue
+		}
+		if txMsg.Type == ipv4.ICMPTypeEcho {
+			if rxMsg.Type != ipv4.ICMPTypeEchoReply {
+				continue
+			}
+		} else {
+			if rxMsg.Type != ipv6.ICMPTypeEchoReply {
+				continue
+			}
+		}
+		if rxMsg.Code != txMsg.Code {
+			continue
+		}
+		rxBody, ok := rxMsg.Body.(*icmp.Echo)
+		if !ok || rxBody.Seq != txBody.Seq || !bytes.Equal(rxBody.Data, txBody.Data) {
+			continue
+		}
+		if source == timestampSourceKernel {
+			rxAt, err = parseTimestampFromCmsgs(oob[:oobn])
+			if err != nil {
+				return 0, fmt.Errorf("failed to get rx timestamp: %v", err)
+			}
+		}
+		return rxAt.Sub(txAt), nil
+	}
+}
+
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	sconn, ok := conn.(*socket.Conn)
 	if !ok {
 		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
 	}
 
 	var to unix.Sockaddr
-	to4 := dst.IP.To4()
-	if to4 != nil {
+	if dst.Addr().Is4() {
 		to = &unix.SockaddrInet4{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
 	} else {
 		to = &unix.SockaddrInet6{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
 	}
 
 	txID := stun.NewTxID()
@@ -84,7 +210,7 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 		return 0, fmt.Errorf("sendto error: %v", err) // don't wrap
 	}
 
-	txCtx, txCancel := context.WithTimeout(context.Background(), time.Second*2)
+	txCtx, txCancel := context.WithTimeout(context.Background(), txRxTimeout)
 	defer txCancel()
 
 	buf := make([]byte, 1024)
@@ -110,7 +236,7 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 		break
 	}
 
-	rxCtx, rxCancel := context.WithTimeout(context.Background(), time.Second*2)
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), txRxTimeout)
 	defer rxCancel()
 
 	for {
@@ -138,6 +264,54 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 
 }
 
-func supportsKernelTS() bool {
-	return true
+func getICMPConn(forDst netip.Addr, source timestampSource) (io.ReadWriteCloser, error) {
+	domain := unix.AF_INET
+	proto := unix.IPPROTO_ICMP
+	if forDst.Is6() {
+		domain = unix.AF_INET6
+		proto = unix.IPPROTO_ICMPV6
+	}
+	conn, err := socket.Socket(domain, unix.SOCK_DGRAM, proto, "icmp", nil)
+	if err != nil {
+		return nil, err
+	}
+	if source == timestampSourceKernel {
+		err = conn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, timestampingFlags)
+	}
+	return conn, err
+}
+
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	case protocolICMP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  false,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	// we may restart faster than TIME_WAIT can clear
+	return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
 }

cmd/systray/README.md

@@ -0,0 +1,11 @@
+# systray
+
+The systray command is a minimal Tailscale systray application for Linux.
+It is designed to provide quick access to common operations like profile switching
+and exit node selection.
+
+## Supported platforms
+
+The `fyne.io/systray` package we use supports Windows, macOS, Linux, and many BSDs,
+so the systray application will likely work for the most part on those platforms.
+Notifications currently only work on Linux, as that is the main target.

cmd/systray/logo.go

@@ -0,0 +1,220 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"image/color"
+	"image/png"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/fogleman/gg"
+)
+
+// tsLogo represents the state of the 3x3 dot grid in the Tailscale logo.
+// A 0 represents a gray dot, any other value is a white dot.
+type tsLogo [9]byte
+
+var (
+	// disconnected is all gray dots
+	disconnected = tsLogo{
+		0, 0, 0,
+		0, 0, 0,
+		0, 0, 0,
+	}
+
+	// connected is the normal Tailscale logo
+	connected = tsLogo{
+		0, 0, 0,
+		1, 1, 1,
+		0, 1, 0,
+	}
+
+	// loading is a special tsLogo value that is not meant to be rendered directly,
+	// but indicates that the loading animation should be shown.
+	loading = tsLogo{'l', 'o', 'a', 'd', 'i', 'n', 'g'}
+
+	// loadingIcons are shown in sequence as an animated loading icon.
+	loadingLogos = []tsLogo{
+		{
+			0, 1, 1,
+			1, 0, 1,
+			0, 0, 1,
+		},
+		{
+			0, 1, 1,
+			0, 0, 1,
+			0, 1, 0,
+		},
+		{
+			0, 1, 1,
+			0, 0, 0,
+			0, 0, 1,
+		},
+		{
+			0, 0, 1,
+			0, 1, 0,
+			0, 0, 0,
+		},
+		{
+			0, 1, 0,
+			0, 0, 0,
+			0, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 1,
+			0, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 0,
+			0, 0, 0,
+		},
+		{
+			0, 0, 1,
+			0, 0, 0,
+			0, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 0,
+			1, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 0,
+			1, 1, 0,
+		},
+		{
+			0, 0, 0,
+			1, 0, 0,
+			1, 1, 0,
+		},
+		{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 0,
+		},
+		{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 1,
+		},
+		{
+			0, 0, 0,
+			1, 1, 1,
+			0, 0, 1,
+		},
+		{
+			0, 1, 0,
+			0, 1, 1,
+			1, 0, 1,
+		},
+	}
+)
+
+var (
+	black = color.NRGBA{0, 0, 0, 255}
+	white = color.NRGBA{255, 255, 255, 255}
+	gray  = color.NRGBA{255, 255, 255, 102}
+)
+
+// render returns a PNG image of the logo.
+func (logo tsLogo) render() *bytes.Buffer {
+	const radius = 25
+	const borderUnits = 1
+	dim := radius * (8 + borderUnits*2)
+
+	dc := gg.NewContext(dim, dim)
+	dc.DrawRectangle(0, 0, float64(dim), float64(dim))
+	dc.SetColor(black)
+	dc.Fill()
+
+	for y := 0; y < 3; y++ {
+		for x := 0; x < 3; x++ {
+			px := (borderUnits + 1 + 3*x) * radius
+			py := (borderUnits + 1 + 3*y) * radius
+			col := white
+			if logo[y*3+x] == 0 {
+				col = gray
+			}
+			dc.DrawCircle(float64(px), float64(py), radius)
+			dc.SetColor(col)
+			dc.Fill()
+		}
+	}
+
+	b := bytes.NewBuffer(nil)
+	png.Encode(b, dc.Image())
+	return b
+}
+
+// setAppIcon renders logo and sets it as the systray icon.
+func setAppIcon(icon tsLogo) {
+	if icon == loading {
+		startLoadingAnimation()
+	} else {
+		stopLoadingAnimation()
+		systray.SetIcon(icon.render().Bytes())
+	}
+}
+
+var (
+	loadingMu sync.Mutex // protects loadingCancel
+
+	// loadingCancel stops the loading animation in the systray icon.
+	// This is nil if the animation is not currently active.
+	loadingCancel func()
+)
+
+// startLoadingAnimation starts the animated loading icon in the system tray.
+// The animation continues until [stopLoadingAnimation] is called.
+// If the loading animation is already active, this func does nothing.
+func startLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		// loading icon already displayed
+		return
+	}
+
+	ctx := context.Background()
+	ctx, loadingCancel = context.WithCancel(ctx)
+
+	go func() {
+		t := time.NewTicker(500 * time.Millisecond)
+		var i int
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				systray.SetIcon(loadingLogos[i].render().Bytes())
+				i++
+				if i >= len(loadingLogos) {
+					i = 0
+				}
+			}
+		}
+	}()
+}
+
+// stopLoadingAnimation stops the animated loading icon in the system tray.
+// If the loading animation is not currently active, this func does nothing.
+func stopLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		loadingCancel()
+		loadingCancel = nil
+	}
+}

cmd/systray/systray.go

@@ -0,0 +1,258 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+// The systray command is a minimal Tailscale systray application for Linux.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/atotto/clipboard"
+	dbus "github.com/godbus/dbus/v5"
+	"github.com/toqueteos/webbrowser"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+)
+
+var (
+	localClient tailscale.LocalClient
+	chState     chan ipn.State // tailscale state changes
+
+	appIcon *os.File
+)
+
+func main() {
+	systray.Run(onReady, onExit)
+}
+
+// Menu represents the systray menu, its items, and the current Tailscale state.
+type Menu struct {
+	mu     sync.Mutex // protects the entire Menu
+	status *ipnstate.Status
+
+	connect    *systray.MenuItem
+	disconnect *systray.MenuItem
+
+	self *systray.MenuItem
+	more *systray.MenuItem
+	quit *systray.MenuItem
+
+	eventCancel func() // cancel eventLoop
+}
+
+func onReady() {
+	log.Printf("starting")
+	ctx := context.Background()
+
+	setAppIcon(disconnected)
+
+	// dbus wants a file path for notification icons, so copy to a temp file.
+	appIcon, _ = os.CreateTemp("", "tailscale-systray.png")
+	io.Copy(appIcon, connected.render())
+
+	chState = make(chan ipn.State, 1)
+
+	status, err := localClient.Status(ctx)
+	if err != nil {
+		log.Print(err)
+	}
+
+	menu := new(Menu)
+	menu.rebuild(status)
+
+	go watchIPNBus(ctx)
+}
+
+// rebuild the systray menu based on the current Tailscale state.
+//
+// We currently rebuild the entire menu because it is not easy to update the existing menu.
+// You cannot iterate over the items in a menu, nor can you remove some items like separators.
+// So for now we rebuild the whole thing, and can optimize this later if needed.
+func (menu *Menu) rebuild(status *ipnstate.Status) {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+	menu.status = status
+	systray.ResetMenu()
+
+	menu.connect = systray.AddMenuItem("Connect", "")
+	menu.disconnect = systray.AddMenuItem("Disconnect", "")
+	menu.disconnect.Hide()
+	systray.AddSeparator()
+
+	if status != nil && status.Self != nil {
+		title := fmt.Sprintf("This Device: %s (%s)", status.Self.HostName, status.Self.TailscaleIPs[0])
+		menu.self = systray.AddMenuItem(title, "")
+	}
+	systray.AddSeparator()
+
+	menu.more = systray.AddMenuItem("More settings", "")
+	menu.more.Enable()
+
+	menu.quit = systray.AddMenuItem("Quit", "Quit the app")
+	menu.quit.Enable()
+
+	ctx := context.Background()
+	ctx, menu.eventCancel = context.WithCancel(ctx)
+	go menu.eventLoop(ctx)
+}
+
+// eventLoop is the main event loop for handling click events on menu items
+// and responding to Tailscale state changes.
+// This method does not return until ctx.Done is closed.
+func (menu *Menu) eventLoop(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case state := <-chState:
+			switch state {
+			case ipn.Running:
+				setAppIcon(loading)
+				status, err := localClient.Status(ctx)
+				if err != nil {
+					log.Printf("error getting tailscale status: %v", err)
+				}
+				menu.rebuild(status)
+				setAppIcon(connected)
+				menu.connect.SetTitle("Connected")
+				menu.connect.Disable()
+				menu.disconnect.Show()
+				menu.disconnect.Enable()
+			case ipn.NoState, ipn.Stopped:
+				menu.connect.SetTitle("Connect")
+				menu.connect.Enable()
+				menu.disconnect.Hide()
+				setAppIcon(disconnected)
+			case ipn.Starting:
+				setAppIcon(loading)
+			}
+		case <-menu.connect.ClickedCh:
+			_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: true,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Print(err)
+				continue
+			}
+
+		case <-menu.disconnect.ClickedCh:
+			_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: false,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("disconnecting: %v", err)
+				continue
+			}
+
+		case <-menu.self.ClickedCh:
+			copyTailscaleIP(menu.status.Self)
+
+		case <-menu.more.ClickedCh:
+			webbrowser.Open("http://100.100.100.100/")
+
+		case <-menu.quit.ClickedCh:
+			systray.Quit()
+		}
+	}
+}
+
+// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
+// This method does not return.
+func watchIPNBus(ctx context.Context) {
+	for {
+		if err := watchIPNBusInner(ctx); err != nil {
+			log.Println(err)
+			if errors.Is(err, context.Canceled) {
+				// If the context got canceled, we will never be able to
+				// reconnect to IPN bus, so exit the process.
+				log.Fatalf("watchIPNBus: %v", err)
+			}
+		}
+		// If our watch connection breaks, wait a bit before reconnecting. No
+		// reason to spam the logs if e.g. tailscaled is restarting or goes
+		// down.
+		time.Sleep(3 * time.Second)
+	}
+}
+
+func watchIPNBusInner(ctx context.Context) error {
+	watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return fmt.Errorf("watching ipn bus: %w", err)
+	}
+	defer watcher.Close()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			n, err := watcher.Next()
+			if err != nil {
+				return fmt.Errorf("ipnbus error: %w", err)
+			}
+			if n.State != nil {
+				chState <- *n.State
+				log.Printf("new state: %v", n.State)
+			}
+		}
+	}
+}
+
+// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
+// and sends a notification with the copied value.
+func copyTailscaleIP(device *ipnstate.PeerStatus) {
+	if device == nil || len(device.TailscaleIPs) == 0 {
+		return
+	}
+	name := strings.Split(device.DNSName, ".")[0]
+	ip := device.TailscaleIPs[0].String()
+	err := clipboard.WriteAll(ip)
+	if err != nil {
+		log.Printf("clipboard error: %v", err)
+	}
+
+	sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
+}
+
+// sendNotification sends a desktop notification with the given title and content.
+func sendNotification(title, content string) {
+	conn, err := dbus.SessionBus()
+	if err != nil {
+		log.Printf("dbus: %v", err)
+		return
+	}
+	timeout := 3 * time.Second
+	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
+	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
+		appIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
+	if call.Err != nil {
+		log.Printf("dbus: %v", call.Err)
+	}
+}
+
+func onExit() {
+	log.Printf("exiting")
+	os.Remove(appIcon.Name())
+}

cmd/tailscale/cli/cli.go

@@ -84,6 +84,13 @@ var localClient = tailscale.LocalClient{
 
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) (err error) {
+	if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 {
+		// We're running on gokrazy and it's the first start.
+		// Don't run the tailscale CLI as a service; just exit.
+		// See https://gokrazy.org/development/process-interface/
+		os.Exit(0)
+	}
+
 	args = CleanUpArgs(args)
 
 	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
@@ -180,6 +187,7 @@ change in the future.
 			configureCmd,
 			netcheckCmd,
 			ipCmd,
+			dnsCmd,
 			statusCmd,
 			pingCmd,
 			ncCmd,

cmd/tailscale/cli/debug.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"time"
@@ -319,9 +320,36 @@ var debugCmd = &ffcli.Command{
 				return fs
 			})(),
 		},
+		{
+			Name:       "resolve",
+			ShortUsage: "tailscale debug resolve <hostname>",
+			Exec:       runDebugResolve,
+			ShortHelp:  "Does a DNS lookup",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("resolve")
+				fs.StringVar(&resolveArgs.net, "net", "ip", "network type to resolve (ip, ip4, ip6)")
+				return fs
+			})(),
+		},
+		{
+			Name:       "go-buildinfo",
+			ShortUsage: "tailscale debug go-buildinfo",
+			ShortHelp:  "Prints Go's runtime/debug.BuildInfo",
+			Exec:       runGoBuildInfo,
+		},
 	},
 }
 
+func runGoBuildInfo(ctx context.Context, args []string) error {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		return errors.New("no Go build info")
+	}
+	e := json.NewEncoder(os.Stdout)
+	e.SetIndent("", "\t")
+	return e.Encode(bi)
+}
+
 var debugArgs struct {
 	file    string
 	cpuSec  int
@@ -1167,3 +1195,26 @@ func runDebugDialTypes(ctx context.Context, args []string) error {
 	fmt.Printf("%s", body)
 	return nil
 }
+
+var resolveArgs struct {
+	net string // "ip", "ip4", "ip6""
+}
+
+func runDebugResolve(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: tailscale debug resolve <hostname>")
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	host := args[0]
+	ips, err := net.DefaultResolver.LookupIP(ctx, resolveArgs.net, host)
+	if err != nil {
+		return err
+	}
+	for _, ip := range ips {
+		fmt.Printf("%s\n", ip)
+	}
+	return nil
+}

cmd/tailscale/cli/dns-query.go

@@ -0,0 +1,163 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/netip"
+	"os"
+	"text/tabwriter"
+
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/types/dnstype"
+)
+
+func runDNSQuery(ctx context.Context, args []string) error {
+	if len(args) < 1 {
+		return flag.ErrHelp
+	}
+	name := args[0]
+	queryType := "A"
+	if len(args) >= 2 {
+		queryType = args[1]
+	}
+	fmt.Printf("DNS query for %q (%s) using internal resolver:\n", name, queryType)
+	fmt.Println()
+	bytes, resolvers, err := localClient.QueryDNS(ctx, name, queryType)
+	if err != nil {
+		fmt.Printf("failed to query DNS: %v\n", err)
+		return nil
+	}
+
+	if len(resolvers) == 1 {
+		fmt.Printf("Forwarding to resolver: %v\n", makeResolverString(*resolvers[0]))
+	} else {
+		fmt.Println("Multiple resolvers available:")
+		for _, r := range resolvers {
+			fmt.Printf("  - %v\n", makeResolverString(*r))
+		}
+	}
+	fmt.Println()
+	var p dnsmessage.Parser
+	header, err := p.Start(bytes)
+	if err != nil {
+		fmt.Printf("failed to parse DNS response: %v\n", err)
+		return err
+	}
+	fmt.Printf("Response code: %v\n", header.RCode.String())
+	fmt.Println()
+	p.SkipAllQuestions()
+	if header.RCode != dnsmessage.RCodeSuccess {
+		fmt.Println("No answers were returned.")
+		return nil
+	}
+	answers, err := p.AllAnswers()
+	if err != nil {
+		fmt.Printf("failed to parse DNS answers: %v\n", err)
+		return err
+	}
+	if len(answers) == 0 {
+		fmt.Println("  (no answers found)")
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "Name\tTTL\tClass\tType\tBody")
+	fmt.Fprintln(w, "----\t---\t-----\t----\t----")
+	for _, a := range answers {
+		fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", a.Header.Name.String(), a.Header.TTL, a.Header.Class.String(), a.Header.Type.String(), makeAnswerBody(a))
+	}
+	w.Flush()
+
+	fmt.Println()
+	return nil
+}
+
+// makeAnswerBody returns a string with the DNS answer body in a human-readable format.
+func makeAnswerBody(a dnsmessage.Resource) string {
+	switch a.Header.Type {
+	case dnsmessage.TypeA:
+		return makeABody(a.Body)
+	case dnsmessage.TypeAAAA:
+		return makeAAAABody(a.Body)
+	case dnsmessage.TypeCNAME:
+		return makeCNAMEBody(a.Body)
+	case dnsmessage.TypeMX:
+		return makeMXBody(a.Body)
+	case dnsmessage.TypeNS:
+		return makeNSBody(a.Body)
+	case dnsmessage.TypeOPT:
+		return makeOPTBody(a.Body)
+	case dnsmessage.TypePTR:
+		return makePTRBody(a.Body)
+	case dnsmessage.TypeSRV:
+		return makeSRVBody(a.Body)
+	case dnsmessage.TypeTXT:
+		return makeTXTBody(a.Body)
+	default:
+		return a.Body.GoString()
+	}
+}
+
+func makeABody(a dnsmessage.ResourceBody) string {
+	if a, ok := a.(*dnsmessage.AResource); ok {
+		return netip.AddrFrom4(a.A).String()
+	}
+	return ""
+}
+func makeAAAABody(aaaa dnsmessage.ResourceBody) string {
+	if a, ok := aaaa.(*dnsmessage.AAAAResource); ok {
+		return netip.AddrFrom16(a.AAAA).String()
+	}
+	return ""
+}
+func makeCNAMEBody(cname dnsmessage.ResourceBody) string {
+	if c, ok := cname.(*dnsmessage.CNAMEResource); ok {
+		return c.CNAME.String()
+	}
+	return ""
+}
+func makeMXBody(mx dnsmessage.ResourceBody) string {
+	if m, ok := mx.(*dnsmessage.MXResource); ok {
+		return fmt.Sprintf("%s (Priority=%d)", m.MX, m.Pref)
+	}
+	return ""
+}
+func makeNSBody(ns dnsmessage.ResourceBody) string {
+	if n, ok := ns.(*dnsmessage.NSResource); ok {
+		return n.NS.String()
+	}
+	return ""
+}
+func makeOPTBody(opt dnsmessage.ResourceBody) string {
+	if o, ok := opt.(*dnsmessage.OPTResource); ok {
+		return o.GoString()
+	}
+	return ""
+}
+func makePTRBody(ptr dnsmessage.ResourceBody) string {
+	if p, ok := ptr.(*dnsmessage.PTRResource); ok {
+		return p.PTR.String()
+	}
+	return ""
+}
+func makeSRVBody(srv dnsmessage.ResourceBody) string {
+	if s, ok := srv.(*dnsmessage.SRVResource); ok {
+		return fmt.Sprintf("Target=%s, Port=%d, Priority=%d, Weight=%d", s.Target.String(), s.Port, s.Priority, s.Weight)
+	}
+	return ""
+}
+func makeTXTBody(txt dnsmessage.ResourceBody) string {
+	if t, ok := txt.(*dnsmessage.TXTResource); ok {
+		return fmt.Sprintf("%q", t.TXT)
+	}
+	return ""
+}
+func makeResolverString(r dnstype.Resolver) string {
+	if len(r.BootstrapResolution) > 0 {
+		return fmt.Sprintf("%s (bootstrap: %v)", r.Addr, r.BootstrapResolution)
+	}
+	return fmt.Sprintf("%s", r.Addr)
+}

cmd/tailscale/cli/dns-status.go

@@ -0,0 +1,242 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+
+	"tailscale.com/ipn"
+	"tailscale.com/types/netmap"
+)
+
+// dnsStatusArgs are the arguments for the "dns status" subcommand.
+var dnsStatusArgs struct {
+	all bool
+}
+
+func runDNSStatus(ctx context.Context, args []string) error {
+	all := dnsStatusArgs.all
+	s, err := localClient.Status(ctx)
+	if err != nil {
+		return err
+	}
+
+	prefs, err := localClient.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+	enabledStr := "disabled.\n\n(Run 'tailscale set --accept-dns=true' to start sending DNS queries to the Tailscale DNS resolver)"
+	if prefs.CorpDNS {
+		enabledStr = "enabled.\n\nTailscale is configured to handle DNS queries on this device.\nRun 'tailscale set --accept-dns=false' to revert to your system default DNS resolver."
+	}
+	fmt.Print("\n")
+	fmt.Println("=== 'Use Tailscale DNS' status ===")
+	fmt.Print("\n")
+	fmt.Printf("Tailscale DNS: %s\n", enabledStr)
+	fmt.Print("\n")
+	fmt.Println("=== MagicDNS configuration ===")
+	fmt.Print("\n")
+	fmt.Println("This is the DNS configuration provided by the coordination server to this device.")
+	fmt.Print("\n")
+	if s.CurrentTailnet == nil {
+		fmt.Println("No tailnet information available; make sure you're logged in to a tailnet.")
+		return nil
+	} else if s.CurrentTailnet.MagicDNSEnabled {
+		fmt.Printf("MagicDNS: enabled tailnet-wide (suffix = %s)", s.CurrentTailnet.MagicDNSSuffix)
+		fmt.Print("\n\n")
+		fmt.Printf("Other devices in your tailnet can reach this device at %s\n", s.Self.DNSName)
+	} else {
+		fmt.Printf("MagicDNS: disabled tailnet-wide.\n")
+	}
+	fmt.Print("\n")
+
+	netMap, err := fetchNetMap()
+	if err != nil {
+		fmt.Printf("Failed to fetch network map: %v\n", err)
+		return err
+	}
+	dnsConfig := netMap.DNS
+	fmt.Println("Resolvers (in preference order):")
+	if len(dnsConfig.Resolvers) == 0 {
+		fmt.Println("  (no resolvers configured, system default will be used: see 'System DNS configuration' below)")
+	}
+	for _, r := range dnsConfig.Resolvers {
+		fmt.Printf("  - %v", r.Addr)
+		if r.BootstrapResolution != nil {
+			fmt.Printf(" (bootstrap: %v)", r.BootstrapResolution)
+		}
+		fmt.Print("\n")
+	}
+	fmt.Print("\n")
+	fmt.Println("Split DNS Routes:")
+	if len(dnsConfig.Routes) == 0 {
+		fmt.Println("  (no routes configured: split DNS disabled)")
+	}
+	for _, k := range slices.Sorted(maps.Keys(dnsConfig.Routes)) {
+		v := dnsConfig.Routes[k]
+		for _, r := range v {
+			fmt.Printf("  - %-30s -> %v", k, r.Addr)
+			if r.BootstrapResolution != nil {
+				fmt.Printf(" (bootstrap: %v)", r.BootstrapResolution)
+			}
+			fmt.Print("\n")
+		}
+	}
+	fmt.Print("\n")
+	if all {
+		fmt.Println("Fallback Resolvers:")
+		if len(dnsConfig.FallbackResolvers) == 0 {
+			fmt.Println("  (no fallback resolvers configured)")
+		}
+		for i, r := range dnsConfig.FallbackResolvers {
+			fmt.Printf("  %d: %v\n", i, r)
+		}
+		fmt.Print("\n")
+	}
+	fmt.Println("Search Domains:")
+	if len(dnsConfig.Domains) == 0 {
+		fmt.Println("  (no search domains configured)")
+	}
+	domains := dnsConfig.Domains
+	slices.Sort(domains)
+	for _, r := range domains {
+		fmt.Printf("  - %v\n", r)
+	}
+	fmt.Print("\n")
+	if all {
+		fmt.Println("Nameservers IP Addresses:")
+		if len(dnsConfig.Nameservers) == 0 {
+			fmt.Println("  (none were provided)")
+		}
+		for _, r := range dnsConfig.Nameservers {
+			fmt.Printf("  - %v\n", r)
+		}
+		fmt.Print("\n")
+		fmt.Println("Certificate Domains:")
+		if len(dnsConfig.CertDomains) == 0 {
+			fmt.Println("  (no certificate domains are configured)")
+		}
+		for _, r := range dnsConfig.CertDomains {
+			fmt.Printf("  - %v\n", r)
+		}
+		fmt.Print("\n")
+		fmt.Println("Additional DNS Records:")
+		if len(dnsConfig.ExtraRecords) == 0 {
+			fmt.Println("  (no extra records are configured)")
+		}
+		for _, er := range dnsConfig.ExtraRecords {
+			if er.Type == "" {
+				fmt.Printf("  - %-50s -> %v\n", er.Name, er.Value)
+			} else {
+				fmt.Printf("  - [%s] %-50s -> %v\n", er.Type, er.Name, er.Value)
+			}
+		}
+		fmt.Print("\n")
+		fmt.Println("Filtered suffixes when forwarding DNS queries as an exit node:")
+		if len(dnsConfig.ExitNodeFilteredSet) == 0 {
+			fmt.Println("  (no suffixes are filtered)")
+		}
+		for _, s := range dnsConfig.ExitNodeFilteredSet {
+			fmt.Printf("  - %s\n", s)
+		}
+		fmt.Print("\n")
+	}
+
+	fmt.Println("=== System DNS configuration ===")
+	fmt.Print("\n")
+	fmt.Println("This is the DNS configuration that Tailscale believes your operating system is using.\nTailscale may use this configuration if 'Override Local DNS' is disabled in the admin console,\nor if no resolvers are provided by the coordination server.")
+	fmt.Print("\n")
+	osCfg, err := localClient.GetDNSOSConfig(ctx)
+	if err != nil {
+		if strings.Contains(err.Error(), "not supported") {
+			// avoids showing the HTTP error code which would be odd here
+			fmt.Println("  (reading the system DNS configuration is not supported on this platform)")
+		} else {
+			fmt.Printf("  (failed to read system DNS configuration: %v)\n", err)
+		}
+	} else if osCfg == nil {
+		fmt.Println("  (no OS DNS configuration available)")
+	} else {
+		fmt.Println("Nameservers:")
+		if len(osCfg.Nameservers) == 0 {
+			fmt.Println("  (no nameservers found, DNS queries might fail\nunless the coordination server is providing a nameserver)")
+		}
+		for _, ns := range osCfg.Nameservers {
+			fmt.Printf("  - %v\n", ns)
+		}
+		fmt.Print("\n")
+		fmt.Println("Search domains:")
+		if len(osCfg.SearchDomains) == 0 {
+			fmt.Println("  (no search domains found)")
+		}
+		for _, sd := range osCfg.SearchDomains {
+			fmt.Printf("  - %v\n", sd)
+		}
+		if all {
+			fmt.Print("\n")
+			fmt.Println("Match domains:")
+			if len(osCfg.MatchDomains) == 0 {
+				fmt.Println("  (no match domains found)")
+			}
+			for _, md := range osCfg.MatchDomains {
+				fmt.Printf("  - %v\n", md)
+			}
+		}
+	}
+	fmt.Print("\n")
+	fmt.Println("[this is a preliminary version of this command; the output format may change in the future]")
+	return nil
+}
+
+func fetchNetMap() (netMap *netmap.NetworkMap, err error) {
+	w, err := localClient.WatchIPNBus(context.Background(), ipn.NotifyInitialNetMap)
+	if err != nil {
+		return nil, err
+	}
+	defer w.Close()
+	notify, err := w.Next()
+	if err != nil {
+		return nil, err
+	}
+	if notify.NetMap == nil {
+		return nil, fmt.Errorf("no network map yet available, please try again later")
+	}
+	return notify.NetMap, nil
+}
+
+func dnsStatusLongHelp() string {
+	return `The 'tailscale dns status' subcommand prints the current DNS status and configuration, including:
+	
+- Whether the built-in DNS forwarder is enabled.
+- The MagicDNS configuration provided by the coordination server.
+- Details on which resolver(s) Tailscale believes the system is using by default.
+
+The --all flag can be used to output advanced debugging information, including fallback resolvers, nameservers, certificate domains, extra records, and the exit node filtered set.
+
+=== Contents of the MagicDNS configuration ===
+
+The MagicDNS configuration is provided by the coordination server to the client and includes the following components:
+
+- MagicDNS enablement status: Indicates whether MagicDNS is enabled across the entire tailnet.
+
+- MagicDNS Suffix: The DNS suffix used for devices within your tailnet.
+
+- DNS Name: The DNS name that other devices in the tailnet can use to reach this device.
+
+- Resolvers: The preferred DNS resolver(s) to be used for resolving queries, in order of preference. If no resolvers are listed here, the system defaults are used.
+
+- Split DNS Routes: Custom DNS resolvers may be used to resolve hostnames in specific domains, this is also known as a 'Split DNS' configuration. The mapping of domains to their respective resolvers is provided here.
+
+- Certificate Domains: The DNS names for which the coordination server will assist in provisioning TLS certificates.
+
+- Extra Records: Additional DNS records that the coordination server might provide to the internal DNS resolver.
+
+- Exit Node Filtered Set: DNS suffixes that the node, when acting as an exit node DNS proxy, will not answer.
+
+For more information about the DNS functionality built into Tailscale, refer to https://tailscale.com/kb/1054/dns.`
+}

cmd/tailscale/cli/dns-stream.go

@@ -0,0 +1,72 @@
+package cli
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+)
+
+func runDNSStream(ctx context.Context, args []string) error {
+	fmt.Printf(`Privacy warning! To stream DNS queries, this tool will set these Tailscale debug flags, which would normally be disabled by default:
+
+   - TS_DEBUG_DNS_FORWARD_SEND=true
+   - TS_DEBUG_DNS_INCLUDE_NAMES=true
+
+TS_DEBUG_DNS_FORWARD_SEND instructs Tailscale to log DNS queries and responses as they are handled by the internal DNS forwarder. 
+
+TS_DEBUG_DNS_INCLUDE_NAMES instructs Tailscale to include queried and resolved DNS hostnames in the logs.
+
+Unless the 'TS_NO_LOGS_NO_SUPPORT' flag was previously set, logs are uploaded to Tailscale for diagnostic and debugging purposes, which can be a concern in privacy-sensitive environments.
+
+If you are concerned about the privacy implications of this, run this tool with the '--no-names' flag, which will avoid logging hostnames.`)
+	fmt.Printf("\n\n")
+	fmt.Println("Press Enter to start streaming DNS logs, or Ctrl+C to quit this tool.")
+
+	buf := bufio.NewReader(os.Stdin)
+	_, err := buf.ReadBytes('\n')
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+
+	err = localClient.DebugEnvknob(ctx, "TS_DEBUG_DNS_FORWARD_SEND", "true")
+	if err != nil {
+		fmt.Printf("failed to set TS_DEBUG_DNS_FORWARD_SEND=true: %v\n", err)
+		return nil
+	}
+	err = localClient.DebugEnvknob(ctx, "TS_DEBUG_DNS_INCLUDE_NAMES", "true")
+	if err != nil {
+		fmt.Printf("failed to set TS_DEBUG_DNS_INCLUDE_NAMES=true: %v\n", err)
+		return nil
+	}
+
+	logs, err := localClient.TailDaemonLogs(ctx)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("Streaming DNS logs. Press Ctrl+C to stop.")
+
+	d := json.NewDecoder(logs)
+	for {
+		var line struct {
+			Text    string `json:"text"`
+			Verbose int    `json:"v"`
+			Time    string `json:"client_time"`
+		}
+		err := d.Decode(&line)
+		if err != nil {
+			return err
+		}
+		text := strings.TrimSpace(line.Text)
+		dnsPrefix := "dns: resolver: forward: "
+		if !strings.HasPrefix(text, dnsPrefix) {
+			continue
+		}
+		text = strings.TrimPrefix(text, dnsPrefix)
+		fmt.Println(text)
+	}
+}

cmd/tailscale/cli/dns.go

@@ -0,0 +1,54 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"flag"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+)
+
+var dnsCmd = &ffcli.Command{
+	Name:       "dns",
+	ShortHelp:  "Diagnose the internal DNS forwarder",
+	LongHelp:   dnsCmdLongHelp(),
+	ShortUsage: "tailscale dns <subcommand> [flags]",
+	UsageFunc:  usageFuncNoDefaultValues,
+	Subcommands: []*ffcli.Command{
+		{
+			Name:       "status",
+			ShortUsage: "tailscale dns status [--all]",
+			Exec:       runDNSStatus,
+			ShortHelp:  "Prints the current DNS status and configuration",
+			LongHelp:   dnsStatusLongHelp(),
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("status")
+				fs.BoolVar(&dnsStatusArgs.all, "all", false, "outputs advanced debugging information (fallback resolvers, nameservers, cert domains, extra records, and exit node filtered set)")
+				return fs
+			})(),
+		},
+		{
+			Name:       "query",
+			ShortUsage: "tailscale dns query <name> [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]",
+			Exec:       runDNSQuery,
+			ShortHelp:  "Perform a DNS query",
+			LongHelp:   "The 'tailscale dns query' subcommand performs a DNS query for the specified name using the internal DNS forwarder (100.100.100.100).\n\nIt also provides information about the resolver(s) used to resolve the query.",
+		},
+		{
+			Name:       "stream",
+			ShortUsage: "tailscale dns stream",
+			Exec:       runDNSStream,
+			ShortHelp:  "Stream DNS queries and responses",
+			LongHelp:   "The 'tailscale dns stream' subcommand streams DNS queries and responses to and from the internal DNS forwarder, which is useful for debugging DNS issues.",
+		},
+
+		// The above work is tracked in https://github.com/tailscale/tailscale/issues/13326
+	},
+}
+
+func dnsCmdLongHelp() string {
+	return `The 'tailscale dns' subcommand provides tools for diagnosing the internal DNS forwarder (100.100.100.100).
+	
+For more information about the DNS functionality built into Tailscale, refer to https://tailscale.com/kb/1054/dns.`
+}

cmd/tailscale/cli/exitnode_test.go

@@ -135,7 +135,7 @@ func TestFilterFormatAndSortExitNodes(t *testing.T) {
 		result := filterFormatAndSortExitNodes(ps, "")
 
 		if res := cmp.Diff(result.Countries, want.Countries, cmpopts.IgnoreUnexported(key.NodePublic{})); res != "" {
-			t.Fatalf(res)
+			t.Fatal(res)
 		}
 	})
 
@@ -230,7 +230,7 @@ func TestFilterFormatAndSortExitNodes(t *testing.T) {
 		result := filterFormatAndSortExitNodes(ps, "Pacific")
 
 		if res := cmp.Diff(result.Countries, want.Countries, cmpopts.IgnoreUnexported(key.NodePublic{})); res != "" {
-			t.Fatalf(res)
+			t.Fatal(res)
 		}
 	})
 }

cmd/tailscale/cli/netcheck.go

@@ -52,9 +52,15 @@ func runNetcheck(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
+
+	// Ensure that we close the portmapper after running a netcheck; this
+	// will release any port mappings created.
+	pm := portmapper.NewClient(logf, netMon, nil, nil, nil)
+	defer pm.Close()
+
 	c := &netcheck.Client{
 		NetMon:      netMon,
-		PortMapper:  portmapper.NewClient(logf, netMon, nil, nil, nil),
+		PortMapper:  pm,
 		UseDNSCache: false, // always resolve, don't cache
 	}
 	if netcheckArgs.verbose {

cmd/tailscale/cli/network-lock.go

@@ -20,6 +20,7 @@ import (
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tka"
+	"tailscale.com/tsconst"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
 )
@@ -443,15 +444,33 @@ func runNetworkLockModify(ctx context.Context, addArgs, removeArgs []string) err
 
 var nlSignCmd = &ffcli.Command{
 	Name:       "sign",
-	ShortUsage: "tailscale lock sign <node-key> [<rotation-key>] or sign <auth-key>",
+	ShortUsage: "tailscale lock sign <node-key> [<rotation-key>]\ntailscale lock sign <auth-key>",
 	ShortHelp:  "Signs a node or pre-approved auth key",
 	LongHelp: `Either:
-  - signs a node key and transmits the signature to the coordination server, or
-  - signs a pre-approved auth key, printing it in a form that can be used to bring up nodes under tailnet lock`,
+  - signs a node key and transmits the signature to the coordination
+    server, or
+  - signs a pre-approved auth key, printing it in a form that can be
+    used to bring up nodes under tailnet lock
+
+If any of the key arguments begin with "file:", the key is retrieved from
+the file at the path specified in the argument suffix.`,
 	Exec: runNetworkLockSign,
 }
 
 func runNetworkLockSign(ctx context.Context, args []string) error {
+	// If any of the arguments start with "file:", replace that argument
+	// with the contents of the file. We do this early, before the check
+	// to see if the first argument is an auth key.
+	for i, arg := range args {
+		if filename, ok := strings.CutPrefix(arg, "file:"); ok {
+			b, err := os.ReadFile(filename)
+			if err != nil {
+				return err
+			}
+			args[i] = strings.TrimSpace(string(b))
+		}
+	}
+
 	if len(args) > 0 && strings.HasPrefix(args[0], "tskey-auth-") {
 		return runTskeyWrapCmd(ctx, args)
 	}
@@ -476,7 +495,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error {
 	err := localClient.NetworkLockSign(ctx, nodeKey, []byte(rotationKey.Verifier()))
 	// Provide a better help message for when someone clicks through the signing flow
 	// on the wrong device.
-	if err != nil && strings.Contains(err.Error(), "this node is not trusted by network lock") {
+	if err != nil && strings.Contains(err.Error(), tsconst.TailnetLockNotTrustedMsg) {
 		fmt.Fprintln(Stderr, "Error: Signing is not available on this device because it does not have a trusted tailnet lock key.")
 		fmt.Fprintln(Stderr)
 		fmt.Fprintln(Stderr, "Try again on a signing device instead. Tailnet admins can see signing devices on the admin panel.")
@@ -789,7 +808,7 @@ func runNetworkLockRevokeKeys(ctx context.Context, args []string) error {
 		}
 
 		fmt.Printf(`Run the following command on another machine with a trusted tailnet lock key:
-	%s lock recover-compromised-key --cosign %X
+	%s lock revoke-keys --cosign %X
 `, os.Args[0], aumBytes)
 		return nil
 	}
@@ -813,10 +832,10 @@ func runNetworkLockRevokeKeys(ctx context.Context, args []string) error {
 		fmt.Printf(`Co-signing completed successfully.
 
 To accumulate an additional signature, run the following command on another machine with a trusted tailnet lock key:
-	%s lock recover-compromised-key --cosign %X
+	%s lock revoke-keys --cosign %X
 
 Alternatively if you are done with co-signing, complete recovery by running the following command:
-	%s lock recover-compromised-key --finish %X
+	%s lock revoke-keys --finish %X
 `, os.Args[0], aumBytes, os.Args[0], aumBytes)
 	}
 

cmd/tailscale/depaware.txt

@@ -5,11 +5,15 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -56,20 +60,16 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
         github.com/toqueteos/webbrowser                              from tailscale.com/cmd/tailscale/cli
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/netipx                                               from tailscale.com/net/tsaddr
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         k8s.io/client-go/util/homedir                                from tailscale.com/cmd/tailscale/cli
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         sigs.k8s.io/yaml                                             from tailscale.com/cmd/tailscale/cli
         sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml
         software.sslmate.com/src/go-pkcs12                           from tailscale.com/cmd/tailscale/cli
@@ -98,11 +98,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/internal/noiseconn                             from tailscale.com/cmd/tailscale/cli
         tailscale.com/ipn                                            from tailscale.com/client/tailscale+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/licenses                                       from tailscale.com/client/web+
-        tailscale.com/metrics                                        from tailscale.com/derp
+        tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/captivedetection                           from tailscale.com/net/netcheck
         tailscale.com/net/dns/recursive                              from tailscale.com/net/dnsfallback
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlhttp+
-        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlhttp
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlhttp+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netcheck                                   from tailscale.com/cmd/tailscale/cli
@@ -127,17 +129,18 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/tempfork/spf13/cobra                           from tailscale.com/cmd/tailscale/cli/ffcomplete+
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tstime                                         from tailscale.com/control/controlhttp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
         tailscale.com/types/lazy                                     from tailscale.com/util/testenv+
         tailscale.com/types/logger                                   from tailscale.com/client/web+
-        tailscale.com/types/netmap                                   from tailscale.com/ipn
+        tailscale.com/types/netmap                                   from tailscale.com/ipn+
         tailscale.com/types/nettype                                  from tailscale.com/net/netcheck+
         tailscale.com/types/opt                                      from tailscale.com/client/tailscale+
         tailscale.com/types/persist                                  from tailscale.com/ipn
@@ -151,9 +154,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dnscache+
         tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy+
         tailscale.com/util/ctxkey                                    from tailscale.com/types/logger
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/groupmember                               from tailscale.com/client/web
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -166,8 +171,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate
@@ -189,8 +197,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
-        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli
+        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
@@ -278,6 +287,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/clientupdate+
@@ -307,9 +317,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
-        runtime/debug                                                from nhooyr.io/websocket/internal/xsync+
+        runtime/debug                                                from github.com/coder/websocket/internal/xsync+
         slices                                                       from tailscale.com/client/web+
-        sort                                                         from archive/tar+
+        sort                                                         from compress/flate+
         strconv                                                      from archive/tar+
         strings                                                      from archive/tar+
         sync                                                         from archive/tar+
@@ -322,3 +332,4 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/tailscaled/depaware.txt

@@ -79,6 +79,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -90,7 +94,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 github.com/djherbis/times                                    from tailscale.com/drive/driveimpl
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/tstun+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -111,7 +115,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
+   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/interfaces                      from github.com/insomniacslk/dhcp/dhcpv4
@@ -135,7 +139,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink+
         github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
    L    github.com/pierrec/lz4/v4                                    from github.com/u-root/uio/uio
@@ -165,6 +169,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tailscale/hujson                                  from tailscale.com/ipn/conffile
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/net/routetable+
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/peercred                                from tailscale.com/ipn/ipnauth
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
    W 💣 github.com/tailscale/wf                                      from tailscale.com/wf
@@ -184,7 +189,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD    github.com/u-root/u-root/pkg/termios                         from tailscale.com/ssh/tailssh
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
@@ -212,7 +216,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
         gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
-        gvisor.dev/gvisor/pkg/tcpip/link/channel                     from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
         gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
@@ -222,6 +225,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack/gro
         gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
         gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
@@ -232,10 +236,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack           from gvisor.dev/gvisor/pkg/tcpip/stack
         gvisor.dev/gvisor/pkg/tcpip/transport/udp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
         gvisor.dev/gvisor/pkg/waiter                                 from gvisor.dev/gvisor/pkg/context+
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
@@ -279,7 +279,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    tailscale.com/ipn/store/awsstore                             from tailscale.com/ipn/store
    L    tailscale.com/ipn/store/kubestore                            from tailscale.com/ipn/store
         tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/ipnlocal+
-   L    tailscale.com/kube                                           from tailscale.com/ipn/store/kubestore
+   L    tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
+   L    tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -288,6 +290,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns+
@@ -329,6 +332,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
+  LD    tailscale.com/sessionrecording                               from tailscale.com/ssh/tailssh
   LD 💣 tailscale.com/ssh/tailssh                                    from tailscale.com/cmd/tailscaled
         tailscale.com/syncs                                          from tailscale.com/cmd/tailscaled+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
@@ -336,12 +340,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/cmd/tailscaled+
         tailscale.com/tstime                                         from tailscale.com/control/controlclient+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
-        tailscale.com/tsweb/varz                                     from tailscale.com/cmd/tailscaled
+        tailscale.com/tsweb/varz                                     from tailscale.com/cmd/tailscaled+
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
@@ -384,7 +388,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/osdiag                                    from tailscale.com/cmd/tailscaled+
    W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
-        tailscale.com/util/osuser                                    from tailscale.com/ipn/localapi+
+        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/progresstracking                          from tailscale.com/ipn/localapi
         tailscale.com/util/race                                      from tailscale.com/net/dns/resolver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -394,11 +398,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/cmd/tailscaled+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
         tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/usermetric                                from tailscale.com/health+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
@@ -416,6 +423,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
+        tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -434,8 +442,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
   LD    golang.org/x/crypto/ssh                                      from github.com/pkg/sftp+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from tailscale.com/appc+
@@ -525,6 +534,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
   LD    log/syslog                                                   from tailscale.com/ssh/tailssh
@@ -560,7 +570,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/appc+
-        sort                                                         from archive/tar+
+        sort                                                         from compress/flate+
         strconv                                                      from archive/tar+
         strings                                                      from archive/tar+
         sync                                                         from archive/tar+
@@ -573,3 +583,4 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/tailscaled/required_version.go

@@ -1,10 +1,10 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !go1.21
+//go:build !go1.23
 
 package main
 
 func init() {
-	you_need_Go_1_21_to_compile_Tailscale()
+	you_need_Go_1_23_to_compile_Tailscale()
 }

cmd/tailscaled/tailscaled.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build go1.21
+//go:build go1.23
 
 // The tailscaled program is the Tailscale client daemon. It's configured
 // and controlled via the tailscale CLI program.
@@ -35,6 +35,7 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive/driveimpl"
 	"tailscale.com/envknob"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/conffile"
 	"tailscale.com/ipn/ipnlocal"
@@ -154,9 +155,11 @@ var beCLI func() // non-nil if CLI is linked in
 func main() {
 	envknob.PanicIfAnyEnvCheckedInInit()
 	envknob.ApplyDiskConfig()
+	applyIntegrationTestEnvKnob()
 
+	defaultVerbosity := envknob.RegisterInt("TS_LOG_VERBOSITY")
 	printVersion := false
-	flag.IntVar(&args.verbose, "verbose", 0, "log verbosity level; 0 is default, 1 or higher are increasingly verbose")
+	flag.IntVar(&args.verbose, "verbose", defaultVerbosity(), "log verbosity level; 0 is default, 1 or higher are increasingly verbose")
 	flag.BoolVar(&args.cleanUp, "cleanup", false, "clean up system state and exit")
 	flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server")
 	flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`)
@@ -394,7 +397,7 @@ func run() (err error) {
 	// Always clean up, even if we're going to run the server. This covers cases
 	// such as when a system was rebooted without shutting down, or tailscaled
 	// crashed, and would for example restore system DNS configuration.
-	dns.CleanUp(logf, netMon, args.tunname)
+	dns.CleanUp(logf, netMon, sys.HealthTracker(), args.tunname)
 	router.CleanUp(logf, netMon, args.tunname)
 	// If the cleanUp flag was passed, then exit.
 	if args.cleanUp {
@@ -414,6 +417,10 @@ func run() (err error) {
 
 	sys.Set(driveimpl.NewFileSystemForRemote(logf))
 
+	if app := envknob.App(); app != "" {
+		hostinfo.SetApp(app)
+	}
+
 	return startIPNServer(context.Background(), logf, pol.PublicID, sys)
 }
 
@@ -895,3 +902,24 @@ func dieOnPipeReadErrorOfFD(fd int) {
 	f.Read(make([]byte, 1))
 	os.Exit(1)
 }
+
+// applyIntegrationTestEnvKnob applies the tailscaled.env=... environment
+// variables specified on the Linux kernel command line, if the VM is being
+// run in NATLab integration tests.
+//
+// They're specified as: tailscaled.env=FOO=bar tailscaled.env=BAR=baz
+func applyIntegrationTestEnvKnob() {
+	if runtime.GOOS != "linux" || !hostinfo.IsNATLabGuestVM() {
+		return
+	}
+	cmdLine, _ := os.ReadFile("/proc/cmdline")
+	for _, s := range strings.Fields(string(cmdLine)) {
+		suf, ok := strings.CutPrefix(s, "tailscaled.env=")
+		if !ok {
+			continue
+		}
+		if k, v, ok := strings.Cut(suf, "="); ok {
+			envknob.Setenv(k, v)
+		}
+	}
+}

cmd/tl-longchain/tl-longchain.go

@@ -0,0 +1,93 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Program tl-longchain prints commands to re-sign Tailscale nodes that have
+// long rotation signature chains.
+//
+// There is an implicit limit on the number of rotation signatures that can
+// be chained before the signature becomes too long. This program helps
+// tailnet admins to identify nodes that have signatures with long chains and
+// prints commands to re-sign those node keys with a fresh direct signature.
+// Commands are printed to stdout, while log messages are printed to stderr.
+//
+// Note that the Tailscale client this command is executed on must have
+// ACL visibility to all other nodes to be able to see their signatures.
+// https://tailscale.com/kb/1087/device-visibility
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tka"
+	"tailscale.com/types/key"
+)
+
+var (
+	flagSocket   = flag.String("socket", "", "custom path to tailscaled socket")
+	maxRotations = flag.Int("rotations", 10, "number of rotation signatures before re-signing (max 16)")
+	showFiltered = flag.Bool("show-filtered", false, "include nodes with invalid signatures")
+)
+
+func main() {
+	flag.Parse()
+
+	lc := tailscale.LocalClient{Socket: *flagSocket}
+	if lc.Socket != "" {
+		lc.UseSocketOnly = true
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	st, err := lc.NetworkLockStatus(ctx)
+	if err != nil {
+		log.Fatalf("could not get Tailnet Lock status: %v", err)
+	}
+	if !st.Enabled {
+		log.Print("Tailnet Lock is not enabled")
+		return
+	}
+	print("Self", *st.NodeKey, *st.NodeKeySignature)
+	if len(st.VisiblePeers) > 0 {
+		log.Print("Visible peers with valid signatures:")
+		for _, peer := range st.VisiblePeers {
+			print(peerInfo(peer), peer.NodeKey, peer.NodeKeySignature)
+		}
+	}
+	if *showFiltered && len(st.FilteredPeers) > 0 {
+		log.Print("Visible peers with invalid signatures:")
+		for _, peer := range st.FilteredPeers {
+			print(peerInfo(peer), peer.NodeKey, peer.NodeKeySignature)
+		}
+	}
+}
+
+// peerInfo returns a string with information about a peer.
+func peerInfo(peer *ipnstate.TKAPeer) string {
+	return fmt.Sprintf("Peer %s (%s) nodeid=%s, current signature kind=%v", peer.Name, peer.TailscaleIPs[0], peer.StableID, peer.NodeKeySignature.SigKind)
+}
+
+// print prints a message about a node key signature and a re-signing command if needed.
+func print(info string, nodeKey key.NodePublic, sig tka.NodeKeySignature) {
+	if l := chainLength(sig); l > *maxRotations {
+		log.Printf("%s: chain length %d, printing command to re-sign", info, l)
+		wrapping, _ := sig.UnverifiedWrappingPublic()
+		fmt.Printf("tailscale lock sign %s %s\n", nodeKey, key.NLPublicFromEd25519Unsafe(wrapping).CLIString())
+	} else {
+		log.Printf("%s: does not need re-signing", info)
+	}
+}
+
+// chainLength returns the length of the rotation signature chain.
+func chainLength(sig tka.NodeKeySignature) int {
+	if sig.SigKind != tka.SigRotation {
+		return 1
+	}
+	return 1 + chainLength(*sig.Nested)
+}

cmd/tsidp/tsidp.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/rsa"
@@ -16,6 +17,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -25,6 +27,7 @@ import (
 	"net/netip"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +38,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -44,13 +48,22 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 	"tailscale.com/util/rands"
+	"tailscale.com/version"
 )
 
+// ctxConn is a key to look up a net.Conn stored in an HTTP request's context.
+type ctxConn struct{}
+
+// funnelClientsFile is the file where client IDs and secrets for OIDC clients
+// accessing the IDP over Funnel are persisted.
+const funnelClientsFile = "oidc-funnel-clients.json"
+
 var (
 	flagVerbose            = flag.Bool("verbose", false, "be verbose")
 	flagPort               = flag.Int("port", 443, "port to listen on")
 	flagLocalPort          = flag.Int("local-port", -1, "allow requests from localhost")
 	flagUseLocalTailscaled = flag.Bool("use-local-tailscaled", false, "use local tailscaled instead of tsnet")
+	flagFunnel             = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet")
 )
 
 func main() {
@@ -61,9 +74,11 @@ func main() {
 	}
 
 	var (
-		lc  *tailscale.LocalClient
-		st  *ipnstate.Status
-		err error
+		lc          *tailscale.LocalClient
+		st          *ipnstate.Status
+		err         error
+		watcherChan chan error
+		cleanup     func()
 
 		lns []net.Listener
 	)
@@ -90,6 +105,18 @@ func main() {
 		if !anySuccess {
 			log.Fatalf("failed to listen on any of %v", st.TailscaleIPs)
 		}
+
+		// tailscaled needs to be setting an HTTP header for funneled requests
+		// that older versions don't provide.
+		// TODO(naman): is this the correct check?
+		if *flagFunnel && !version.AtLeast(st.Version, "1.71.0") {
+			log.Fatalf("Local tailscaled not new enough to support -funnel. Update Tailscale or use tsnet mode.")
+		}
+		cleanup, watcherChan, err = serveOnLocalTailscaled(ctx, lc, st, uint16(*flagPort), *flagFunnel)
+		if err != nil {
+			log.Fatalf("could not serve on local tailscaled: %v", err)
+		}
+		defer cleanup()
 	} else {
 		ts := &tsnet.Server{
 			Hostname: "idp",
@@ -105,7 +132,15 @@ func main() {
 		if err != nil {
 			log.Fatalf("getting local client: %v", err)
 		}
-		ln, err := ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		var ln net.Listener
+		if *flagFunnel {
+			if err := ipn.CheckFunnelAccess(uint16(*flagPort), st.Self); err != nil {
+				log.Fatalf("%v", err)
+			}
+			ln, err = ts.ListenFunnel("tcp", fmt.Sprintf(":%d", *flagPort))
+		} else {
+			ln, err = ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		}
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -113,13 +148,26 @@ func main() {
 	}
 
 	srv := &idpServer{
-		lc: lc,
+		lc:          lc,
+		funnel:      *flagFunnel,
+		localTSMode: *flagUseLocalTailscaled,
 	}
 	if *flagPort != 443 {
 		srv.serverURL = fmt.Sprintf("https://%s:%d", strings.TrimSuffix(st.Self.DNSName, "."), *flagPort)
 	} else {
 		srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, "."))
 	}
+	if *flagFunnel {
+		f, err := os.Open(funnelClientsFile)
+		if err == nil {
+			srv.funnelClients = make(map[string]*funnelClient)
+			if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil {
+				log.Fatalf("could not parse %s: %v", funnelClientsFile, err)
+			}
+		} else if !errors.Is(err, os.ErrNotExist) {
+			log.Fatalf("could not open %s: %v", funnelClientsFile, err)
+		}
+	}
 
 	log.Printf("Running tsidp at %s ...", srv.serverURL)
 
@@ -134,35 +182,129 @@ func main() {
 	}
 
 	for _, ln := range lns {
-		go http.Serve(ln, srv)
+		server := http.Server{
+			Handler: srv,
+			ConnContext: func(ctx context.Context, c net.Conn) context.Context {
+				return context.WithValue(ctx, ctxConn{}, c)
+			},
+		}
+		go server.Serve(ln)
 	}
-	select {}
+	// need to catch os.Interrupt, otherwise deferred cleanup code doesn't run
+	exitChan := make(chan os.Signal, 1)
+	signal.Notify(exitChan, os.Interrupt)
+	select {
+	case <-exitChan:
+		log.Printf("interrupt, exiting")
+		return
+	case <-watcherChan:
+		if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
+			log.Printf("watcher closed, exiting")
+			return
+		}
+		log.Fatalf("watcher error: %v", err)
+		return
+	}
+}
+
+// serveOnLocalTailscaled starts a serve session using an already-running
+// tailscaled instead of starting a fresh tsnet server, making something
+// listening on clientDNSName:dstPort accessible over serve/funnel.
+func serveOnLocalTailscaled(ctx context.Context, lc *tailscale.LocalClient, st *ipnstate.Status, dstPort uint16, shouldFunnel bool) (cleanup func(), watcherChan chan error, err error) {
+	// In order to support funneling out in local tailscaled mode, we need
+	// to add a serve config to forward the listeners we bound above and
+	// allow those forwarders to be funneled out.
+	sc, err := lc.GetServeConfig(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get serve config: %v", err)
+	}
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+
+	// We watch the IPN bus just to get a session ID. The session expires
+	// when we stop watching the bus, and that auto-deletes the foreground
+	// serve/funnel configs we are creating below.
+	watcher, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not set up ipn bus watcher: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+	n, err := watcher.Next()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get initial state from ipn bus watcher: %v", err)
+	}
+	if n.SessionID == "" {
+		err = fmt.Errorf("missing sessionID in ipn.Notify")
+		return nil, nil, err
+	}
+	watcherChan = make(chan error)
+	go func() {
+		for {
+			_, err = watcher.Next()
+			if err != nil {
+				watcherChan <- err
+				return
+			}
+		}
+	}()
+
+	// Create a foreground serve config that gets cleaned up when tsidp
+	// exits and the session ID associated with this config is invalidated.
+	foregroundSc := new(ipn.ServeConfig)
+	mak.Set(&sc.Foreground, n.SessionID, foregroundSc)
+	serverURL := strings.TrimSuffix(st.Self.DNSName, ".")
+	fmt.Printf("setting funnel for %s:%v\n", serverURL, dstPort)
+
+	foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel)
+	foregroundSc.SetWebHandler(&ipn.HTTPHandler{
+		Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))),
+	}, serverURL, uint16(*flagPort), "/", true)
+	err = lc.SetServeConfig(ctx, sc)
+	if err != nil {
+		return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err)
+	}
+
+	return func() { watcher.Close() }, watcherChan, nil
 }
 
 type idpServer struct {
 	lc          *tailscale.LocalClient
 	loopbackURL string
 	serverURL   string // "https://foo.bar.ts.net"
+	funnel      bool
+	localTSMode bool
 
 	lazyMux        lazy.SyncValue[*http.ServeMux]
 	lazySigningKey lazy.SyncValue[*signingKey]
 	lazySigner     lazy.SyncValue[jose.Signer]
 
-	mu          sync.Mutex              // guards the fields below
-	code        map[string]*authRequest // keyed by random hex
-	accessToken map[string]*authRequest // keyed by random hex
+	mu            sync.Mutex               // guards the fields below
+	code          map[string]*authRequest  // keyed by random hex
+	accessToken   map[string]*authRequest  // keyed by random hex
+	funnelClients map[string]*funnelClient // keyed by client ID
 }
 
 type authRequest struct {
 	// localRP is true if the request is from a relying party running on the
-	// same machine as the idp server. It is mutually exclusive with rpNodeID.
+	// same machine as the idp server. It is mutually exclusive with rpNodeID
+	// and funnelRP.
 	localRP bool
 
 	// rpNodeID is the NodeID of the relying party (who requested the auth, such
 	// as Proxmox or Synology), not the user node who is being authenticated. It
-	// is mutually exclusive with localRP.
+	// is mutually exclusive with localRP and funnelRP.
 	rpNodeID tailcfg.NodeID
 
+	// funnelRP is non-nil if the request is from a relying party outside the
+	// tailnet, via Tailscale Funnel. It is mutually exclusive with rpNodeID
+	// and localRP.
+	funnelRP *funnelClient
+
 	// clientID is the "client_id" sent in the authorized request.
 	clientID string
 
@@ -181,9 +323,12 @@ type authRequest struct {
 	validTill time.Time
 }
 
-func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string, lc *tailscale.LocalClient) error {
+// allowRelyingParty validates that a relying party identified either by a
+// known remoteAddr or a valid client ID/secret pair is allowed to proceed
+// with the authorization flow associated with this authRequest.
+func (ar *authRequest) allowRelyingParty(r *http.Request, lc *tailscale.LocalClient) error {
 	if ar.localRP {
-		ra, err := netip.ParseAddrPort(remoteAddr)
+		ra, err := netip.ParseAddrPort(r.RemoteAddr)
 		if err != nil {
 			return err
 		}
@@ -192,7 +337,18 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 		}
 		return nil
 	}
-	who, err := lc.WhoIs(ctx, remoteAddr)
+	if ar.funnelRP != nil {
+		clientID, clientSecret, ok := r.BasicAuth()
+		if !ok {
+			clientID = r.FormValue("client_id")
+			clientSecret = r.FormValue("client_secret")
+		}
+		if ar.funnelRP.ID != clientID || ar.funnelRP.Secret != clientSecret {
+			return fmt.Errorf("tsidp: invalid client credentials")
+		}
+		return nil
+	}
+	who, err := lc.WhoIs(r.Context(), r.RemoteAddr)
 	if err != nil {
 		return fmt.Errorf("tsidp: error getting WhoIs: %w", err)
 	}
@@ -203,24 +359,60 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 }
 
 func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
-	who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr)
+	// This URL is visited by the user who is being authenticated. If they are
+	// visiting the URL over Funnel, that means they are not part of the
+	// tailnet that they are trying to be authenticated for.
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	uq := r.URL.Query()
+
+	redirectURI := uq.Get("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must specify redirect_uri", http.StatusBadRequest)
+		return
+	}
+
+	var remoteAddr string
+	if s.localTSMode {
+		// in local tailscaled mode, the local tailscaled is forwarding us
+		// HTTP requests, so reading r.RemoteAddr will just get us our own
+		// address.
+		remoteAddr = r.Header.Get("X-Forwarded-For")
+	} else {
+		remoteAddr = r.RemoteAddr
+	}
+	who, err := s.lc.WhoIs(r.Context(), remoteAddr)
 	if err != nil {
 		log.Printf("Error getting WhoIs: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	uq := r.URL.Query()
-
 	code := rands.HexString(32)
 	ar := &authRequest{
 		nonce:       uq.Get("nonce"),
 		remoteUser:  who,
-		redirectURI: uq.Get("redirect_uri"),
+		redirectURI: redirectURI,
 		clientID:    uq.Get("client_id"),
 	}
 
-	if r.URL.Path == "/authorize/localhost" {
+	if r.URL.Path == "/authorize/funnel" {
+		s.mu.Lock()
+		c, ok := s.funnelClients[ar.clientID]
+		s.mu.Unlock()
+		if !ok {
+			http.Error(w, "tsidp: invalid client ID", http.StatusBadRequest)
+			return
+		}
+		if ar.redirectURI != c.RedirectURI {
+			http.Error(w, "tsidp: redirect_uri mismatch", http.StatusBadRequest)
+			return
+		}
+		ar.funnelRP = c
+	} else if r.URL.Path == "/authorize/localhost" {
 		ar.localRP = true
 	} else {
 		var ok bool
@@ -237,8 +429,10 @@ func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
 
 	q := make(url.Values)
 	q.Set("code", code)
-	q.Set("state", uq.Get("state"))
-	u := uq.Get("redirect_uri") + "?" + q.Encode()
+	if state := uq.Get("state"); state != "" {
+		q.Set("state", state)
+	}
+	u := redirectURI + "?" + q.Encode()
 	log.Printf("Redirecting to %q", u)
 
 	http.Redirect(w, r, u, http.StatusFound)
@@ -251,6 +445,7 @@ func (s *idpServer) newMux() *http.ServeMux {
 	mux.HandleFunc("/authorize/", s.authorize)
 	mux.HandleFunc("/userinfo", s.serveUserInfo)
 	mux.HandleFunc("/token", s.serveToken)
+	mux.HandleFunc("/clients/", s.serveClients)
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
 			io.WriteString(w, "<html><body><h1>Tailscale OIDC IdP</h1>")
@@ -284,11 +479,6 @@ func (s *idpServer) serveUserInfo(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: invalid token", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
-		log.Printf("Error allowing relying party: %v", err)
-		http.Error(w, err.Error(), http.StatusForbidden)
-		return
-	}
 
 	if ar.validTill.Before(time.Now()) {
 		http.Error(w, "tsidp: token expired", http.StatusBadRequest)
@@ -348,7 +538,7 @@ func (s *idpServer) serveToken(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: code not found", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
+	if err := ar.allowRelyingParty(r, s.lc); err != nil {
 		log.Printf("Error allowing relying party: %v", err)
 		http.Error(w, err.Error(), http.StatusForbidden)
 		return
@@ -581,7 +771,9 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 	var authorizeEndpoint string
 	rpEndpoint := s.serverURL
-	if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
+	if isFunnelRequest(r) {
+		authorizeEndpoint = fmt.Sprintf("%s/authorize/funnel", s.serverURL)
+	} else if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
 		authorizeEndpoint = fmt.Sprintf("%s/authorize/%d", s.serverURL, who.Node.ID)
 	} else if ap.Addr().IsLoopback() {
 		rpEndpoint = s.loopbackURL
@@ -611,6 +803,148 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// funnelClient represents an OIDC client/relying party that is accessing the
+// IDP over Funnel.
+type funnelClient struct {
+	ID          string `json:"client_id"`
+	Secret      string `json:"client_secret,omitempty"`
+	Name        string `json:"name,omitempty"`
+	RedirectURI string `json:"redirect_uri"`
+}
+
+// /clients is a privileged endpoint that allows the visitor to create new
+// Funnel-capable OIDC clients, so it is only accessible over the tailnet.
+func (s *idpServer) serveClients(w http.ResponseWriter, r *http.Request) {
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	path := strings.TrimPrefix(r.URL.Path, "/clients/")
+
+	if path == "new" {
+		s.serveNewClient(w, r)
+		return
+	}
+
+	if path == "" {
+		s.serveGetClientsList(w, r)
+		return
+	}
+
+	s.mu.Lock()
+	c, ok := s.funnelClients[path]
+	s.mu.Unlock()
+	if !ok {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	switch r.Method {
+	case "DELETE":
+		s.serveDeleteClient(w, r, path)
+	case "GET":
+		json.NewEncoder(w).Encode(&funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	default:
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *idpServer) serveNewClient(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	redirectURI := r.FormValue("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must provide redirect_uri", http.StatusBadRequest)
+		return
+	}
+	clientID := rands.HexString(32)
+	clientSecret := rands.HexString(64)
+	newClient := funnelClient{
+		ID:          clientID,
+		Secret:      clientSecret,
+		Name:        r.FormValue("name"),
+		RedirectURI: redirectURI,
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.funnelClients, clientID, &newClient)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// delete the new client to avoid inconsistent state between memory
+		// and disk
+		delete(s.funnelClients, clientID)
+		return
+	}
+	json.NewEncoder(w).Encode(newClient)
+}
+
+func (s *idpServer) serveGetClientsList(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	redactedClients := make([]funnelClient, 0, len(s.funnelClients))
+	for _, c := range s.funnelClients {
+		redactedClients = append(redactedClients, funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	}
+	s.mu.Unlock()
+	json.NewEncoder(w).Encode(redactedClients)
+}
+
+func (s *idpServer) serveDeleteClient(w http.ResponseWriter, r *http.Request, clientID string) {
+	if r.Method != "DELETE" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.funnelClients == nil {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	if _, ok := s.funnelClients[clientID]; !ok {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	deleted := s.funnelClients[clientID]
+	delete(s.funnelClients, clientID)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// restore the deleted value to avoid inconsistent state between memory
+		// and disk
+		s.funnelClients[clientID] = deleted
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// storeFunnelClientsLocked writes the current mapping of OIDC client ID/secret
+// pairs for RPs that access the IDP over funnel. s.mu must be held while
+// calling this.
+func (s *idpServer) storeFunnelClientsLocked() error {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(s.funnelClients); err != nil {
+		return err
+	}
+	return os.WriteFile(funnelClientsFile, buf.Bytes(), 0600)
+}
+
 const (
 	minimumRSAKeySize = 2048
 )
@@ -700,3 +1034,24 @@ func parseID[T ~int64](input string) (_ T, ok bool) {
 	}
 	return T(i), true
 }
+
+// isFunnelRequest checks if an HTTP request is coming over Tailscale Funnel.
+func isFunnelRequest(r *http.Request) bool {
+	// If we're funneling through the local tailscaled, it will set this HTTP
+	// header.
+	if r.Header.Get("Tailscale-Funnel-Request") != "" {
+		return true
+	}
+
+	// If the funneled connection is from tsnet, then the net.Conn will be of
+	// type ipn.FunnelConn.
+	netConn := r.Context().Value(ctxConn{})
+	// if the conn is wrapped inside TLS, unwrap it
+	if tlsConn, ok := netConn.(*tls.Conn); ok {
+		netConn = tlsConn.NetConn()
+	}
+	if _, ok := netConn.(*ipn.FunnelConn); ok {
+		return true
+	}
+	return false
+}

cmd/tta/fw_linux.go

@@ -0,0 +1,128 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"encoding/binary"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/expr"
+	"tailscale.com/types/ptr"
+)
+
+func init() {
+	addFirewall = addFirewallLinux
+}
+
+func addFirewallLinux() error {
+	c, err := nftables.New()
+	if err != nil {
+		return err
+	}
+
+	// Create a new table
+	table := &nftables.Table{
+		Family: nftables.TableFamilyIPv4, // TableFamilyINet doesn't work (why?. oh well.)
+		Name:   "filter",
+	}
+	c.AddTable(table)
+
+	// Create a new chain for incoming traffic
+	inputChain := &nftables.Chain{
+		Name:     "input",
+		Table:    table,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookInput,
+		Priority: nftables.ChainPriorityFilter,
+		Policy:   ptr.To(nftables.ChainPolicyDrop),
+	}
+	c.AddChain(inputChain)
+
+	// Allow traffic from the loopback interface
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte("lo"),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	// Accept established and related connections
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Ct{
+				Register: 1,
+				Key:      expr.CtKeySTATE,
+			},
+			&expr.Bitwise{
+				SourceRegister: 1,
+				DestRegister:   1,
+				Len:            4,
+				Mask:           binary.NativeEndian.AppendUint32(nil, 0x06), // CT_STATE_BIT_ESTABLISHED | CT_STATE_BIT_RELATED
+				Xor:            binary.NativeEndian.AppendUint32(nil, 0),
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     binary.NativeEndian.AppendUint32(nil, 0x00),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	// Allow TCP packets in that don't have the SYN bit set, even if they're not
+	// ESTABLISHED or RELATED. This is because the test suite gets TCP
+	// connections up & idle (for HTTP) before it conditionally installs these
+	// firewall rules. But because conntrack wasn't previously active, existing
+	// TCP flows aren't ESTABLISHED and get dropped. So this rule allows
+	// previously established TCP connections that predates the firewall rules
+	// to continue working, as they don't have conntrack state.
+	c.AddRule(&nftables.Rule{
+		Table: table,
+		Chain: inputChain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     []byte{0x06}, // TCP
+			},
+			&expr.Payload{ // get TCP flags
+				DestRegister: 1,
+				Base:         2,
+				Offset:       13, // flags
+				Len:          1,
+			},
+			&expr.Bitwise{
+				SourceRegister: 1,
+				DestRegister:   1,
+				Len:            1,
+				Mask:           []byte{2}, // TCP_SYN
+				Xor:            []byte{0},
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpNeq,
+				Register: 1,
+				Data:     []byte{2}, // TCP_SYN
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+	})
+
+	return c.Flush()
+}

cmd/tta/tta.go

@@ -0,0 +1,349 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The tta server is the Tailscale Test Agent.
+//
+// It runs on each Tailscale node being integration tested and permits the test
+// harness to control the node. It connects out to the test drver (rather than
+// accepting any TCP connections inbound, which might be blocked depending on
+// the scenario being tested) and then the test driver turns the TCP connection
+// around and sends request back.
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/atomicfile"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/hostinfo"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/must"
+	"tailscale.com/util/set"
+	"tailscale.com/version/distro"
+)
+
+var (
+	driverAddr = flag.String("driver", "test-driver.tailscale:8008", "address of the test driver; by default we use the DNS name test-driver.tailscale which is special cased in the emulated network's DNS server")
+)
+
+func absify(cmd string) string {
+	if distro.Get() == distro.Gokrazy && !strings.Contains(cmd, "/") {
+		return "/user/" + cmd
+	}
+	return cmd
+}
+
+func serveCmd(w http.ResponseWriter, cmd string, args ...string) {
+	log.Printf("Got serveCmd for %q %v", cmd, args)
+	out, err := exec.Command(absify(cmd), args...).CombinedOutput()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if err != nil {
+		w.Header().Set("Exec-Err", err.Error())
+		w.WriteHeader(500)
+		log.Printf("Err on serveCmd for %q %v, %d bytes of output: %v", cmd, args, len(out), err)
+	} else {
+		log.Printf("Did serveCmd for %q %v, %d bytes of output", cmd, args, len(out))
+	}
+	w.Write(out)
+}
+
+type localClientRoundTripper struct {
+	lc tailscale.LocalClient
+}
+
+func (rt *localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.RequestURI = ""
+	return rt.lc.DoLocalRequest(req)
+}
+
+func main() {
+	var logBuf logBuffer
+	log.SetOutput(io.MultiWriter(os.Stderr, &logBuf))
+
+	if distro.Get() == distro.Gokrazy {
+		if !hostinfo.IsNATLabGuestVM() {
+			// "Exiting immediately with status code 0 when the
+			// GOKRAZY_FIRST_START=1 environment variable is set means “don’t
+			// start the program on boot”"
+			return
+		}
+	}
+	flag.Parse()
+
+	debug := false
+	if distro.Get() == distro.Gokrazy {
+		cmdLine, _ := os.ReadFile("/proc/cmdline")
+		explicitNS := false
+		for _, s := range strings.Fields(string(cmdLine)) {
+			if ns, ok := strings.CutPrefix(s, "tta.nameserver="); ok {
+				err := atomicfile.WriteFile("/tmp/resolv.conf", []byte("nameserver "+ns+"\n"), 0644)
+				log.Printf("Wrote /tmp/resolv.conf: %v", err)
+				explicitNS = true
+				continue
+			}
+			if v, ok := strings.CutPrefix(s, "tta.debug="); ok {
+				debug, _ = strconv.ParseBool(v)
+				continue
+			}
+		}
+		if !explicitNS {
+			nsRx := regexp.MustCompile(`(?m)^nameserver (.*)`)
+			for t := time.Now(); time.Since(t) < 10*time.Second; time.Sleep(10 * time.Millisecond) {
+				all, _ := os.ReadFile("/etc/resolv.conf")
+				if nsRx.Match(all) {
+					break
+				}
+			}
+		}
+	}
+
+	log.Printf("Tailscale Test Agent running.")
+
+	gokRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+	gokRP.Transport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if network != "tcp" {
+				return nil, errors.New("unexpected network")
+			}
+			if addr != "gokrazy:80" {
+				return nil, errors.New("unexpected addr")
+			}
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", "/run/gokrazy-http.sock")
+		},
+	}
+
+	var ttaMux http.ServeMux // agent mux
+	var serveMux http.ServeMux
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("X-TTA-GoKrazy") == "1" {
+			gokRP.ServeHTTP(w, r)
+			return
+		}
+		ttaMux.ServeHTTP(w, r)
+	})
+	var hs http.Server
+	hs.Handler = &serveMux
+	revSt := revDialState{
+		needConnCh: make(chan bool, 1),
+		debug:      debug,
+	}
+	hs.ConnState = revSt.connState
+	conns := make(chan net.Conn, 1)
+
+	lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock")))
+	lcRP.Transport = new(localClientRoundTripper)
+	ttaMux.HandleFunc("/localapi/", func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("Got localapi request: %v", r.URL)
+		t0 := time.Now()
+		lcRP.ServeHTTP(w, r)
+		log.Printf("Did localapi request in %v: %v", time.Since(t0).Round(time.Millisecond), r.URL)
+	})
+
+	ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "TTA\n")
+		return
+	})
+	ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale")
+	})
+	ttaMux.HandleFunc("/fw", addFirewallHandler)
+	ttaMux.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
+		logBuf.mu.Lock()
+		defer logBuf.mu.Unlock()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Write(logBuf.buf.Bytes())
+	})
+	go hs.Serve(chanListener(conns))
+
+	// For doing agent operations locally from gokrazy:
+	// (e.g. with "wget -O - localhost:8123/fw" or "wget -O - localhost:8123/logs"
+	// to get early tta logs before the port 124 connection is established)
+	go func() {
+		err := http.ListenAndServe("127.0.0.1:8123", &ttaMux)
+		if err != nil {
+			log.Fatalf("ListenAndServe: %v", err)
+		}
+	}()
+
+	revSt.runDialOutLoop(conns)
+}
+
+func connect() (net.Conn, error) {
+	var d net.Dialer
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	c, err := d.DialContext(ctx, "tcp", *driverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+type chanListener <-chan net.Conn
+
+func (cl chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl
+	if !ok {
+		return nil, errors.New("closed")
+	}
+	return c, nil
+}
+
+func (cl chanListener) Close() error {
+	return nil
+}
+
+func (cl chanListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("52.0.0.34"), // TS..DR(iver)
+		Port: 123,
+	}
+}
+
+type revDialState struct {
+	needConnCh chan bool
+	debug      bool
+
+	mu     sync.Mutex
+	newSet set.Set[net.Conn] // conns in StateNew
+	onNew  map[net.Conn]func()
+}
+
+func (s *revDialState) connState(c net.Conn, cs http.ConnState) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	oldLen := len(s.newSet)
+	switch cs {
+	case http.StateNew:
+		if f, ok := s.onNew[c]; ok {
+			f()
+			delete(s.onNew, c)
+		}
+		s.newSet.Make()
+		s.newSet.Add(c)
+	default:
+		s.newSet.Delete(c)
+	}
+	s.vlogf("ConnState: %p now %v; newSet %v=>%v", c, s, oldLen, len(s.newSet))
+	if len(s.newSet) < 2 {
+		select {
+		case s.needConnCh <- true:
+		default:
+		}
+	}
+}
+
+func (s *revDialState) waitNeedConnect() {
+	for {
+		s.mu.Lock()
+		need := len(s.newSet) < 2
+		s.mu.Unlock()
+		if need {
+			return
+		}
+		<-s.needConnCh
+	}
+}
+
+func (s *revDialState) vlogf(format string, arg ...any) {
+	if !s.debug {
+		return
+	}
+	log.Printf(format, arg...)
+}
+
+func (s *revDialState) runDialOutLoop(conns chan<- net.Conn) {
+	var lastErr string
+	connected := false
+
+	for {
+		s.vlogf("[dial-driver] waiting need connect...")
+		s.waitNeedConnect()
+		s.vlogf("[dial-driver] connecting...")
+		t0 := time.Now()
+		c, err := connect()
+		if err != nil {
+			s := err.Error()
+			if s != lastErr {
+				log.Printf("[dial-driver] connect failure: %v", s)
+			}
+			lastErr = s
+			time.Sleep(time.Second)
+			continue
+		}
+		if !connected {
+			connected = true
+			log.Printf("Connected to %v", *driverAddr)
+		}
+		s.vlogf("[dial-driver] connected %v => %v after %v", c.LocalAddr(), c.RemoteAddr(), time.Since(t0))
+
+		inHTTP := make(chan struct{})
+		s.mu.Lock()
+		mak.Set(&s.onNew, c, func() { close(inHTTP) })
+		s.mu.Unlock()
+
+		s.vlogf("[dial-driver] sending...")
+		conns <- c
+		s.vlogf("[dial-driver] sent; waiting")
+		select {
+		case <-inHTTP:
+			s.vlogf("[dial-driver] conn in HTTP")
+		case <-time.After(2 * time.Second):
+			s.vlogf("[dial-driver] timeout waiting for conn to be accepted into HTTP")
+		}
+	}
+}
+
+func addFirewallHandler(w http.ResponseWriter, r *http.Request) {
+	if addFirewall == nil {
+		http.Error(w, "firewall not supported", 500)
+		return
+	}
+	err := addFirewall()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	io.WriteString(w, "OK\n")
+}
+
+var addFirewall func() error // set by fw_linux.go
+
+// logBuffer is a bytes.Buffer that is safe for concurrent use
+// intended to capture early logs from the process, even if
+// gokrazy's syslog streaming isn't working or yet working.
+// It only captures the first 1MB of logs, as that's considered
+// plenty for early debugging. At runtime, it's assumed that
+// syslog log streaming is working.
+type logBuffer struct {
+	mu  sync.Mutex
+	buf bytes.Buffer
+}
+
+func (lb *logBuffer) Write(p []byte) (n int, err error) {
+	lb.mu.Lock()
+	defer lb.mu.Unlock()
+	const maxSize = 1 << 20 // more than plenty; see type comment
+	if lb.buf.Len() > maxSize {
+		return len(p), nil
+	}
+	return lb.buf.Write(p)
+}

cmd/viewer/tests/tests.go

@@ -13,7 +13,7 @@ import (
 	"tailscale.com/types/views"
 )
 
-//go:generate go run tailscale.com/cmd/viewer --type=StructWithPtrs,StructWithoutPtrs,Map,StructWithSlices,OnlyGetClone,StructWithEmbedded,GenericIntStruct,GenericNoPtrsStruct,GenericCloneableStruct,StructWithContainers --clone-only-type=OnlyGetClone
+//go:generate go run tailscale.com/cmd/viewer --type=StructWithPtrs,StructWithoutPtrs,Map,StructWithSlices,OnlyGetClone,StructWithEmbedded,GenericIntStruct,GenericNoPtrsStruct,GenericCloneableStruct,StructWithContainers,StructWithTypeAliasFields,GenericTypeAliasStruct --clone-only-type=OnlyGetClone
 
 type StructWithoutPtrs struct {
 	Int int
@@ -152,13 +152,84 @@ func ContainerViewOf[T views.ViewCloner[T, V], V views.StructView[T]](c *Contain
 	return ContainerView[T, V]{c}
 }
 
+// MapContainer is a predefined map-like container type.
+// Unlike [Container], it has two type parameters, where the value
+// is the second parameter.
+type MapContainer[K comparable, V views.Cloner[V]] struct {
+	Items map[K]V
+}
+
+func (c *MapContainer[K, V]) Clone() *MapContainer[K, V] {
+	if c == nil {
+		return nil
+	}
+	var m map[K]V
+	if c.Items != nil {
+		m = make(map[K]V, len(c.Items))
+		for i := range m {
+			m[i] = c.Items[i].Clone()
+		}
+	}
+	return &MapContainer[K, V]{m}
+}
+
+// MapContainerView is a pre-defined readonly view of a [MapContainer][K, T].
+type MapContainerView[K comparable, T views.ViewCloner[T, V], V views.StructView[T]] struct {
+	// ж is the underlying mutable value, named with a hard-to-type
+	// character that looks pointy like a pointer.
+	// It is named distinctively to make you think of how dangerous it is to escape
+	// to callers. You must not let callers be able to mutate it.
+	ж *MapContainer[K, T]
+}
+
+func (cv MapContainerView[K, T, V]) Items() views.MapFn[K, T, V] {
+	return views.MapFnOf(cv.ж.Items, func(t T) V { return t.View() })
+}
+
+func MapContainerViewOf[K comparable, T views.ViewCloner[T, V], V views.StructView[T]](c *MapContainer[K, T]) MapContainerView[K, T, V] {
+	return MapContainerView[K, T, V]{c}
+}
+
 type GenericBasicStruct[T BasicType] struct {
 	Value T
 }
 
 type StructWithContainers struct {
-	IntContainer             Container[int]
-	CloneableContainer       Container[*StructWithPtrs]
-	BasicGenericContainer    Container[GenericBasicStruct[int]]
-	ClonableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	IntContainer              Container[int]
+	CloneableContainer        Container[*StructWithPtrs]
+	BasicGenericContainer     Container[GenericBasicStruct[int]]
+	CloneableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	CloneableMap              MapContainer[int, *StructWithPtrs]
+	CloneableGenericMap       MapContainer[int, *GenericNoPtrsStruct[int]]
+}
+
+type (
+	StructWithPtrsAlias        = StructWithPtrs
+	StructWithoutPtrsAlias     = StructWithoutPtrs
+	StructWithPtrsAliasView    = StructWithPtrsView
+	StructWithoutPtrsAliasView = StructWithoutPtrsView
+)
+
+type StructWithTypeAliasFields struct {
+	WithPtr    StructWithPtrsAlias
+	WithoutPtr StructWithoutPtrsAlias
+
+	WithPtrByPtr    *StructWithPtrsAlias
+	WithoutPtrByPtr *StructWithoutPtrsAlias
+
+	SliceWithPtrs    []*StructWithPtrsAlias
+	SliceWithoutPtrs []*StructWithoutPtrsAlias
+
+	MapWithPtrs    map[string]*StructWithPtrsAlias
+	MapWithoutPtrs map[string]*StructWithoutPtrsAlias
+
+	MapOfSlicesWithPtrs    map[string][]*StructWithPtrsAlias
+	MapOfSlicesWithoutPtrs map[string][]*StructWithoutPtrsAlias
+}
+
+type integer = constraints.Integer
+
+type GenericTypeAliasStruct[T integer, T2 views.ViewCloner[T2, V2], V2 views.StructView[T2]] struct {
+	NonCloneable T
+	Cloneable    T2
 }

cmd/viewer/tests/tests_clone.go

@@ -426,14 +426,120 @@ func (src *StructWithContainers) Clone() *StructWithContainers {
 	dst := new(StructWithContainers)
 	*dst = *src
 	dst.CloneableContainer = *src.CloneableContainer.Clone()
-	dst.ClonableGenericContainer = *src.ClonableGenericContainer.Clone()
+	dst.CloneableGenericContainer = *src.CloneableGenericContainer.Clone()
+	dst.CloneableMap = *src.CloneableMap.Clone()
+	dst.CloneableGenericMap = *src.CloneableGenericMap.Clone()
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _StructWithContainersCloneNeedsRegeneration = StructWithContainers(struct {
-	IntContainer             Container[int]
-	CloneableContainer       Container[*StructWithPtrs]
-	BasicGenericContainer    Container[GenericBasicStruct[int]]
-	ClonableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	IntContainer              Container[int]
+	CloneableContainer        Container[*StructWithPtrs]
+	BasicGenericContainer     Container[GenericBasicStruct[int]]
+	CloneableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	CloneableMap              MapContainer[int, *StructWithPtrs]
+	CloneableGenericMap       MapContainer[int, *GenericNoPtrsStruct[int]]
 }{})
+
+// Clone makes a deep copy of StructWithTypeAliasFields.
+// The result aliases no memory with the original.
+func (src *StructWithTypeAliasFields) Clone() *StructWithTypeAliasFields {
+	if src == nil {
+		return nil
+	}
+	dst := new(StructWithTypeAliasFields)
+	*dst = *src
+	dst.WithPtr = *src.WithPtr.Clone()
+	dst.WithPtrByPtr = src.WithPtrByPtr.Clone()
+	if dst.WithoutPtrByPtr != nil {
+		dst.WithoutPtrByPtr = ptr.To(*src.WithoutPtrByPtr)
+	}
+	if src.SliceWithPtrs != nil {
+		dst.SliceWithPtrs = make([]*StructWithPtrsAlias, len(src.SliceWithPtrs))
+		for i := range dst.SliceWithPtrs {
+			if src.SliceWithPtrs[i] == nil {
+				dst.SliceWithPtrs[i] = nil
+			} else {
+				dst.SliceWithPtrs[i] = src.SliceWithPtrs[i].Clone()
+			}
+		}
+	}
+	if src.SliceWithoutPtrs != nil {
+		dst.SliceWithoutPtrs = make([]*StructWithoutPtrsAlias, len(src.SliceWithoutPtrs))
+		for i := range dst.SliceWithoutPtrs {
+			if src.SliceWithoutPtrs[i] == nil {
+				dst.SliceWithoutPtrs[i] = nil
+			} else {
+				dst.SliceWithoutPtrs[i] = ptr.To(*src.SliceWithoutPtrs[i])
+			}
+		}
+	}
+	if dst.MapWithPtrs != nil {
+		dst.MapWithPtrs = map[string]*StructWithPtrsAlias{}
+		for k, v := range src.MapWithPtrs {
+			if v == nil {
+				dst.MapWithPtrs[k] = nil
+			} else {
+				dst.MapWithPtrs[k] = v.Clone()
+			}
+		}
+	}
+	if dst.MapWithoutPtrs != nil {
+		dst.MapWithoutPtrs = map[string]*StructWithoutPtrsAlias{}
+		for k, v := range src.MapWithoutPtrs {
+			if v == nil {
+				dst.MapWithoutPtrs[k] = nil
+			} else {
+				dst.MapWithoutPtrs[k] = ptr.To(*v)
+			}
+		}
+	}
+	if dst.MapOfSlicesWithPtrs != nil {
+		dst.MapOfSlicesWithPtrs = map[string][]*StructWithPtrsAlias{}
+		for k := range src.MapOfSlicesWithPtrs {
+			dst.MapOfSlicesWithPtrs[k] = append([]*StructWithPtrsAlias{}, src.MapOfSlicesWithPtrs[k]...)
+		}
+	}
+	if dst.MapOfSlicesWithoutPtrs != nil {
+		dst.MapOfSlicesWithoutPtrs = map[string][]*StructWithoutPtrsAlias{}
+		for k := range src.MapOfSlicesWithoutPtrs {
+			dst.MapOfSlicesWithoutPtrs[k] = append([]*StructWithoutPtrsAlias{}, src.MapOfSlicesWithoutPtrs[k]...)
+		}
+	}
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with the command at the top of this file.
+var _StructWithTypeAliasFieldsCloneNeedsRegeneration = StructWithTypeAliasFields(struct {
+	WithPtr                StructWithPtrsAlias
+	WithoutPtr             StructWithoutPtrsAlias
+	WithPtrByPtr           *StructWithPtrsAlias
+	WithoutPtrByPtr        *StructWithoutPtrsAlias
+	SliceWithPtrs          []*StructWithPtrsAlias
+	SliceWithoutPtrs       []*StructWithoutPtrsAlias
+	MapWithPtrs            map[string]*StructWithPtrsAlias
+	MapWithoutPtrs         map[string]*StructWithoutPtrsAlias
+	MapOfSlicesWithPtrs    map[string][]*StructWithPtrsAlias
+	MapOfSlicesWithoutPtrs map[string][]*StructWithoutPtrsAlias
+}{})
+
+// Clone makes a deep copy of GenericTypeAliasStruct.
+// The result aliases no memory with the original.
+func (src *GenericTypeAliasStruct[T, T2, V2]) Clone() *GenericTypeAliasStruct[T, T2, V2] {
+	if src == nil {
+		return nil
+	}
+	dst := new(GenericTypeAliasStruct[T, T2, V2])
+	*dst = *src
+	dst.Cloneable = src.Cloneable.Clone()
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with the command at the top of this file.
+func _GenericTypeAliasStructCloneNeedsRegeneration[T integer, T2 views.ViewCloner[T2, V2], V2 views.StructView[T2]](GenericTypeAliasStruct[T, T2, V2]) {
+	_GenericTypeAliasStructCloneNeedsRegeneration(struct {
+		NonCloneable T
+		Cloneable    T2
+	}{})
+}

cmd/viewer/tests/tests_view.go

@@ -14,7 +14,7 @@ import (
 	"tailscale.com/types/views"
 )
 
-//go:generate go run tailscale.com/cmd/cloner  -clonefunc=false -type=StructWithPtrs,StructWithoutPtrs,Map,StructWithSlices,OnlyGetClone,StructWithEmbedded,GenericIntStruct,GenericNoPtrsStruct,GenericCloneableStruct,StructWithContainers
+//go:generate go run tailscale.com/cmd/cloner  -clonefunc=false -type=StructWithPtrs,StructWithoutPtrs,Map,StructWithSlices,OnlyGetClone,StructWithEmbedded,GenericIntStruct,GenericNoPtrsStruct,GenericCloneableStruct,StructWithContainers,StructWithTypeAliasFields,GenericTypeAliasStruct
 
 // View returns a readonly view of StructWithPtrs.
 func (p *StructWithPtrs) View() StructWithPtrsView {
@@ -657,14 +657,183 @@ func (v StructWithContainersView) CloneableContainer() ContainerView[*StructWith
 func (v StructWithContainersView) BasicGenericContainer() Container[GenericBasicStruct[int]] {
 	return v.ж.BasicGenericContainer
 }
-func (v StructWithContainersView) ClonableGenericContainer() ContainerView[*GenericNoPtrsStruct[int], GenericNoPtrsStructView[int]] {
-	return ContainerViewOf(&v.ж.ClonableGenericContainer)
+func (v StructWithContainersView) CloneableGenericContainer() ContainerView[*GenericNoPtrsStruct[int], GenericNoPtrsStructView[int]] {
+	return ContainerViewOf(&v.ж.CloneableGenericContainer)
+}
+func (v StructWithContainersView) CloneableMap() MapContainerView[int, *StructWithPtrs, StructWithPtrsView] {
+	return MapContainerViewOf(&v.ж.CloneableMap)
+}
+func (v StructWithContainersView) CloneableGenericMap() MapContainerView[int, *GenericNoPtrsStruct[int], GenericNoPtrsStructView[int]] {
+	return MapContainerViewOf(&v.ж.CloneableGenericMap)
 }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _StructWithContainersViewNeedsRegeneration = StructWithContainers(struct {
-	IntContainer             Container[int]
-	CloneableContainer       Container[*StructWithPtrs]
-	BasicGenericContainer    Container[GenericBasicStruct[int]]
-	ClonableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	IntContainer              Container[int]
+	CloneableContainer        Container[*StructWithPtrs]
+	BasicGenericContainer     Container[GenericBasicStruct[int]]
+	CloneableGenericContainer Container[*GenericNoPtrsStruct[int]]
+	CloneableMap              MapContainer[int, *StructWithPtrs]
+	CloneableGenericMap       MapContainer[int, *GenericNoPtrsStruct[int]]
 }{})
+
+// View returns a readonly view of StructWithTypeAliasFields.
+func (p *StructWithTypeAliasFields) View() StructWithTypeAliasFieldsView {
+	return StructWithTypeAliasFieldsView{ж: p}
+}
+
+// StructWithTypeAliasFieldsView provides a read-only view over StructWithTypeAliasFields.
+//
+// Its methods should only be called if `Valid()` returns true.
+type StructWithTypeAliasFieldsView struct {
+	// ж is the underlying mutable value, named with a hard-to-type
+	// character that looks pointy like a pointer.
+	// It is named distinctively to make you think of how dangerous it is to escape
+	// to callers. You must not let callers be able to mutate it.
+	ж *StructWithTypeAliasFields
+}
+
+// Valid reports whether underlying value is non-nil.
+func (v StructWithTypeAliasFieldsView) Valid() bool { return v.ж != nil }
+
+// AsStruct returns a clone of the underlying value which aliases no memory with
+// the original.
+func (v StructWithTypeAliasFieldsView) AsStruct() *StructWithTypeAliasFields {
+	if v.ж == nil {
+		return nil
+	}
+	return v.ж.Clone()
+}
+
+func (v StructWithTypeAliasFieldsView) MarshalJSON() ([]byte, error) { return json.Marshal(v.ж) }
+
+func (v *StructWithTypeAliasFieldsView) UnmarshalJSON(b []byte) error {
+	if v.ж != nil {
+		return errors.New("already initialized")
+	}
+	if len(b) == 0 {
+		return nil
+	}
+	var x StructWithTypeAliasFields
+	if err := json.Unmarshal(b, &x); err != nil {
+		return err
+	}
+	v.ж = &x
+	return nil
+}
+
+func (v StructWithTypeAliasFieldsView) WithPtr() StructWithPtrsView        { return v.ж.WithPtr.View() }
+func (v StructWithTypeAliasFieldsView) WithoutPtr() StructWithoutPtrsAlias { return v.ж.WithoutPtr }
+func (v StructWithTypeAliasFieldsView) WithPtrByPtr() StructWithPtrsAliasView {
+	return v.ж.WithPtrByPtr.View()
+}
+func (v StructWithTypeAliasFieldsView) WithoutPtrByPtr() *StructWithoutPtrsAlias {
+	if v.ж.WithoutPtrByPtr == nil {
+		return nil
+	}
+	x := *v.ж.WithoutPtrByPtr
+	return &x
+}
+
+func (v StructWithTypeAliasFieldsView) SliceWithPtrs() views.SliceView[*StructWithPtrsAlias, StructWithPtrsAliasView] {
+	return views.SliceOfViews[*StructWithPtrsAlias, StructWithPtrsAliasView](v.ж.SliceWithPtrs)
+}
+func (v StructWithTypeAliasFieldsView) SliceWithoutPtrs() views.SliceView[*StructWithoutPtrsAlias, StructWithoutPtrsAliasView] {
+	return views.SliceOfViews[*StructWithoutPtrsAlias, StructWithoutPtrsAliasView](v.ж.SliceWithoutPtrs)
+}
+
+func (v StructWithTypeAliasFieldsView) MapWithPtrs() views.MapFn[string, *StructWithPtrsAlias, StructWithPtrsAliasView] {
+	return views.MapFnOf(v.ж.MapWithPtrs, func(t *StructWithPtrsAlias) StructWithPtrsAliasView {
+		return t.View()
+	})
+}
+
+func (v StructWithTypeAliasFieldsView) MapWithoutPtrs() views.MapFn[string, *StructWithoutPtrsAlias, StructWithoutPtrsAliasView] {
+	return views.MapFnOf(v.ж.MapWithoutPtrs, func(t *StructWithoutPtrsAlias) StructWithoutPtrsAliasView {
+		return t.View()
+	})
+}
+
+func (v StructWithTypeAliasFieldsView) MapOfSlicesWithPtrs() views.MapFn[string, []*StructWithPtrsAlias, views.SliceView[*StructWithPtrsAlias, StructWithPtrsAliasView]] {
+	return views.MapFnOf(v.ж.MapOfSlicesWithPtrs, func(t []*StructWithPtrsAlias) views.SliceView[*StructWithPtrsAlias, StructWithPtrsAliasView] {
+		return views.SliceOfViews[*StructWithPtrsAlias, StructWithPtrsAliasView](t)
+	})
+}
+
+func (v StructWithTypeAliasFieldsView) MapOfSlicesWithoutPtrs() views.MapFn[string, []*StructWithoutPtrsAlias, views.SliceView[*StructWithoutPtrsAlias, StructWithoutPtrsAliasView]] {
+	return views.MapFnOf(v.ж.MapOfSlicesWithoutPtrs, func(t []*StructWithoutPtrsAlias) views.SliceView[*StructWithoutPtrsAlias, StructWithoutPtrsAliasView] {
+		return views.SliceOfViews[*StructWithoutPtrsAlias, StructWithoutPtrsAliasView](t)
+	})
+}
+
+// A compilation failure here means this code must be regenerated, with the command at the top of this file.
+var _StructWithTypeAliasFieldsViewNeedsRegeneration = StructWithTypeAliasFields(struct {
+	WithPtr                StructWithPtrsAlias
+	WithoutPtr             StructWithoutPtrsAlias
+	WithPtrByPtr           *StructWithPtrsAlias
+	WithoutPtrByPtr        *StructWithoutPtrsAlias
+	SliceWithPtrs          []*StructWithPtrsAlias
+	SliceWithoutPtrs       []*StructWithoutPtrsAlias
+	MapWithPtrs            map[string]*StructWithPtrsAlias
+	MapWithoutPtrs         map[string]*StructWithoutPtrsAlias
+	MapOfSlicesWithPtrs    map[string][]*StructWithPtrsAlias
+	MapOfSlicesWithoutPtrs map[string][]*StructWithoutPtrsAlias
+}{})
+
+// View returns a readonly view of GenericTypeAliasStruct.
+func (p *GenericTypeAliasStruct[T, T2, V2]) View() GenericTypeAliasStructView[T, T2, V2] {
+	return GenericTypeAliasStructView[T, T2, V2]{ж: p}
+}
+
+// GenericTypeAliasStructView[T, T2, V2] provides a read-only view over GenericTypeAliasStruct[T, T2, V2].
+//
+// Its methods should only be called if `Valid()` returns true.
+type GenericTypeAliasStructView[T integer, T2 views.ViewCloner[T2, V2], V2 views.StructView[T2]] struct {
+	// ж is the underlying mutable value, named with a hard-to-type
+	// character that looks pointy like a pointer.
+	// It is named distinctively to make you think of how dangerous it is to escape
+	// to callers. You must not let callers be able to mutate it.
+	ж *GenericTypeAliasStruct[T, T2, V2]
+}
+
+// Valid reports whether underlying value is non-nil.
+func (v GenericTypeAliasStructView[T, T2, V2]) Valid() bool { return v.ж != nil }
+
+// AsStruct returns a clone of the underlying value which aliases no memory with
+// the original.
+func (v GenericTypeAliasStructView[T, T2, V2]) AsStruct() *GenericTypeAliasStruct[T, T2, V2] {
+	if v.ж == nil {
+		return nil
+	}
+	return v.ж.Clone()
+}
+
+func (v GenericTypeAliasStructView[T, T2, V2]) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.ж)
+}
+
+func (v *GenericTypeAliasStructView[T, T2, V2]) UnmarshalJSON(b []byte) error {
+	if v.ж != nil {
+		return errors.New("already initialized")
+	}
+	if len(b) == 0 {
+		return nil
+	}
+	var x GenericTypeAliasStruct[T, T2, V2]
+	if err := json.Unmarshal(b, &x); err != nil {
+		return err
+	}
+	v.ж = &x
+	return nil
+}
+
+func (v GenericTypeAliasStructView[T, T2, V2]) NonCloneable() T { return v.ж.NonCloneable }
+func (v GenericTypeAliasStructView[T, T2, V2]) Cloneable() V2   { return v.ж.Cloneable.View() }
+
+// A compilation failure here means this code must be regenerated, with the command at the top of this file.
+func _GenericTypeAliasStructViewNeedsRegeneration[T integer, T2 views.ViewCloner[T2, V2], V2 views.StructView[T2]](GenericTypeAliasStruct[T, T2, V2]) {
+	_GenericTypeAliasStructViewNeedsRegeneration(struct {
+		NonCloneable T
+		Cloneable    T2
+	}{})
+}

cmd/viewer/viewer.go

@@ -230,7 +230,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 				writeTemplate("sliceField")
 			}
 			continue
-		case *types.Struct, *types.Named:
+		case *types.Struct:
 			strucT := underlying
 			args.FieldType = it.QualifiedName(fieldType)
 			if codegen.ContainsPointers(strucT) {
@@ -262,7 +262,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 			mElem := m.Elem()
 			var template string
 			switch u := mElem.(type) {
-			case *types.Struct, *types.Named:
+			case *types.Struct, *types.Named, *types.Alias:
 				strucT := u
 				args.FieldType = it.QualifiedName(fieldType)
 				if codegen.ContainsPointers(strucT) {
@@ -281,7 +281,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 				slice := u
 				sElem := slice.Elem()
 				switch x := sElem.(type) {
-				case *types.Basic, *types.Named:
+				case *types.Basic, *types.Named, *types.Alias:
 					sElem := it.QualifiedName(sElem)
 					args.MapValueView = fmt.Sprintf("views.Slice[%v]", sElem)
 					args.MapValueType = sElem
@@ -292,7 +292,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 					template = "unsupportedField"
 					if _, isIface := pElem.Underlying().(*types.Interface); !isIface {
 						switch pElem.(type) {
-						case *types.Struct, *types.Named:
+						case *types.Struct, *types.Named, *types.Alias:
 							ptrType := it.QualifiedName(ptr)
 							viewType := appendNameSuffix(it.QualifiedName(pElem), "View")
 							args.MapFn = fmt.Sprintf("views.SliceOfViews[%v,%v](t)", ptrType, viewType)
@@ -313,7 +313,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 				pElem := ptr.Elem()
 				if _, isIface := pElem.Underlying().(*types.Interface); !isIface {
 					switch pElem.(type) {
-					case *types.Struct, *types.Named:
+					case *types.Struct, *types.Named, *types.Alias:
 						args.MapValueType = it.QualifiedName(ptr)
 						args.MapValueView = appendNameSuffix(it.QualifiedName(pElem), "View")
 						args.MapFn = "t.View()"
@@ -422,7 +422,7 @@ func viewTypeForValueType(typ types.Type) types.Type {
 func viewTypeForContainerType(typ types.Type) (*types.Named, *types.Func) {
 	// The container type should be an instantiated generic type,
 	// with its first type parameter specifying the element type.
-	containerType, ok := typ.(*types.Named)
+	containerType, ok := codegen.NamedTypeOf(typ)
 	if !ok || containerType.TypeArgs().Len() == 0 {
 		return nil, nil
 	}
@@ -435,7 +435,7 @@ func viewTypeForContainerType(typ types.Type) (*types.Named, *types.Func) {
 	if !ok {
 		return nil, nil
 	}
-	containerViewGenericType, ok := containerViewTypeObj.Type().(*types.Named)
+	containerViewGenericType, ok := codegen.NamedTypeOf(containerViewTypeObj.Type())
 	if !ok || containerViewGenericType.TypeParams().Len() != containerType.TypeArgs().Len()+1 {
 		return nil, nil
 	}
@@ -448,7 +448,7 @@ func viewTypeForContainerType(typ types.Type) (*types.Named, *types.Func) {
 	}
 	// ...and add the element view type.
 	// For that, we need to first determine the named elem type...
-	elemType, ok := baseType(containerType.TypeArgs().At(0)).(*types.Named)
+	elemType, ok := codegen.NamedTypeOf(baseType(containerType.TypeArgs().At(containerType.TypeArgs().Len() - 1)))
 	if !ok {
 		return nil, nil
 	}
@@ -473,7 +473,7 @@ func viewTypeForContainerType(typ types.Type) (*types.Named, *types.Func) {
 	}
 	// If elemType is an instantiated generic type, instantiate the elemViewType as well.
 	if elemTypeArgs := elemType.TypeArgs(); elemTypeArgs != nil {
-		elemViewType = must.Get(types.Instantiate(nil, elemViewType, collectTypes(elemTypeArgs), false)).(*types.Named)
+		elemViewType, _ = codegen.NamedTypeOf(must.Get(types.Instantiate(nil, elemViewType, collectTypes(elemTypeArgs), false)))
 	}
 	// And finally set the elemViewType as the last type argument.
 	containerViewTypeArgs[len(containerViewTypeArgs)-1] = elemViewType
@@ -567,7 +567,7 @@ func main() {
 		if cloneOnlyType[typeName] {
 			continue
 		}
-		typ, ok := namedTypes[typeName]
+		typ, ok := namedTypes[typeName].(*types.Named)
 		if !ok {
 			log.Fatalf("could not find type %s", typeName)
 		}

cmd/vnet/run-krazy.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+echo "Type 'C-a c' to enter monitor; q to quit."
+
+# If the USE_V6 environment is set to 1, set the nameserver explicitly to.
+EXTRA_ARG=""
+if [ "$USE_V6" = "1" ]; then
+    EXTRA_ARG="tta.nameserver=2411::411"
+fi
+
+set -eux
+qemu-system-x86_64 -M microvm,isa-serial=off \
+    -m 1G \
+    -nodefaults -no-user-config -nographic \
+    -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
+    -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1 tailscaled.env=TS_DEBUG_RAW_DISCO=1 ${EXTRA_ARG}" \
+    -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/natlabapp.img,format=raw \
+    -device virtio-blk-device,drive=blk0 \
+    -device virtio-rng-device \
+    -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \
+    -device virtio-serial-device \
+    -device virtio-net-device,netdev=net0,mac=52:cc:cc:cc:cc:01 \
+    -chardev stdio,id=virtiocon0,mux=on \
+    -device virtconsole,chardev=virtiocon0 \
+    -mon chardev=virtiocon0,mode=readline \
+    -audio none
+

cmd/vnet/vnet-main.go

@@ -0,0 +1,139 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The vnet binary runs a virtual network stack in userspace for qemu instances
+// to connect to and simulate various network conditions.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+)
+
+var (
+	listen   = flag.String("listen", "/tmp/qemu.sock", "path to listen on")
+	nat      = flag.String("nat", "easy", "type of NAT to use")
+	nat2     = flag.String("nat2", "hard", "type of NAT to use for second network")
+	portmap  = flag.Bool("portmap", false, "enable portmapping; requires --v4")
+	dgram    = flag.Bool("dgram", false, "enable datagram mode; for use with macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment")
+	blend    = flag.Bool("blend", true, "blend reality (controlplane.tailscale.com and DERPs) into the virtual network")
+	pcapFile = flag.String("pcap", "", "if non-empty, filename to write pcap")
+	v4       = flag.Bool("v4", true, "enable IPv4")
+	v6       = flag.Bool("v6", true, "enable IPv6")
+)
+
+func main() {
+	flag.Parse()
+
+	if _, err := os.Stat(*listen); err == nil {
+		os.Remove(*listen)
+	}
+
+	var srv net.Listener
+	var err error
+	var conn *net.UnixConn
+	if *dgram {
+		addr, err := net.ResolveUnixAddr("unixgram", *listen)
+		if err != nil {
+			log.Fatalf("ResolveUnixAddr: %v", err)
+		}
+		conn, err = net.ListenUnixgram("unixgram", addr)
+		if err != nil {
+			log.Fatalf("ListenUnixgram: %v", err)
+		}
+		defer conn.Close()
+	} else {
+		srv, err = net.Listen("unix", *listen)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var c vnet.Config
+	c.SetPCAPFile(*pcapFile)
+	c.SetBlendReality(*blend)
+
+	var net1opt = []any{vnet.NAT(*nat)}
+	if *v4 {
+		net1opt = append(net1opt, "2.1.1.1", "192.168.1.1/24")
+	}
+	if *v6 {
+		net1opt = append(net1opt, "2000:52::1/64")
+	}
+
+	node1 := c.AddNode(c.AddNetwork(net1opt...))
+	c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", vnet.NAT(*nat2)))
+	if *portmap && *v4 {
+		node1.Network().AddService(vnet.NATPMP)
+	}
+
+	s, err := vnet.New(&c)
+	if err != nil {
+		log.Fatalf("newServer: %v", err)
+	}
+
+	if *blend {
+		if err := s.PopulateDERPMapIPs(); err != nil {
+			log.Printf("warning: ignoring failure to populate DERP map: %v", err)
+		}
+	}
+
+	s.WriteStartingBanner(os.Stdout)
+	nc := s.NodeAgentClient(node1)
+	go func() {
+		rp := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://gokrazy")))
+		d := rp.Director
+		rp.Director = func(r *http.Request) {
+			d(r)
+			r.Header.Set("X-TTA-GoKrazy", "1")
+		}
+		rp.Transport = nc.HTTPClient.Transport
+		http.ListenAndServe(":8080", rp)
+	}()
+	go func() {
+		var last string
+		getStatus := func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			st, err := nc.Status(ctx)
+			if err != nil {
+				log.Printf("NodeStatus: %v", err)
+				return
+			}
+			if st.BackendState != last {
+				last = st.BackendState
+				log.Printf("NodeStatus: %v", logger.AsJSON(st))
+			}
+		}
+		for {
+			time.Sleep(5 * time.Second)
+			//continue
+			getStatus()
+		}
+	}()
+
+	if conn != nil {
+		s.ServeUnixConn(conn, vnet.ProtocolUnixDGRAM)
+		return
+	}
+
+	for {
+		c, err := srv.Accept()
+		if err != nil {
+			log.Printf("Accept: %v", err)
+			continue
+		}
+		go s.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+	}
+}

control/controlclient/direct.go

@@ -333,6 +333,9 @@ func (c *Direct) Close() error {
 		}
 	}
 	c.noiseClient = nil
+	if tr, ok := c.httpc.Transport.(*http.Transport); ok {
+		tr.CloseIdleConnections()
+	}
 	return nil
 }
 

control/controlclient/map.go

@@ -19,7 +19,6 @@ import (
 	"sync"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/control/controlknobs"
 	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
@@ -313,10 +312,8 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) {
 		}
 	}
 	if packetFilterChanged {
-		keys := xmaps.Keys(ms.namedPacketFilters)
-		sort.Strings(keys)
 		var concat []tailcfg.FilterRule
-		for _, v := range keys {
+		for _, v := range slices.Sorted(maps.Keys(ms.namedPacketFilters)) {
 			concat = ms.namedPacketFilters[v].AppendTo(concat)
 		}
 		ms.lastPacketFilterRules = views.SliceOf(concat)

control/controlclient/sign_supported.go

@@ -38,7 +38,7 @@ var getMachineCertificateSubjectOnce struct {
 // Example: "CN=Tailscale Inc Test Root CA,OU=Tailscale Inc Test Certificate Authority,O=Tailscale Inc,ST=ON,C=CA"
 func getMachineCertificateSubject() string {
 	getMachineCertificateSubjectOnce.Do(func() {
-		getMachineCertificateSubjectOnce.v, _ = syspolicy.GetString("MachineCertificateSubject", "")
+		getMachineCertificateSubjectOnce.v, _ = syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
 	})
 
 	return getMachineCertificateSubjectOnce.v

control/controlhttp/client.go

@@ -46,6 +46,7 @@ import (
 	"tailscale.com/net/sockstats"
 	"tailscale.com/net/tlsdial"
 	"tailscale.com/net/tshttpproxy"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
 	"tailscale.com/util/multierr"
@@ -497,11 +498,9 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr,
 	tr.DisableCompression = true
 
 	// (mis)use httptrace to extract the underlying net.Conn from the
-	// transport. We make exactly 1 request using this transport, so
-	// there will be exactly 1 GotConn call. Additionally, the
-	// transport handles 101 Switching Protocols correctly, such that
-	// the Conn will not be reused or kept alive by the transport once
-	// the response has been handed back from RoundTrip.
+	// transport. The transport handles 101 Switching Protocols correctly,
+	// such that the Conn will not be reused or kept alive by the transport
+	// once the response has been handed back from RoundTrip.
 	//
 	// In theory, the machinery of net/http should make it such that
 	// the trace callback happens-before we get the response, but
@@ -517,10 +516,16 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr,
 	// unexpected EOFs...), and we're bound to forget someday and
 	// introduce a protocol optimization at a higher level that starts
 	// eagerly transmitting from the server.
-	connCh := make(chan net.Conn, 1)
+	var lastConn syncs.AtomicValue[net.Conn]
 	trace := httptrace.ClientTrace{
+		// Even though we only make a single HTTP request which should
+		// require a single connection, the context (with the attached
+		// trace configuration) might be used by our custom dialer to
+		// make other HTTP requests (e.g. BootstrapDNS). We only care
+		// about the last connection made, which should be the one to
+		// the control server.
 		GotConn: func(info httptrace.GotConnInfo) {
-			connCh <- info.Conn
+			lastConn.Store(info.Conn)
 		},
 	}
 	ctx = httptrace.WithClientTrace(ctx, &trace)
@@ -548,11 +553,7 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr,
 	// is still a read buffer attached to it within resp.Body. So, we
 	// must direct I/O through resp.Body, but we can still use the
 	// underlying net.Conn for stuff like deadlines.
-	var switchedConn net.Conn
-	select {
-	case switchedConn = <-connCh:
-	default:
-	}
+	switchedConn := lastConn.Load()
 	if switchedConn == nil {
 		resp.Body.Close()
 		return nil, fmt.Errorf("httptrace didn't provide a connection")

control/controlhttp/client_js.go

@@ -10,7 +10,7 @@ import (
 	"net"
 	"net/url"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
 	"tailscale.com/net/wsconn"
 )

control/controlhttp/http_test.go

@@ -11,10 +11,12 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"net/http/httputil"
 	"net/netip"
 	"net/url"
 	"runtime"
+	"slices"
 	"strconv"
 	"sync"
 	"testing"
@@ -41,6 +43,8 @@ type httpTestParam struct {
 	makeHTTPHangAfterUpgrade bool
 
 	doEarlyWrite bool
+
+	httpInDial bool
 }
 
 func TestControlHTTP(t *testing.T) {
@@ -120,6 +124,12 @@ func TestControlHTTP(t *testing.T) {
 			name:         "early_write",
 			doEarlyWrite: true,
 		},
+		// Dialer needed to make another HTTP request along the way (e.g. to
+		// resolve the hostname via BootstrapDNS).
+		{
+			name:       "http_request_in_dial",
+			httpInDial: true,
+		},
 	}
 
 	for _, test := range tests {
@@ -217,6 +227,29 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 		Clock:                clock,
 	}
 
+	if param.httpInDial {
+		// Spin up a separate server to get a different port on localhost.
+		secondServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return }))
+		defer secondServer.Close()
+
+		prev := a.Dialer
+		a.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
+			ctx, cancel := context.WithTimeout(ctx, time.Second)
+			defer cancel()
+			req, err := http.NewRequestWithContext(ctx, "GET", secondServer.URL, nil)
+			if err != nil {
+				t.Errorf("http.NewRequest: %v", err)
+			}
+			r, err := http.DefaultClient.Do(req)
+			if err != nil {
+				t.Errorf("http.Get: %v", err)
+			}
+			r.Body.Close()
+
+			return prev(ctx, network, addr)
+		}
+	}
+
 	if proxy != nil {
 		proxyEnv := proxy.Start(t)
 		defer proxy.Close()
@@ -238,6 +271,7 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 		t.Fatalf("dialing controlhttp: %v", err)
 	}
 	defer conn.Close()
+
 	si := <-sch
 	if si.conn != nil {
 		defer si.conn.Close()
@@ -266,6 +300,19 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 			t.Errorf("early write = %q; want %q", buf, earlyWriteMsg)
 		}
 	}
+
+	// When no proxy is used, the RemoteAddr of the returned connection should match
+	// one of the listeners of the test server.
+	if proxy == nil {
+		var expectedAddrs []string
+		for _, ln := range []net.Listener{httpLn, httpsLn} {
+			expectedAddrs = append(expectedAddrs, fmt.Sprintf("127.0.0.1:%d", ln.Addr().(*net.TCPAddr).Port))
+			expectedAddrs = append(expectedAddrs, fmt.Sprintf("[::1]:%d", ln.Addr().(*net.TCPAddr).Port))
+		}
+		if !slices.Contains(expectedAddrs, conn.RemoteAddr().String()) {
+			t.Errorf("unexpected remote addr: %s, want %s", conn.RemoteAddr(), expectedAddrs)
+		}
+	}
 }
 
 type serverResult struct {

control/controlhttp/server.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"time"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/wsconn"