cmd/tailscale: add --json-docs flag

This prints all command and flag docs as JSON. To be used for generating
the contents of https://tailscale.com/kb/1080/cli.

Updates https://github.com/tailscale/tailscale-www/issues/4722

.github/workflows/checklocks.yml

@@ -24,5 +24,11 @@ jobs:
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks
 
       - name: Run checklocks vet
-        # TODO: remove || true once we have applied checklocks annotations everywhere.
-        run: ./tool/go vet -vettool=/tmp/checklocks ./... || true
+        # TODO(#12625): add more packages as we add annotations
+        run: |-
+          ./tool/go vet -vettool=/tmp/checklocks \
+            ./envknob           \
+            ./ipn/store/mem     \
+            ./net/stun/stuntest \
+            ./net/wsconn        \
+            ./proxymap

.github/workflows/go-licenses.yml

@@ -1,64 +0,0 @@
-name: go-licenses
-
-on:
-  # run action when a change lands in the main branch which updates go.mod or
-  # our license template file. Also allow manual triggering.
-  push:
-    branches:
-      - main
-    paths:
-      - go.mod
-      - .github/licenses.tmpl
-      - .github/workflows/go-licenses.yml
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  update-licenses:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version-file: go.mod
-
-      - name: Install go-licenses
-        run: |
-          go install github.com/google/go-licenses@v1.2.2-0.20220825154955-5eedde1c6584
-
-      - name: Run go-licenses
-        env:
-          # include all build tags to include platform-specific dependencies
-          GOFLAGS: "-tags=android,cgo,darwin,freebsd,ios,js,linux,openbsd,wasm,windows"
-        run: |
-          [ -d licenses ] || mkdir licenses
-          go-licenses report tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled > licenses/tailscale.md --template .github/licenses.tmpl
-
-      - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
-        id: generate-token
-        with:
-          app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
-          private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
-
-      - name: Send pull request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
-        with:
-          token: ${{ steps.generate-token.outputs.token }}
-          author: License Updater <noreply+license-updater@tailscale.com>
-          committer: License Updater <noreply+license-updater@tailscale.com>
-          branch: licenses/cli
-          commit-message: "licenses: update tailscale{,d} licenses"
-          title: "licenses: update tailscale{,d} licenses"
-          body: Triggered by ${{ github.repository }}@${{ github.sha }}
-          signoff: true
-          delete-branch: true
-          team-reviewers: opensource-license-reviewers

.github/workflows/installer.yml

@@ -32,7 +32,6 @@ jobs:
           - "ubuntu:18.04"
           - "ubuntu:20.04"
           - "ubuntu:22.04"
-          - "ubuntu:22.10"
           - "ubuntu:23.04"
           - "elementary/docker:stable"
           - "elementary/docker:unstable"
@@ -68,6 +67,11 @@ jobs:
       image: ${{ matrix.image }}
       options: --user root
     steps:
+    - name: install dependencies (pacman)
+      # Refresh the package databases to ensure that the tailscale package is
+      # defined.
+      run: pacman -Sy
+      if: contains(matrix.image, 'archlinux')
     - name: install dependencies (yum)
       # tar and gzip are needed by the actions/checkout below.
       run: yum install -y --allowerasing tar gzip ${{ matrix.deps }}
@@ -91,7 +95,10 @@ jobs:
         || contains(matrix.image, 'parrotsec')
         || contains(matrix.image, 'kalilinux')
     - name: checkout
-      uses: actions/checkout@v4
+      # We cannot use v4, as it requires a newer glibc version than some of the
+      # tested images provide. See
+      # https://github.com/actions/checkout/issues/1487
+      uses: actions/checkout@v3
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/ssh-integrationtest.yml

@@ -0,0 +1,23 @@
+# Run the ssh integration tests with `make sshintegrationtest`.
+# These tests can also be running locally.
+name: "ssh-integrationtest"
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - "ssh/**"
+      - "tempfork/gliderlabs/ssh/**"
+      - ".github/workflows/ssh-integrationtest"
+jobs:
+  ssh-integrationtest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Run SSH integration tests
+        run: |
+          make sshintegrationtest

.github/workflows/test.yml

@@ -194,7 +194,7 @@ jobs:
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
-      run: ./tool/go test ./util/linuxfw
+      run: ./tool/go test ./util/linuxfw ./derp/xdp
 
   vm:
     runs-on: ["self-hosted", "linux", "vm"]
@@ -254,9 +254,6 @@ jobs:
             goarch: amd64
           - goos: openbsd
             goarch: amd64
-          # Plan9 (disabled until 3p dependencies are fixed)
-          # - goos: plan9
-          #   goarch: amd64
 
     runs-on: ubuntu-22.04
     steps:
@@ -305,6 +302,47 @@ jobs:
         GOOS: ios
         GOARCH: arm64
 
+  crossmin: # cross-compile for platforms where we only check cmd/tailscale{,d}
+    strategy:
+      fail-fast: false # don't abort the entire matrix if one element fails
+      matrix:
+        include:
+          # Plan9
+          - goos: plan9
+            goarch: amd64
+          # AIX
+          - goos: aix
+            goarch: ppc64
+
+    runs-on: ubuntu-22.04
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Restore Cache
+      uses: actions/cache@v3
+      with:
+        # Note: unlike the other setups, this is only grabbing the mod download
+        # cache, rather than the whole mod directory, as the download cache
+        # contains zips that can be unpacked in parallel faster than they can be
+        # fetched and extracted by tar
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod/cache
+          ~\AppData\Local\go-build
+        # The -2- here should be incremented when the scheme of data to be
+        # cached changes (e.g. path above changes).
+        key: ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }}
+        restore-keys: |
+          ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }}
+          ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-
+    - name: build core
+      run: ./tool/go build ./cmd/tailscale ./cmd/tailscaled
+      env:
+        GOOS: ${{ matrix.goos }}
+        GOARCH: ${{ matrix.goarch }}
+        GOARM: ${{ matrix.goarm }}
+        CGO_ENABLED: "0"
+
   android:
     # similar to cross above, but android fails to build a few pieces of the
     # repo. We should fix those pieces, they're small, but as a stepping stone,
@@ -318,7 +356,7 @@ jobs:
       # some Android breakages early.
       # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482
     - name: build some
-      run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/interfaces ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version
+      run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/netmon ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version
       env:
         GOOS: android
         GOARCH: arm64
@@ -442,7 +480,7 @@ jobs:
       uses: actions/checkout@v4
     - name: check that 'go generate' is clean
       run: |
-        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator')
+        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
         ./tool/go generate $pkgs
         echo
         echo

.gitignore

@@ -9,6 +9,7 @@
 
 cmd/tailscale/tailscale
 cmd/tailscaled/tailscaled
+ssh/tailssh/testcontainers/tailscaled
 
 # Test binary, built with `go test -c`
 *.test

Dockerfile

@@ -1,17 +1,13 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
-############################################################################
+# Note that this Dockerfile is currently NOT used to build any of the published
+# Tailscale container images and may have drifted from the image build mechanism
+# we use.
+# Tailscale images are currently built using https://github.com/tailscale/mkctr,
+# and the build script can be found in ./build_docker.sh.
 #
-# WARNING: Tailscale is not yet officially supported in container
-# environments, such as Docker and Kubernetes. Though it should work, we
-# don't regularly test it, and we know there are some feature limitations.
 #
-# See current bugs tagged "containers":
-#    https://github.com/tailscale/tailscale/labels/containers
-#
-############################################################################
-
 # This Dockerfile includes all the tailscale binaries.
 #
 # To build the Dockerfile:

Makefile

@@ -1,5 +1,5 @@
 IMAGE_REPO ?= tailscale/tailscale
-SYNO_ARCH ?= "amd64"
+SYNO_ARCH ?= "x86_64"
 SYNO_DSM ?= "7"
 TAGS ?= "latest"
 
@@ -21,6 +21,7 @@ updatedeps: ## Update depaware deps
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \
+		tailscale.com/cmd/k8s-operator \
 		tailscale.com/cmd/stund
 
 depaware: ## Run depaware checks
@@ -30,6 +31,7 @@ depaware: ## Run depaware checks
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \
+		tailscale.com/cmd/k8s-operator \
 		tailscale.com/cmd/stund
 
 buildwindows: ## Build tailscale CLI for windows/amd64
@@ -100,6 +102,23 @@ publishdevoperator: ## Build and publish k8s-operator image to location specifie
 	@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
 	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=operator ./build_docker.sh
 
+publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO}
+	@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
+	@test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1)
+	@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
+	@test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1)
+	@test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1)
+	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh
+
+.PHONY: sshintegrationtest
+sshintegrationtest: ## Run the SSH integration tests in various Docker containers
+	@GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \
+	GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
+	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
+
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"
 	@grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[0;36m%-20s\033[m %s\n", $$1, $$2}'

VERSION.txt

@@ -1 +1 @@
-1.63.0
+1.71.0

appc/appconnector.go

@@ -11,20 +11,64 @@ package appc
 
 import (
 	"context"
+	"fmt"
 	"net/netip"
 	"slices"
 	"strings"
 	"sync"
+	"time"
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/execqueue"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/slicesx"
 )
 
+// rateLogger responds to calls to update by adding a count for the current period and
+// calling the callback if any previous period has finished since update was last called
+type rateLogger struct {
+	interval    time.Duration
+	start       time.Time
+	periodStart time.Time
+	periodCount int64
+	now         func() time.Time
+	callback    func(int64, time.Time, int64)
+}
+
+func (rl *rateLogger) currentIntervalStart(now time.Time) time.Time {
+	millisSince := now.Sub(rl.start).Milliseconds() % rl.interval.Milliseconds()
+	return now.Add(-(time.Duration(millisSince)) * time.Millisecond)
+}
+
+func (rl *rateLogger) update(numRoutes int64) {
+	now := rl.now()
+	periodEnd := rl.periodStart.Add(rl.interval)
+	if periodEnd.Before(now) {
+		if rl.periodCount != 0 {
+			rl.callback(rl.periodCount, rl.periodStart, numRoutes)
+		}
+		rl.periodCount = 0
+		rl.periodStart = rl.currentIntervalStart(now)
+	}
+	rl.periodCount++
+}
+
+func newRateLogger(now func() time.Time, interval time.Duration, callback func(int64, time.Time, int64)) *rateLogger {
+	nowTime := now()
+	return &rateLogger{
+		callback:    callback,
+		now:         now,
+		interval:    interval,
+		start:       nowTime,
+		periodStart: nowTime,
+	}
+}
+
 // RouteAdvertiser is an interface that allows the AppConnector to advertise
 // newly discovered routes that need to be served through the AppConnector.
 type RouteAdvertiser interface {
@@ -36,6 +80,55 @@ type RouteAdvertiser interface {
 	UnadvertiseRoute(...netip.Prefix) error
 }
 
+var (
+	metricStoreRoutesRateBuckets = []int64{1, 2, 3, 4, 5, 10, 100, 1000}
+	metricStoreRoutesNBuckets    = []int64{1, 2, 3, 4, 5, 10, 100, 1000, 10000}
+	metricStoreRoutesRate        []*clientmetric.Metric
+	metricStoreRoutesN           []*clientmetric.Metric
+)
+
+func initMetricStoreRoutes() {
+	for _, n := range metricStoreRoutesRateBuckets {
+		metricStoreRoutesRate = append(metricStoreRoutesRate, clientmetric.NewCounter(fmt.Sprintf("appc_store_routes_rate_%d", n)))
+	}
+	metricStoreRoutesRate = append(metricStoreRoutesRate, clientmetric.NewCounter("appc_store_routes_rate_over"))
+	for _, n := range metricStoreRoutesNBuckets {
+		metricStoreRoutesN = append(metricStoreRoutesN, clientmetric.NewCounter(fmt.Sprintf("appc_store_routes_n_routes_%d", n)))
+	}
+	metricStoreRoutesN = append(metricStoreRoutesN, clientmetric.NewCounter("appc_store_routes_n_routes_over"))
+}
+
+func recordMetric(val int64, buckets []int64, metrics []*clientmetric.Metric) {
+	if len(buckets) < 1 {
+		return
+	}
+	// finds the first bucket where val <=, or len(buckets) if none match
+	// for bucket values of 1, 10, 100; 0-1 goes to [0], 2-10 goes to [1], 11-100 goes to [2], 101+ goes to [3]
+	bucket, _ := slices.BinarySearch(buckets, val)
+	metrics[bucket].Add(1)
+}
+
+func metricStoreRoutes(rate, nRoutes int64) {
+	if len(metricStoreRoutesRate) == 0 {
+		initMetricStoreRoutes()
+	}
+	recordMetric(rate, metricStoreRoutesRateBuckets, metricStoreRoutesRate)
+	recordMetric(nRoutes, metricStoreRoutesNBuckets, metricStoreRoutesN)
+}
+
+// RouteInfo is a data structure used to persist the in memory state of an AppConnector
+// so that we can know, even after a restart, which routes came from ACLs and which were
+// learned from domains.
+type RouteInfo struct {
+	// Control is the routes from the 'routes' section of an app connector acl.
+	Control []netip.Prefix `json:",omitempty"`
+	// Domains are the routes discovered by observing DNS lookups for configured domains.
+	Domains map[string][]netip.Addr `json:",omitempty"`
+	// Wildcards are the configured DNS lookup domains to observe. When a DNS query matches Wildcards,
+	// its result is added to Domains.
+	Wildcards []string `json:",omitempty"`
+}
+
 // AppConnector is an implementation of an AppConnector that performs
 // its function as a subsystem inside of a tailscale node. At the control plane
 // side App Connector routing is configured in terms of domains rather than IP
@@ -49,6 +142,9 @@ type AppConnector struct {
 	logf            logger.Logf
 	routeAdvertiser RouteAdvertiser
 
+	// storeRoutesFunc will be called to persist routes if it is not nil.
+	storeRoutesFunc func(*RouteInfo) error
+
 	// mu guards the fields that follow
 	mu sync.Mutex
 
@@ -64,14 +160,68 @@ type AppConnector struct {
 
 	// queue provides ordering for update operations
 	queue execqueue.ExecQueue
+
+	writeRateMinute *rateLogger
+	writeRateDay    *rateLogger
 }
 
 // NewAppConnector creates a new AppConnector.
-func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser) *AppConnector {
-	return &AppConnector{
+func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInfo *RouteInfo, storeRoutesFunc func(*RouteInfo) error) *AppConnector {
+	ac := &AppConnector{
 		logf:            logger.WithPrefix(logf, "appc: "),
 		routeAdvertiser: routeAdvertiser,
+		storeRoutesFunc: storeRoutesFunc,
 	}
+	if routeInfo != nil {
+		ac.domains = routeInfo.Domains
+		ac.wildcards = routeInfo.Wildcards
+		ac.controlRoutes = routeInfo.Control
+	}
+	ac.writeRateMinute = newRateLogger(time.Now, time.Minute, func(c int64, s time.Time, l int64) {
+		ac.logf("routeInfo write rate: %d in minute starting at %v (%d routes)", c, s, l)
+		metricStoreRoutes(c, l)
+	})
+	ac.writeRateDay = newRateLogger(time.Now, 24*time.Hour, func(c int64, s time.Time, l int64) {
+		ac.logf("routeInfo write rate: %d in 24 hours starting at %v (%d routes)", c, s, l)
+	})
+	return ac
+}
+
+// ShouldStoreRoutes returns true if the appconnector was created with the controlknob on
+// and is storing its discovered routes persistently.
+func (e *AppConnector) ShouldStoreRoutes() bool {
+	return e.storeRoutesFunc != nil
+}
+
+// storeRoutesLocked takes the current state of the AppConnector and persists it
+func (e *AppConnector) storeRoutesLocked() error {
+	if !e.ShouldStoreRoutes() {
+		return nil
+	}
+
+	// log write rate and write size
+	numRoutes := int64(len(e.controlRoutes))
+	for _, rs := range e.domains {
+		numRoutes += int64(len(rs))
+	}
+	e.writeRateMinute.update(numRoutes)
+	e.writeRateDay.update(numRoutes)
+
+	return e.storeRoutesFunc(&RouteInfo{
+		Control:   e.controlRoutes,
+		Domains:   e.domains,
+		Wildcards: e.wildcards,
+	})
+}
+
+// ClearRoutes removes all route state from the AppConnector.
+func (e *AppConnector) ClearRoutes() error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.controlRoutes = nil
+	e.domains = nil
+	e.wildcards = nil
+	return e.storeRoutesLocked()
 }
 
 // UpdateDomainsAndRoutes starts an asynchronous update of the configuration
@@ -125,10 +275,26 @@ func (e *AppConnector) updateDomains(domains []string) {
 		for _, wc := range e.wildcards {
 			if dnsname.HasSuffix(d, wc) {
 				e.domains[d] = addrs
+				delete(oldDomains, d)
 				break
 			}
 		}
 	}
+
+	// Everything left in oldDomains is a domain we're no longer tracking
+	// and if we are storing route info we can unadvertise the routes
+	if e.ShouldStoreRoutes() {
+		toRemove := []netip.Prefix{}
+		for _, addrs := range oldDomains {
+			for _, a := range addrs {
+				toRemove = append(toRemove, netip.PrefixFrom(a, a.BitLen()))
+			}
+		}
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
+		}
+	}
+
 	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
 }
 
@@ -152,6 +318,14 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 
 	var toRemove []netip.Prefix
 
+	// If we're storing routes and know e.controlRoutes is a good
+	// representation of what should be in AdvertisedRoutes we can stop
+	// advertising routes that used to be in e.controlRoutes but are not
+	// in routes.
+	if e.ShouldStoreRoutes() {
+		toRemove = routesWithout(e.controlRoutes, routes)
+	}
+
 nextRoute:
 	for _, r := range routes {
 		for _, addr := range e.domains {
@@ -170,6 +344,9 @@ nextRoute:
 	}
 
 	e.controlRoutes = routes
+	if err := e.storeRoutesLocked(); err != nil {
+		e.logf("failed to store route info: %v", err)
+	}
 }
 
 // Domains returns the currently configured domain list.
@@ -304,8 +481,10 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			}
 		}
 
-		e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
-		e.scheduleAdvertisement(domain, toAdvertise...)
+		if len(toAdvertise) > 0 {
+			e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
+			e.scheduleAdvertisement(domain, toAdvertise...)
+		}
 	}
 }
 
@@ -380,6 +559,9 @@ func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Pref
 				e.logf("[v2] advertised route for %v: %v", domain, addr)
 			}
 		}
+		if err := e.storeRoutesLocked(); err != nil {
+			e.logf("failed to store route info: %v", err)
+		}
 	})
 }
 
@@ -400,3 +582,15 @@ func (e *AppConnector) addDomainAddrLocked(domain string, addr netip.Addr) {
 func compareAddr(l, r netip.Addr) int {
 	return l.Compare(r)
 }
+
+// routesWithout returns a without b where a and b
+// are unsorted slices of netip.Prefix
+func routesWithout(a, b []netip.Prefix) []netip.Prefix {
+	m := make(map[netip.Prefix]bool, len(b))
+	for _, p := range b {
+		m[p] = true
+	}
+	return slicesx.Filter(make([]netip.Prefix, 0, len(a)), a, func(p netip.Prefix) bool {
+		return !m[p]
+	})
+}

appc/appconnector_test.go

@@ -9,202 +9,249 @@ import (
 	"reflect"
 	"slices"
 	"testing"
+	"time"
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/tstest"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
 
+func fakeStoreRoutes(*RouteInfo) error { return nil }
+
 func TestUpdateDomains(t *testing.T) {
-	ctx := context.Background()
-	a := NewAppConnector(t.Logf, nil)
-	a.UpdateDomains([]string{"example.com"})
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, &appctest.RouteCollector{}, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, &appctest.RouteCollector{}, nil, nil)
+		}
+		a.UpdateDomains([]string{"example.com"})
 
-	a.Wait(ctx)
-	if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.Wait(ctx)
+		if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	addr := netip.MustParseAddr("192.0.0.8")
-	a.domains["example.com"] = append(a.domains["example.com"], addr)
-	a.UpdateDomains([]string{"example.com"})
-	a.Wait(ctx)
+		addr := netip.MustParseAddr("192.0.0.8")
+		a.domains["example.com"] = append(a.domains["example.com"], addr)
+		a.UpdateDomains([]string{"example.com"})
+		a.Wait(ctx)
 
-	if got, want := a.domains["example.com"], []netip.Addr{addr}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		if got, want := a.domains["example.com"], []netip.Addr{addr}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// domains are explicitly downcased on set.
-	a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
-	a.Wait(ctx)
-	if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
+		// domains are explicitly downcased on set.
+		a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
+		a.Wait(ctx)
+		if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 	}
 }
 
 func TestUpdateRoutes(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"*.example.com"})
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		a.updateDomains([]string{"*.example.com"})
 
-	// This route should be collapsed into the range
-	a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
-	a.Wait(ctx)
+		// This route should be collapsed into the range
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		a.Wait(ctx)
 
-	if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
-		t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	}
+		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
+			t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		}
 
-	// This route should not be collapsed or removed
-	a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
-	a.Wait(ctx)
+		// This route should not be collapsed or removed
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		a.Wait(ctx)
 
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
-	a.updateRoutes(routes)
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
 
-	slices.SortFunc(rc.Routes(), prefixCompare)
-	rc.SetRoutes(slices.Compact(rc.Routes()))
-	slices.SortFunc(routes, prefixCompare)
+		slices.SortFunc(rc.Routes(), prefixCompare)
+		rc.SetRoutes(slices.Compact(rc.Routes()))
+		slices.SortFunc(routes, prefixCompare)
 
-	// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
-	}
+		// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
+		}
 
-	// Ensure that the contained /32 is removed, replaced by the /24.
-	wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
-	if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
-		t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		// Ensure that the contained /32 is removed, replaced by the /24.
+		wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
-	rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
-	a.updateRoutes(routes)
+	for _, shouldStore := range []bool{false, true} {
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
+		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+		a.updateRoutes(routes)
 
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		}
 	}
 }
 
 func TestDomainRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(context.Background())
+	for _, shouldStore := range []bool{false, true} {
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(context.Background())
 
-	want := map[string][]netip.Addr{
-		"example.com": {netip.MustParseAddr("192.0.0.8")},
-	}
+		want := map[string][]netip.Addr{
+			"example.com": {netip.MustParseAddr("192.0.0.8")},
+		}
 
-	if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
-		t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
+			t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		}
 	}
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
 
-	// a has no domains configured, so it should not advertise any routes
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a has no domains configured, so it should not advertise any routes
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// matches a routed domain.
-	a.updateDomains([]string{"www.example.com", "example.com"})
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// matches a routed domain.
+		a.updateDomains([]string{"www.example.com", "example.com"})
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// even if only found in the middle of the chain
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// even if only found in the middle of the chain
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// don't re-advertise routes that have already been advertised
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
+		// don't re-advertise routes that have already been advertised
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
 
-	// don't advertise addresses that are already in a control provided route
-	pfx := netip.MustParsePrefix("192.0.2.0/24")
-	a.updateRoutes([]netip.Prefix{pfx})
-	wantRoutes = append(wantRoutes, pfx)
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
-	if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
-		t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		// don't advertise addresses that are already in a control provided route
+		pfx := netip.MustParsePrefix("192.0.2.0/24")
+		a.updateRoutes([]netip.Prefix{pfx})
+		wantRoutes = append(wantRoutes, pfx)
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
+		if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
+			t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		}
 	}
 }
 
 func TestWildcardDomains(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
 
-	a.updateDomains([]string{"*.example.com"})
-	a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
-		t.Errorf("routes: got %v; want %v", got, want)
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com"})
+		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
+			t.Errorf("routes: got %v; want %v", got, want)
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if _, ok := a.domains["foo.example.com"]; !ok {
-		t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if _, ok := a.domains["foo.example.com"]; !ok {
+			t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if len(a.wildcards) != 1 {
-		t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if len(a.wildcards) != 1 {
+			t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		}
 	}
 }
 
@@ -310,3 +357,248 @@ func prefixCompare(a, b netip.Prefix) int {
 	}
 	return a.Addr().Compare(b.Addr())
 }
+
+func prefixes(in ...string) []netip.Prefix {
+	toRet := make([]netip.Prefix, len(in))
+	for i, s := range in {
+		toRet[i] = netip.MustParsePrefix(s)
+	}
+	return toRet
+}
+
+func TestUpdateRouteRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		// nothing has yet been advertised
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{}, prefixes("1.2.3.1/32", "1.2.3.2/32"))
+		a.Wait(ctx)
+		// the routes passed to UpdateDomainsAndRoutes have been advertised
+		assertRoutes("simple update", prefixes("1.2.3.1/32", "1.2.3.2/32"), []netip.Prefix{})
+
+		// one route the same, one different
+		a.UpdateDomainsAndRoutes([]string{}, prefixes("1.2.3.1/32", "1.2.3.3/32"))
+		a.Wait(ctx)
+		// old behavior: routes are not removed, resulting routes are both old and new
+		// (we have dupe 1.2.3.1 routes because the test RouteAdvertiser doesn't have the deduplication
+		// the real one does)
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.1/32", "1.2.3.3/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior: routes are removed, resulting routes are new only
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.1/32", "1.2.3.3/32")
+			wantRemovedRoutes = prefixes("1.2.3.2/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestUpdateDomainRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com", "b.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// adding domains doesn't immediately cause any routes to be advertised
+		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
+
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
+		a.Wait(ctx)
+		// observing dns responses causes routes to be advertised
+		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// old behavior, routes are not removed
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior, routes are removed for b.example.com
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.2/32")
+			wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestUpdateWildcardRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com", "*.b.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// adding domains doesn't immediately cause any routes to be advertised
+		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
+
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
+		a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
+		a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
+		a.Wait(ctx)
+		// observing dns responses causes routes to be advertised
+		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// old behavior, routes are not removed
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior, routes are removed for *.b.example.com
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.2/32")
+			wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestRoutesWithout(t *testing.T) {
+	assert := func(msg string, got, want []netip.Prefix) {
+		if !slices.Equal(want, got) {
+			t.Errorf("%s: want %v, got %v", msg, want, got)
+		}
+	}
+
+	assert("empty routes", routesWithout([]netip.Prefix{}, []netip.Prefix{}), []netip.Prefix{})
+	assert("a empty", routesWithout([]netip.Prefix{}, prefixes("1.1.1.1/32", "1.1.1.2/32")), []netip.Prefix{})
+	assert("b empty", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), []netip.Prefix{}), prefixes("1.1.1.1/32", "1.1.1.2/32"))
+	assert("no overlap", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.3/32", "1.1.1.4/32")), prefixes("1.1.1.1/32", "1.1.1.2/32"))
+	assert("a has fewer", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32")), []netip.Prefix{})
+	assert("a has more", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32"), prefixes("1.1.1.1/32", "1.1.1.3/32")), prefixes("1.1.1.2/32", "1.1.1.4/32"))
+}
+
+func TestRateLogger(t *testing.T) {
+	clock := tstest.Clock{}
+	wasCalled := false
+	rl := newRateLogger(func() time.Time { return clock.Now() }, 1*time.Second, func(count int64, _ time.Time, _ int64) {
+		if count != 3 {
+			t.Fatalf("count for prev period: got %d, want 3", count)
+		}
+		wasCalled = true
+	})
+
+	for i := 0; i < 3; i++ {
+		clock.Advance(1 * time.Millisecond)
+		rl.update(0)
+		if wasCalled {
+			t.Fatalf("wasCalled: got true, want false")
+		}
+	}
+
+	clock.Advance(1 * time.Second)
+	rl.update(0)
+	if !wasCalled {
+		t.Fatalf("wasCalled: got false, want true")
+	}
+
+	wasCalled = false
+	rl = newRateLogger(func() time.Time { return clock.Now() }, 1*time.Hour, func(count int64, _ time.Time, _ int64) {
+		if count != 3 {
+			t.Fatalf("count for prev period: got %d, want 3", count)
+		}
+		wasCalled = true
+	})
+
+	for i := 0; i < 3; i++ {
+		clock.Advance(1 * time.Minute)
+		rl.update(0)
+		if wasCalled {
+			t.Fatalf("wasCalled: got true, want false")
+		}
+	}
+
+	clock.Advance(1 * time.Hour)
+	rl.update(0)
+	if !wasCalled {
+		t.Fatalf("wasCalled: got false, want true")
+	}
+}
+
+func TestRouteStoreMetrics(t *testing.T) {
+	metricStoreRoutes(1, 1)
+	metricStoreRoutes(1, 1)         // the 1 buckets value should be 2
+	metricStoreRoutes(5, 5)         // the 5 buckets value should be 1
+	metricStoreRoutes(6, 6)         // the 10 buckets value should be 1
+	metricStoreRoutes(10001, 10001) // the over buckets value should be 1
+	wanted := map[string]int64{
+		"appc_store_routes_n_routes_1":    2,
+		"appc_store_routes_rate_1":        2,
+		"appc_store_routes_n_routes_5":    1,
+		"appc_store_routes_rate_5":        1,
+		"appc_store_routes_n_routes_10":   1,
+		"appc_store_routes_rate_10":       1,
+		"appc_store_routes_n_routes_over": 1,
+		"appc_store_routes_rate_over":     1,
+	}
+	for _, x := range clientmetric.Metrics() {
+		if x.Value() != wanted[x.Name()] {
+			t.Errorf("%s: want: %d, got: %d", x.Name(), wanted[x.Name()], x.Value())
+		}
+	}
+}
+
+func TestMetricBucketsAreSorted(t *testing.T) {
+	if !slices.IsSorted(metricStoreRoutesRateBuckets) {
+		t.Errorf("metricStoreRoutesRateBuckets must be in order")
+	}
+	if !slices.IsSorted(metricStoreRoutesNBuckets) {
+		t.Errorf("metricStoreRoutesNBuckets must be in order")
+	}
+}

appc/appctest/appctest.go

@@ -1,6 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+// Package appctest contains code to help test App Connectors.
 package appctest
 
 import (

build_dist.sh

@@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do
 	--extra-small)
 		shift
 		ldflags="$ldflags -w -s"
-		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube"
+		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion"
 		;;
 	--box)
 		shift

build_docker.sh

@@ -1,21 +1,11 @@
 #!/usr/bin/env sh
-
 #
-# Runs `go build` with flags configured for docker distribution. All
-# it does differently from `go build` is burn git commit and version
-# information into the binaries inside docker, so that we can track down user
-# issues.
-#
-############################################################################
-#
-# WARNING: Tailscale is not yet officially supported in container
-# environments, such as Docker and Kubernetes. Though it should work, we
-# don't regularly test it, and we know there are some feature limitations.
-#
-# See current bugs tagged "containers":
-#    https://github.com/tailscale/tailscale/labels/containers
-#
-############################################################################
+# This script builds Tailscale container images using
+# github.com/tailscale/mkctr.
+# By default the images will be tagged with the current version and git
+# hash of this repository as produced by ./cmd/mkversion.
+# This is the image build mechanim used to build the official Tailscale
+# container images.
 
 set -eu
 
@@ -49,6 +39,7 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
@@ -70,6 +61,22 @@ case "$TARGET" in
       --target="${PLATFORM}" \
       /usr/local/bin/operator
     ;;
+  k8s-nameserver)
+    DEFAULT_REPOS="tailscale/k8s-nameserver"
+    REPOS="${REPOS:-${DEFAULT_REPOS}}"
+    go run github.com/tailscale/mkctr \
+      --gopaths="tailscale.com/cmd/k8s-nameserver:/usr/local/bin/k8s-nameserver" \
+      --ldflags=" \
+        -X tailscale.com/version.longStamp=${VERSION_LONG} \
+        -X tailscale.com/version.shortStamp=${VERSION_SHORT} \
+        -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
+      --base="${BASE}" \
+      --tags="${TAGS}" \
+      --repos="${REPOS}" \
+      --push="${PUSH}" \
+      --target="${PLATFORM}" \
+      /usr/local/bin/k8s-nameserver
+    ;;
   *)
     echo "unknown target: $TARGET"
     exit 1

client/tailscale/acl.go

@@ -37,6 +37,16 @@ type ACLTest struct {
 	Allow []string `json:"allow,omitempty"` // old name for accept
 }
 
+// NodeAttrGrant defines additional string attributes that apply to specific devices.
+type NodeAttrGrant struct {
+	// Target specifies which nodes the attributes apply to. The nodes can be a
+	// tag (tag:server), user (alice@example.com), group (group:kids), or *.
+	Target []string `json:"target,omitempty"`
+
+	// Attr are the attributes to set on Target(s).
+	Attr []string `json:"attr,omitempty"`
+}
+
 // ACLDetails contains all the details for an ACL.
 type ACLDetails struct {
 	Tests     []ACLTest           `json:"tests,omitempty"`
@@ -44,6 +54,7 @@ type ACLDetails struct {
 	Groups    map[string][]string `json:"groups,omitempty"`
 	TagOwners map[string][]string `json:"tagowners,omitempty"`
 	Hosts     map[string]string   `json:"hosts,omitempty"`
+	NodeAttrs []NodeAttrGrant     `json:"nodeAttrs,omitempty"`
 }
 
 // ACL contains an ACLDetails and metadata.
@@ -150,7 +161,12 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 // ACLTestFailureSummary specifies the JSON format sent to the
 // JavaScript client to be rendered in the HTML.
 type ACLTestFailureSummary struct {
-	User     string   `json:"user,omitempty"`
+	// User is the source ("src") value of the ACL test that failed.
+	// The name "user" is a legacy holdover from the original naming and
+	// is kept for compatibility but it may also contain any value
+	// that's valid in a ACL test "src" field.
+	User string `json:"user,omitempty"`
+
 	Errors   []string `json:"errors,omitempty"`
 	Warnings []string `json:"warnings,omitempty"`
 }

client/tailscale/apitype/apitype.go

@@ -49,3 +49,11 @@ type ReloadConfigResponse struct {
 	Reloaded bool   // whether the config was reloaded
 	Err      string // any error message
 }
+
+// ExitNodeSuggestionResponse is the response to a LocalAPI suggest-exit-node GET request.
+// It returns the StableNodeID, name, and location of a suggested exit node for the client making the request.
+type ExitNodeSuggestionResponse struct {
+	ID       tailcfg.StableNodeID
+	Name     string
+	Location tailcfg.LocationView `json:",omitempty"`
+}

client/tailscale/devices.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"net/url"
 
@@ -39,6 +40,7 @@ type Device struct {
 	// It's currently just 1 element, the 100.x.y.z Tailscale IP.
 	Addresses []string `json:"addresses"`
 	DeviceID  string   `json:"id"`
+	NodeID    string   `json:"nodeId"`
 	User      string   `json:"user"`
 	Name      string   `json:"name"`
 	Hostname  string   `json:"hostname"`
@@ -213,6 +215,9 @@ func (c *Client) DeleteDevice(ctx context.Context, deviceID string) (err error)
 	if err != nil {
 		return err
 	}
+
+	log.Printf("RESP: %di, path: %s", resp.StatusCode, path)
+
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {

client/tailscale/localclient.go

@@ -28,6 +28,7 @@ import (
 
 	"go4.org/mem"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/drive"
 	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -35,7 +36,6 @@ import (
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
-	"tailscale.com/tailfs"
 	"tailscale.com/tka"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
@@ -103,7 +103,7 @@ func (lc *LocalClient) defaultDialer(ctx context.Context, network, addr string)
 			return d.DialContext(ctx, "tcp", "127.0.0.1:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(lc.socket())
+	return safesocket.ConnectContext(ctx, lc.socket())
 }
 
 // DoLocalRequest makes an HTTP request to the local machine's Tailscale daemon.
@@ -253,11 +253,16 @@ func (lc *LocalClient) sendWithHeaders(
 	}
 	if res.StatusCode != wantStatus {
 		err = fmt.Errorf("%v: %s", res.Status, bytes.TrimSpace(slurp))
-		return nil, nil, bestError(err, slurp)
+		return nil, nil, httpStatusError{bestError(err, slurp), res.StatusCode}
 	}
 	return slurp, res.Header, nil
 }
 
+type httpStatusError struct {
+	error
+	HTTPStatus int
+}
+
 func (lc *LocalClient) get200(ctx context.Context, path string) ([]byte, error) {
 	return lc.send(ctx, "GET", path, 200, nil)
 }
@@ -278,9 +283,50 @@ func decodeJSON[T any](b []byte) (ret T, err error) {
 }
 
 // WhoIs returns the owner of the remoteAddr, which must be an IP or IP:port.
+//
+// If not found, the error is ErrPeerNotFound.
+//
+// For connections proxied by tailscaled, this looks up the owner of the given
+// address as TCP first, falling back to UDP; if you want to only check a
+// specific address family, use WhoIsProto.
 func (lc *LocalClient) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr))
 	if err != nil {
+		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
+			return nil, ErrPeerNotFound
+		}
+		return nil, err
+	}
+	return decodeJSON[*apitype.WhoIsResponse](body)
+}
+
+// ErrPeerNotFound is returned by WhoIs and WhoIsNodeKey when a peer is not found.
+var ErrPeerNotFound = errors.New("peer not found")
+
+// WhoIsNodeKey returns the owner of the given wireguard public key.
+//
+// If not found, the error is ErrPeerNotFound.
+func (lc *LocalClient) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(key.String()))
+	if err != nil {
+		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
+			return nil, ErrPeerNotFound
+		}
+		return nil, err
+	}
+	return decodeJSON[*apitype.WhoIsResponse](body)
+}
+
+// WhoIsProto returns the owner of the remoteAddr, which must be an IP or
+// IP:port, for the given protocol (tcp or udp).
+//
+// If not found, the error is ErrPeerNotFound.
+func (lc *LocalClient) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/whois?proto="+url.QueryEscape(proto)+"&addr="+url.QueryEscape(remoteAddr))
+	if err != nil {
+		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
+			return nil, ErrPeerNotFound
+		}
 		return nil, err
 	}
 	return decodeJSON[*apitype.WhoIsResponse](body)
@@ -699,6 +745,27 @@ func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
 	return nil
 }
 
+// SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this
+// node. This can be done to improve performance of tailnet nodes acting as exit
+// nodes or subnet routers.
+// See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
+	body, err := lc.get200(ctx, "/localapi/v0/set-udp-gro-forwarding")
+	if err != nil {
+		return err
+	}
+	var jres struct {
+		Warning string
+	}
+	if err := json.Unmarshal(body, &jres); err != nil {
+		return fmt.Errorf("invalid JSON from set-udp-gro-forwarding: %w", err)
+	}
+	if jres.Warning != "" {
+		return errors.New(jres.Warning)
+	}
+	return nil
+}
+
 // CheckPrefs validates the provided preferences, without making any changes.
 //
 // The CLI uses this before a Start call to fail fast if the preferences won't
@@ -778,6 +845,17 @@ func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
 //
 // The ctx is only used for the duration of the call, not the lifetime of the net.Conn.
 func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
+	return lc.UserDial(ctx, "tcp", host, port)
+}
+
+// UserDial connects to the host's port via Tailscale for the given network.
+//
+// The host may be a base DNS name (resolved from the netmap inside tailscaled),
+// a FQDN, or an IP address.
+//
+// The ctx is only used for the duration of the call, not the lifetime of the
+// net.Conn.
+func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
 	connCh := make(chan net.Conn, 1)
 	trace := httptrace.ClientTrace{
 		GotConn: func(info httptrace.GotConnInfo) {
@@ -790,10 +868,11 @@ func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (n
 		return nil, err
 	}
 	req.Header = http.Header{
-		"Upgrade":    []string{"ts-dial"},
-		"Connection": []string{"upgrade"},
-		"Dial-Host":  []string{host},
-		"Dial-Port":  []string{fmt.Sprint(port)},
+		"Upgrade":      []string{"ts-dial"},
+		"Connection":   []string{"upgrade"},
+		"Dial-Host":    []string{host},
+		"Dial-Port":    []string{fmt.Sprint(port)},
+		"Dial-Network": []string{network},
 	}
 	res, err := lc.DoLocalRequest(req)
 	if err != nil {
@@ -854,7 +933,20 @@ func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err e
 //
 // API maturity: this is considered a stable API.
 func (lc *LocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
-	res, err := lc.send(ctx, "GET", "/localapi/v0/cert/"+domain+"?type=pair", 200, nil)
+	return lc.CertPairWithValidity(ctx, domain, 0)
+}
+
+// CertPairWithValidity returns a cert and private key for the provided DNS
+// domain.
+//
+// It returns a cached certificate from disk if it's still valid.
+// When minValidity is non-zero, the returned certificate will be valid for at
+// least the given duration, if permitted by the CA. If the certificate is
+// valid, but for less than minValidity, it will be synchronously renewed.
+//
+// API maturity: this is considered a stable API.
+func (lc *LocalClient) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) {
+	res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1418,53 +1510,62 @@ func (lc *LocalClient) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion,
 	return &cv, nil
 }
 
-// TailFSSetFileServerAddr instructs TailFS to use the server at addr to access
+// SetUseExitNode toggles the use of an exit node on or off.
+// To turn it on, there must have been a previously used exit node.
+// The most previously used one is reused.
+// This is a convenience method for GUIs. To select an actual one, update the prefs.
+func (lc *LocalClient) SetUseExitNode(ctx context.Context, on bool) error {
+	_, err := lc.send(ctx, "POST", "/localapi/v0/set-use-exit-node-enabled?enabled="+strconv.FormatBool(on), http.StatusOK, nil)
+	return err
+}
+
+// DriveSetServerAddr instructs Taildrive to use the server at addr to access
 // the filesystem. This is used on platforms like Windows and MacOS to let
-// TailFS know to use the file server running in the GUI app.
-func (lc *LocalClient) TailFSSetFileServerAddr(ctx context.Context, addr string) error {
-	_, err := lc.send(ctx, "PUT", "/localapi/v0/tailfs/fileserver-address", http.StatusCreated, strings.NewReader(addr))
+// Taildrive know to use the file server running in the GUI app.
+func (lc *LocalClient) DriveSetServerAddr(ctx context.Context, addr string) error {
+	_, err := lc.send(ctx, "PUT", "/localapi/v0/drive/fileserver-address", http.StatusCreated, strings.NewReader(addr))
 	return err
 }
 
-// TailFSShareSet adds or updates the given share in the list of shares that
-// TailFS will serve to remote nodes. If a share with the same name already
+// DriveShareSet adds or updates the given share in the list of shares that
+// Taildrive will serve to remote nodes. If a share with the same name already
 // exists, the existing share is replaced/updated.
-func (lc *LocalClient) TailFSShareSet(ctx context.Context, share *tailfs.Share) error {
-	_, err := lc.send(ctx, "PUT", "/localapi/v0/tailfs/shares", http.StatusCreated, jsonBody(share))
+func (lc *LocalClient) DriveShareSet(ctx context.Context, share *drive.Share) error {
+	_, err := lc.send(ctx, "PUT", "/localapi/v0/drive/shares", http.StatusCreated, jsonBody(share))
 	return err
 }
 
-// TailFSShareRemove removes the share with the given name from the list of
-// shares that TailFS will serve to remote nodes.
-func (lc *LocalClient) TailFSShareRemove(ctx context.Context, name string) error {
+// DriveShareRemove removes the share with the given name from the list of
+// shares that Taildrive will serve to remote nodes.
+func (lc *LocalClient) DriveShareRemove(ctx context.Context, name string) error {
 	_, err := lc.send(
 		ctx,
 		"DELETE",
-		"/localapi/v0/tailfs/shares",
+		"/localapi/v0/drive/shares",
 		http.StatusNoContent,
 		strings.NewReader(name))
 	return err
 }
 
-// TailFSShareRename renames the share from old to new name.
-func (lc *LocalClient) TailFSShareRename(ctx context.Context, oldName, newName string) error {
+// DriveShareRename renames the share from old to new name.
+func (lc *LocalClient) DriveShareRename(ctx context.Context, oldName, newName string) error {
 	_, err := lc.send(
 		ctx,
 		"POST",
-		"/localapi/v0/tailfs/shares",
+		"/localapi/v0/drive/shares",
 		http.StatusNoContent,
 		jsonBody([2]string{oldName, newName}))
 	return err
 }
 
-// TailFSShareList returns the list of shares that TailFS is currently serving
+// DriveShareList returns the list of shares that drive is currently serving
 // to remote nodes.
-func (lc *LocalClient) TailFSShareList(ctx context.Context) ([]*tailfs.Share, error) {
-	result, err := lc.get200(ctx, "/localapi/v0/tailfs/shares")
+func (lc *LocalClient) DriveShareList(ctx context.Context) ([]*drive.Share, error) {
+	result, err := lc.get200(ctx, "/localapi/v0/drive/shares")
 	if err != nil {
 		return nil, err
 	}
-	var shares []*tailfs.Share
+	var shares []*drive.Share
 	err = json.Unmarshal(result, &shares)
 	return shares, err
 }
@@ -1505,3 +1606,12 @@ func (w *IPNBusWatcher) Next() (ipn.Notify, error) {
 	}
 	return n, nil
 }
+
+// SuggestExitNode requests an exit node suggestion and returns the exit node's details.
+func (lc *LocalClient) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/suggest-exit-node")
+	if err != nil {
+		return apitype.ExitNodeSuggestionResponse{}, err
+	}
+	return decodeJSON[apitype.ExitNodeSuggestionResponse](body)
+}

client/tailscale/localclient_test.go

@@ -6,9 +6,14 @@
 package tailscale
 
 import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"tailscale.com/tstest/deptest"
+	"tailscale.com/types/key"
 )
 
 func TestGetServeConfigFromJSON(t *testing.T) {
@@ -30,13 +35,40 @@ func TestGetServeConfigFromJSON(t *testing.T) {
 	}
 }
 
+func TestWhoIsPeerNotFound(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+	}))
+	defer ts.Close()
+
+	lc := &LocalClient{
+		Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			var std net.Dialer
+			return std.DialContext(ctx, network, ts.Listener.Addr().(*net.TCPAddr).String())
+		},
+	}
+	var k key.NodePublic
+	if err := k.UnmarshalText([]byte("nodekey:5c8f86d5fc70d924e55f02446165a5dae8f822994ad26bcf4b08fd841f9bf261")); err != nil {
+		t.Fatal(err)
+	}
+	res, err := lc.WhoIsNodeKey(context.Background(), k)
+	if err != ErrPeerNotFound {
+		t.Errorf("got (%v, %v), want ErrPeerNotFound", res, err)
+	}
+	res, err = lc.WhoIs(context.Background(), "1.2.3.4:5678")
+	if err != ErrPeerNotFound {
+		t.Errorf("got (%v, %v), want ErrPeerNotFound", res, err)
+	}
+}
+
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
 			// Make sure we don't again accidentally bring in a dependency on
-			// TailFS or its transitive dependencies
-			"tailscale.com/tailfs/tailfsimpl": "https://github.com/tailscale/tailscale/pull/10631",
-			"github.com/studio-b12/gowebdav":  "https://github.com/tailscale/tailscale/pull/10631",
+			// drive or its transitive dependencies
+			"testing":                        "do not use testing package in production code",
+			"tailscale.com/drive/driveimpl":  "https://github.com/tailscale/tailscale/pull/10631",
+			"github.com/studio-b12/gowebdav": "https://github.com/tailscale/tailscale/pull/10631",
 		},
 	}.Check(t)
 }

client/web/auth.go

@@ -223,7 +223,7 @@ func (s *Server) awaitUserAuth(ctx context.Context, session *browserSession) err
 
 func (s *Server) newSessionID() (string, error) {
 	raw := make([]byte, 16)
-	for i := 0; i < 5; i++ {
+	for range 5 {
 		if _, err := rand.Read(raw); err != nil {
 			return "", err
 		}

client/web/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.1",
   "license": "BSD-3-Clause",
   "engines": {
-    "node": "18.16.1",
+    "node": "18.20.4",
     "yarn": "1.22.19"
   },
   "type": "module",
@@ -34,7 +34,7 @@
     "prettier-plugin-organize-imports": "^3.2.2",
     "tailwindcss": "^3.3.3",
     "typescript": "^5.3.3",
-    "vite": "^5.1.4",
+    "vite": "^5.1.7",
     "vite-plugin-svgr": "^4.2.0",
     "vite-tsconfig-paths": "^3.5.0",
     "vitest": "^1.3.1"

client/web/web.go

@@ -1150,7 +1150,15 @@ func (s *Server) tailscaleUp(ctx context.Context, st *ipnstate.Status, opt tails
 		if !isRunning {
 			ipnOptions := ipn.Options{AuthKey: opt.AuthKey}
 			if opt.ControlURL != "" {
-				ipnOptions.UpdatePrefs = &ipn.Prefs{ControlURL: opt.ControlURL}
+				_, err := s.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+					Prefs: ipn.Prefs{
+						ControlURL: opt.ControlURL,
+					},
+					ControlURLSet: true,
+				})
+				if err != nil {
+					s.logf("edit prefs: %v", err)
+				}
 			}
 			if err := s.lc.Start(ctx, ipnOptions); err != nil {
 				s.logf("start: %v", err)

client/web/yarn.lock

@@ -20,7 +20,7 @@
     "@jridgewell/gen-mapping" "^0.3.0"
     "@jridgewell/trace-mapping" "^0.3.9"
 
-"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5", "@babel/code-frame@^7.23.4":
+"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.4.tgz#03ae5af150be94392cb5c7ccd97db5a19a5da6aa"
   integrity sha512-r1IONyb6Ia+jYR2vvIDhdWdlTGhqbBoFqLTQidzZ4kepUFH15ejXvFHxCVbtl7BOXIudsIubf4E81xeA3h3IXA==
@@ -63,7 +63,7 @@
     eslint-visitor-keys "^2.1.0"
     semver "^6.3.1"
 
-"@babel/generator@^7.22.10", "@babel/generator@^7.23.0", "@babel/generator@^7.23.3", "@babel/generator@^7.23.4":
+"@babel/generator@^7.23.3", "@babel/generator@^7.23.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.4.tgz#4a41377d8566ec18f807f42962a7f3551de83d1c"
   integrity sha512-esuS49Cga3HcThFNebGhlgsrVLkvhqvYDTzgjfFFlHJcIfLe5jFmRRfCQ1KuBfc4Jrtn3ndLgKWAKjBE+IraYQ==
@@ -87,7 +87,7 @@
   dependencies:
     "@babel/types" "^7.22.15"
 
-"@babel/helper-compilation-targets@^7.22.10", "@babel/helper-compilation-targets@^7.22.15", "@babel/helper-compilation-targets@^7.22.6":
+"@babel/helper-compilation-targets@^7.22.15", "@babel/helper-compilation-targets@^7.22.6":
   version "7.22.15"
   resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52"
   integrity sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==
@@ -160,14 +160,14 @@
   dependencies:
     "@babel/types" "^7.23.0"
 
-"@babel/helper-module-imports@^7.22.15", "@babel/helper-module-imports@^7.22.5":
+"@babel/helper-module-imports@^7.22.15":
   version "7.22.15"
   resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz#16146307acdc40cc00c3b2c647713076464bdbf0"
   integrity sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==
   dependencies:
     "@babel/types" "^7.22.15"
 
-"@babel/helper-module-transforms@^7.22.9", "@babel/helper-module-transforms@^7.23.3":
+"@babel/helper-module-transforms@^7.23.3":
   version "7.23.3"
   resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz#d7d12c3c5d30af5b3c0fcab2a6d5217773e2d0f1"
   integrity sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==
@@ -229,17 +229,17 @@
   dependencies:
     "@babel/types" "^7.22.5"
 
-"@babel/helper-string-parser@^7.22.5", "@babel/helper-string-parser@^7.23.4":
+"@babel/helper-string-parser@^7.23.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83"
   integrity sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==
 
-"@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.22.5":
+"@babel/helper-validator-identifier@^7.22.20":
   version "7.22.20"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0"
   integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==
 
-"@babel/helper-validator-option@^7.22.15", "@babel/helper-validator-option@^7.22.5":
+"@babel/helper-validator-option@^7.22.15":
   version "7.22.15"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040"
   integrity sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==
@@ -253,7 +253,7 @@
     "@babel/template" "^7.22.15"
     "@babel/types" "^7.22.19"
 
-"@babel/helpers@^7.22.10", "@babel/helpers@^7.23.2":
+"@babel/helpers@^7.23.2":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.4.tgz#7d2cfb969aa43222032193accd7329851facf3c1"
   integrity sha512-HfcMizYz10cr3h29VqyfGL6ZWIjTwWfvYBMsBVGwpcbhNGe3wQ1ZXZRPzZoAHhd9OqHadHqjQ89iVKINXnbzuw==
@@ -262,7 +262,7 @@
     "@babel/traverse" "^7.23.4"
     "@babel/types" "^7.23.4"
 
-"@babel/highlight@^7.22.10", "@babel/highlight@^7.22.13", "@babel/highlight@^7.23.4":
+"@babel/highlight@^7.23.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.23.4.tgz#edaadf4d8232e1a961432db785091207ead0621b"
   integrity sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==
@@ -271,7 +271,7 @@
     chalk "^2.4.2"
     js-tokens "^4.0.0"
 
-"@babel/parser@^7.22.10", "@babel/parser@^7.22.15", "@babel/parser@^7.22.5", "@babel/parser@^7.23.0", "@babel/parser@^7.23.3", "@babel/parser@^7.23.4":
+"@babel/parser@^7.22.15", "@babel/parser@^7.23.3", "@babel/parser@^7.23.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.4.tgz#409fbe690c333bb70187e2de4021e1e47a026661"
   integrity sha512-vf3Xna6UEprW+7t6EtOmFpHNAuxw3xqPZghy+brsnusscJRW5BMUzzHZc5ICjULee81WeUV2jjakG09MDglJXQ==
@@ -1093,7 +1093,7 @@
   dependencies:
     regenerator-runtime "^0.14.0"
 
-"@babel/template@^7.22.15", "@babel/template@^7.22.5":
+"@babel/template@^7.22.15":
   version "7.22.15"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38"
   integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==
@@ -1102,7 +1102,7 @@
     "@babel/parser" "^7.22.15"
     "@babel/types" "^7.22.15"
 
-"@babel/traverse@^7.22.10", "@babel/traverse@^7.23.3", "@babel/traverse@^7.23.4":
+"@babel/traverse@^7.23.3", "@babel/traverse@^7.23.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.4.tgz#c2790f7edf106d059a0098770fe70801417f3f85"
   integrity sha512-IYM8wSUwunWTB6tFC2dkKZhxbIjHoWemdK+3f8/wq8aKhbUscxD5MX72ubd90fxvFknaLPeGw5ycU84V1obHJg==
@@ -1118,7 +1118,7 @@
     debug "^4.1.0"
     globals "^11.1.0"
 
-"@babel/types@^7.21.3", "@babel/types@^7.22.10", "@babel/types@^7.22.15", "@babel/types@^7.22.19", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.3", "@babel/types@^7.23.4", "@babel/types@^7.4.4":
+"@babel/types@^7.21.3", "@babel/types@^7.22.15", "@babel/types@^7.22.19", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.3", "@babel/types@^7.23.4", "@babel/types@^7.4.4":
   version "7.23.4"
   resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.4.tgz#7206a1810fc512a7f7f7d4dace4cb4c1c9dbfb8e"
   integrity sha512-7uIFwVYpoplT5jp/kVv6EF93VaJ8H+Yn5IczYiaAi98ajzjfoZfslet/e0sLh+wVBjb2qqIut1b0S26VSafsSQ==
@@ -2474,7 +2474,7 @@ camelcase@^6.2.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
-caniuse-lite@^1.0.30001517, caniuse-lite@^1.0.30001520, caniuse-lite@^1.0.30001541:
+caniuse-lite@^1.0.30001520, caniuse-lite@^1.0.30001541:
   version "1.0.30001565"
   resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001565.tgz#a528b253c8a2d95d2b415e11d8b9942acc100c4f"
   integrity sha512-xrE//a3O7TP0vaJ8ikzkD2c2NgcVUvsEe2IvFTntV4Yd1Z9FVzh+gW+enX96L0psrbaFMcVcH2l90xNuGDWc8w==
@@ -2587,11 +2587,6 @@ confusing-browser-globals@^1.0.11:
   resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81"
   integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==
 
-convert-source-map@^1.7.0:
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f"
-  integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==
-
 convert-source-map@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a"
@@ -2772,7 +2767,7 @@ dot-case@^3.0.4:
     no-case "^3.0.4"
     tslib "^2.0.3"
 
-electron-to-chromium@^1.4.477, electron-to-chromium@^1.4.535:
+electron-to-chromium@^1.4.535:
   version "1.4.596"
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.596.tgz#6752d1aa795d942d49dfc5d3764d6ea283fab1d7"
   integrity sha512-zW3zbZ40Icb2BCWjm47nxwcFGYlIgdXkAx85XDO7cyky9J4QQfq8t0W19/TLZqq3JPQXtlv8BPIGmfa9Jb4scg==
@@ -3323,7 +3318,7 @@ gensync@^1.0.0-beta.2:
   resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
   integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==
 
-get-func-name@^2.0.0, get-func-name@^2.0.1, get-func-name@^2.0.2:
+get-func-name@^2.0.1, get-func-name@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41"
   integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==
@@ -3486,13 +3481,6 @@ has-tostringtag@^1.0.0:
   dependencies:
     has-symbols "^1.0.2"
 
-has@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
-  integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==
-  dependencies:
-    function-bind "^1.1.1"
-
 hasown@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.0.tgz#f4c513d454a57b7c7e1650778de226b11700546c"
@@ -4087,7 +4075,7 @@ mz@^2.7.0:
     object-assign "^4.0.1"
     thenify-all "^1.0.0"
 
-nanoid@^3.3.6, nanoid@^3.3.7:
+nanoid@^3.3.7:
   version "3.3.7"
   resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8"
   integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==
@@ -5121,7 +5109,7 @@ typescript@^5.3.3:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37"
   integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==
 
-ufo@^1.1.2, ufo@^1.3.2:
+ufo@^1.3.2:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.4.0.tgz#39845b31be81b4f319ab1d99fd20c56cac528d32"
   integrity sha512-Hhy+BhRBleFjpJ2vchUNN40qgkh0366FWJGqVLYBHev0vpHTrXSA0ryT+74UiW6KWsldNurQMKGqCm1M2zBciQ==
@@ -5169,7 +5157,7 @@ universalify@^0.2.0:
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0"
   integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==
 
-update-browserslist-db@^1.0.11, update-browserslist-db@^1.0.13:
+update-browserslist-db@^1.0.13:
   version "1.0.13"
   resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4"
   integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==
@@ -5247,10 +5235,10 @@ vite-tsconfig-paths@^3.5.0:
     recrawl-sync "^2.0.3"
     tsconfig-paths "^4.0.0"
 
-vite@^5.0.0, vite@^5.1.4:
-  version "5.1.4"
-  resolved "https://registry.yarnpkg.com/vite/-/vite-5.1.4.tgz#14e9d3e7a6e488f36284ef13cebe149f060bcfb6"
-  integrity sha512-n+MPqzq+d9nMVTKyewqw6kSt+R3CkvF9QAKY8obiQn8g1fwTscKxyfaYnC632HtBXAQGc1Yjomphwn1dtwGAHg==
+vite@^5.0.0, vite@^5.1.7:
+  version "5.1.7"
+  resolved "https://registry.yarnpkg.com/vite/-/vite-5.1.7.tgz#9f685a2c4c70707fef6d37341b0e809c366da619"
+  integrity sha512-sgnEEFTZYMui/sTlH1/XEnVNHMujOahPLGMxn1+5sIT45Xjng1Ec1K78jRP15dSmVgg5WBin9yO81j3o9OxofA==
   dependencies:
     esbuild "^0.19.3"
     postcss "^8.4.35"

clientupdate/clientupdate.go

@@ -37,11 +37,18 @@ import (
 )
 
 const (
-	CurrentTrack  = ""
 	StableTrack   = "stable"
 	UnstableTrack = "unstable"
 )
 
+var CurrentTrack = func() string {
+	if version.IsUnstableBuild() {
+		return UnstableTrack
+	} else {
+		return StableTrack
+	}
+}()
+
 func versionToTrack(v string) (string, error) {
 	_, rest, ok := strings.Cut(v, ".")
 	if !ok {
@@ -106,7 +113,7 @@ func (args Arguments) validate() error {
 		return fmt.Errorf("only one of Version(%q) or Track(%q) can be set", args.Version, args.Track)
 	}
 	switch args.Track {
-	case StableTrack, UnstableTrack, CurrentTrack:
+	case StableTrack, UnstableTrack, "":
 		// All valid values.
 	default:
 		return fmt.Errorf("unsupported track %q", args.Track)
@@ -119,11 +126,17 @@ type Updater struct {
 	// Update is a platform-specific method that updates the installation. May be
 	// nil (not all platforms support updates from within Tailscale).
 	Update func() error
+
+	// currentVersion is the short form of the current client version as
+	// returned by version.Short(), typically "x.y.z". Used for tests to
+	// override the actual current version.
+	currentVersion string
 }
 
 func NewUpdater(args Arguments) (*Updater, error) {
 	up := Updater{
-		Arguments: args,
+		Arguments:      args,
+		currentVersion: version.Short(),
 	}
 	if up.Stdout == nil {
 		up.Stdout = os.Stdout
@@ -139,18 +152,15 @@ func NewUpdater(args Arguments) (*Updater, error) {
 	if args.ForAutoUpdate && !canAutoUpdate {
 		return nil, errors.ErrUnsupported
 	}
-	if up.Track == CurrentTrack {
-		switch {
-		case up.Version != "":
+	if up.Track == "" {
+		if up.Version != "" {
 			var err error
 			up.Track, err = versionToTrack(args.Version)
 			if err != nil {
 				return nil, err
 			}
-		case version.IsUnstableBuild():
-			up.Track = UnstableTrack
-		default:
-			up.Track = StableTrack
+		} else {
+			up.Track = CurrentTrack
 		}
 	}
 	if up.Arguments.PkgsAddr == "" {
@@ -238,6 +248,11 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 // CanAutoUpdate reports whether auto-updating via the clientupdate package
 // is supported for the current os/distro.
 func CanAutoUpdate() bool {
+	if version.IsMacSysExt() {
+		// Macsys uses Sparkle for auto-updates, which doesn't have an update
+		// function in this package.
+		return true
+	}
 	_, canAutoUpdate := (&Updater{}).getUpdateFunction()
 	return canAutoUpdate
 }
@@ -259,13 +274,16 @@ func Update(args Arguments) error {
 }
 
 func (up *Updater) confirm(ver string) bool {
-	switch cmpver.Compare(version.Short(), ver) {
-	case 0:
-		up.Logf("already running %v version %v; no update needed", up.Track, ver)
-		return false
-	case 1:
-		up.Logf("installed %v version %v is newer than the latest available version %v; no update needed", up.Track, version.Short(), ver)
-		return false
+	// Only check version when we're not switching tracks.
+	if up.Track == "" || up.Track == CurrentTrack {
+		switch c := cmpver.Compare(up.currentVersion, ver); {
+		case c == 0:
+			up.Logf("already running %v version %v; no update needed", up.Track, ver)
+			return false
+		case c > 0:
+			up.Logf("installed %v version %v is newer than the latest available version %v; no update needed", up.Track, up.currentVersion, ver)
+			return false
+		}
 	}
 	if up.Confirm != nil {
 		return up.Confirm(ver)
@@ -436,7 +454,7 @@ func (up *Updater) updateDebLike() error {
 		return fmt.Errorf("apt-get update failed: %w; output:\n%s", err, out)
 	}
 
-	for i := 0; i < 2; i++ {
+	for range 2 {
 		out, err := exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver).CombinedOutput()
 		if err != nil {
 			if !bytes.Contains(out, []byte(`dpkg was interrupted`)) {
@@ -651,6 +669,9 @@ func (up *Updater) updateAlpineLike() (err error) {
 		return fmt.Errorf(`failed to parse latest version from "apk info tailscale": %w`, err)
 	}
 	if !up.confirm(ver) {
+		if err := checkOutdatedAlpineRepo(up.Logf, ver, up.Track); err != nil {
+			up.Logf("failed to check whether Alpine release is outdated: %v", err)
+		}
 		return nil
 	}
 
@@ -678,7 +699,7 @@ func parseAlpinePackageVersion(out []byte) (string, error) {
 			return "", fmt.Errorf("malformed info line: %q", line)
 		}
 		ver := parts[1]
-		if cmpver.Compare(ver, maxVer) == 1 {
+		if cmpver.Compare(ver, maxVer) > 0 {
 			maxVer = ver
 		}
 	}
@@ -688,6 +709,37 @@ func parseAlpinePackageVersion(out []byte) (string, error) {
 	return "", errors.New("tailscale version not found in output")
 }
 
+var apkRepoVersionRE = regexp.MustCompile(`v[0-9]+\.[0-9]+`)
+
+func checkOutdatedAlpineRepo(logf logger.Logf, apkVer, track string) error {
+	latest, err := LatestTailscaleVersion(track)
+	if err != nil {
+		return err
+	}
+	if latest == apkVer {
+		// Actually on latest release.
+		return nil
+	}
+	f, err := os.Open("/etc/apk/repositories")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	// Read the first repo line. Typically, there are multiple repos that all
+	// contain the same version in the path, like:
+	//   https://dl-cdn.alpinelinux.org/alpine/v3.20/main
+	//   https://dl-cdn.alpinelinux.org/alpine/v3.20/community
+	s := bufio.NewScanner(f)
+	if !s.Scan() {
+		return s.Err()
+	}
+	alpineVer := apkRepoVersionRE.FindString(s.Text())
+	if alpineVer != "" {
+		logf("The latest Tailscale release for Linux is %q, but your apk repository only provides %q.\nYour Alpine version is %q, you may need to upgrade the system to get the latest Tailscale version: https://wiki.alpinelinux.org/wiki/Upgrading_Alpine", latest, apkVer, alpineVer)
+	}
+	return nil
+}
+
 func (up *Updater) updateMacSys() error {
 	return errors.New("NOTREACHED: On MacSys builds, `tailscale update` is handled in Swift to launch the GUI updater")
 }
@@ -846,7 +898,7 @@ func (up *Updater) installMSI(msi string) error {
 			break
 		}
 		up.Logf("Install attempt failed: %v", err)
-		uninstallVersion := version.Short()
+		uninstallVersion := up.currentVersion
 		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
 			uninstallVersion = v
 		}
@@ -1017,6 +1069,20 @@ func (up *Updater) updateLinuxBinary() error {
 	return nil
 }
 
+func restartSystemdUnit(ctx context.Context) error {
+	if _, err := exec.LookPath("systemctl"); err != nil {
+		// Likely not a systemd-managed distro.
+		return errors.ErrUnsupported
+	}
+	if out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput(); err != nil {
+		return fmt.Errorf("systemctl daemon-reload failed: %w\noutput: %s", err, out)
+	}
+	if out, err := exec.Command("systemctl", "restart", "tailscaled.service").CombinedOutput(); err != nil {
+		return fmt.Errorf("systemctl restart failed: %w\noutput: %s", err, out)
+	}
+	return nil
+}
+
 func (up *Updater) downloadLinuxTarball(ver string) (string, error) {
 	dlDir, err := os.UserCacheDir()
 	if err != nil {
@@ -1283,22 +1349,31 @@ func requestedTailscaleVersion(ver, track string) (string, error) {
 // LatestTailscaleVersion returns the latest released version for the given
 // track from pkgs.tailscale.com.
 func LatestTailscaleVersion(track string) (string, error) {
-	if track == CurrentTrack {
-		if version.IsUnstableBuild() {
-			track = UnstableTrack
-		} else {
-			track = StableTrack
-		}
+	if track == "" {
+		track = CurrentTrack
 	}
 
 	latest, err := latestPackages(track)
 	if err != nil {
 		return "", err
 	}
-	if latest.Version == "" {
-		return "", fmt.Errorf("no latest version found for %q track", track)
+	ver := latest.Version
+	switch runtime.GOOS {
+	case "windows":
+		ver = latest.MSIsVersion
+	case "darwin":
+		ver = latest.MacZipsVersion
+	case "linux":
+		ver = latest.TarballsVersion
+		if distro.Get() == distro.Synology {
+			ver = latest.SPKsVersion
+		}
 	}
-	return latest.Version, nil
+
+	if ver == "" {
+		return "", fmt.Errorf("no latest version found for OS %q on %q track", runtime.GOOS, track)
+	}
+	return ver, nil
 }
 
 type trackPackages struct {

clientupdate/clientupdate_test.go

@@ -663,7 +663,7 @@ func genTarball(t *testing.T, path string, files map[string]string) {
 
 func TestWriteFileOverwrite(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "test")
-	for i := 0; i < 2; i++ {
+	for i := range 2 {
 		content := fmt.Sprintf("content %d", i)
 		if err := writeFile(strings.NewReader(content), path, 0600); err != nil {
 			t.Fatal(err)
@@ -846,3 +846,107 @@ func TestParseUnraidPluginVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestConfirm(t *testing.T) {
+	curTrack := CurrentTrack
+	defer func() { CurrentTrack = curTrack }()
+
+	tests := []struct {
+		desc      string
+		fromTrack string
+		toTrack   string
+		fromVer   string
+		toVer     string
+		confirm   func(string) bool
+		want      bool
+	}{
+		{
+			desc:      "on latest stable",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.66.0",
+			want:      false,
+		},
+		{
+			desc:      "stable upgrade",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.68.0",
+			want:      true,
+		},
+		{
+			desc:      "unstable upgrade",
+			fromTrack: UnstableTrack,
+			toTrack:   UnstableTrack,
+			fromVer:   "1.67.1",
+			toVer:     "1.67.2",
+			want:      true,
+		},
+		{
+			desc:      "from stable to unstable",
+			fromTrack: StableTrack,
+			toTrack:   UnstableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.67.1",
+			want:      true,
+		},
+		{
+			desc:      "from unstable to stable",
+			fromTrack: UnstableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.67.1",
+			toVer:     "1.66.0",
+			want:      true,
+		},
+		{
+			desc:      "confirm callback rejects",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.66.1",
+			confirm: func(string) bool {
+				return false
+			},
+			want: false,
+		},
+		{
+			desc:      "confirm callback allows",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.66.1",
+			confirm: func(string) bool {
+				return true
+			},
+			want: true,
+		},
+		{
+			desc:      "downgrade",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.1",
+			toVer:     "1.66.0",
+			want:      false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			CurrentTrack = tt.fromTrack
+			up := Updater{
+				currentVersion: tt.fromVer,
+				Arguments: Arguments{
+					Track:   tt.toTrack,
+					Confirm: tt.confirm,
+					Logf:    t.Logf,
+				},
+			}
+
+			if got := up.confirm(tt.toVer); got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

clientupdate/distsign/distsign_test.go

@@ -445,7 +445,7 @@ type testServer struct {
 
 func newTestServer(t *testing.T) *testServer {
 	var roots []rootKeyPair
-	for i := 0; i < 3; i++ {
+	for range 3 {
 		roots = append(roots, newRootKeyPair(t))
 	}
 

clientupdate/systemd_linux.go

@@ -1,37 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package clientupdate
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"github.com/coreos/go-systemd/v22/dbus"
-)
-
-func restartSystemdUnit(ctx context.Context) error {
-	c, err := dbus.NewWithContext(ctx)
-	if err != nil {
-		// Likely not a systemd-managed distro.
-		return errors.ErrUnsupported
-	}
-	defer c.Close()
-	if err := c.ReloadContext(ctx); err != nil {
-		return fmt.Errorf("failed to reload tailsacled.service: %w", err)
-	}
-	ch := make(chan string, 1)
-	if _, err := c.RestartUnitContext(ctx, "tailscaled.service", "replace", ch); err != nil {
-		return fmt.Errorf("failed to restart tailsacled.service: %w", err)
-	}
-	select {
-	case res := <-ch:
-		if res != "done" {
-			return fmt.Errorf("systemd service restart failed with result %q", res)
-		}
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-	return nil
-}

clientupdate/systemd_other.go

@@ -1,15 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !linux
-
-package clientupdate
-
-import (
-	"context"
-	"errors"
-)
-
-func restartSystemdUnit(ctx context.Context) error {
-	return errors.ErrUnsupported
-}

cmd/cloner/cloner.go

@@ -78,7 +78,11 @@ func main() {
 		w("	return false")
 		w("}")
 	}
-	cloneOutput := pkg.Name + "_clone.go"
+	cloneOutput := pkg.Name + "_clone"
+	if *flagBuildTags == "test" {
+		cloneOutput += "_test"
+	}
+	cloneOutput += ".go"
 	if err := codegen.WritePackageFile("tailscale.com/cmd/cloner", pkg, cloneOutput, it, buf); err != nil {
 		log.Fatal(err)
 	}
@@ -91,18 +95,21 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 	}
 
 	name := typ.Obj().Name()
+	typeParams := typ.Origin().TypeParams()
+	_, typeParamNames := codegen.FormatTypeParams(typeParams, it)
+	nameWithParams := name + typeParamNames
 	fmt.Fprintf(buf, "// Clone makes a deep copy of %s.\n", name)
 	fmt.Fprintf(buf, "// The result aliases no memory with the original.\n")
-	fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", name, name)
+	fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", nameWithParams, nameWithParams)
 	writef := func(format string, args ...any) {
 		fmt.Fprintf(buf, "\t"+format+"\n", args...)
 	}
 	writef("if src == nil {")
 	writef("\treturn nil")
 	writef("}")
-	writef("dst := new(%s)", name)
+	writef("dst := new(%s)", nameWithParams)
 	writef("*dst = *src")
-	for i := 0; i < t.NumFields(); i++ {
+	for i := range t.NumFields() {
 		fname := t.Field(i).Name()
 		ft := t.Field(i).Type()
 		if !codegen.ContainsPointers(ft) || codegen.HasNoClone(t.Tag(i)) {
@@ -126,16 +133,23 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname)
 				writef("for i := range dst.%s {", fname)
 				if ptr, isPtr := ft.Elem().(*types.Pointer); isPtr {
-					if _, isBasic := ptr.Elem().Underlying().(*types.Basic); isBasic {
-						it.Import("tailscale.com/types/ptr")
-						writef("if src.%s[i] == nil { dst.%s[i] = nil } else {", fname, fname)
-						writef("\tdst.%s[i] = ptr.To(*src.%s[i])", fname, fname)
-						writef("}")
+					writef("if src.%s[i] == nil { dst.%s[i] = nil } else {", fname, fname)
+					if codegen.ContainsPointers(ptr.Elem()) {
+						if _, isIface := ptr.Elem().Underlying().(*types.Interface); isIface {
+							it.Import("tailscale.com/types/ptr")
+							writef("\tdst.%s[i] = ptr.To((*src.%s[i]).Clone())", fname, fname)
+						} else {
+							writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+						}
 					} else {
-						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+						it.Import("tailscale.com/types/ptr")
+						writef("\tdst.%s[i] = ptr.To(*src.%s[i])", fname, fname)
 					}
+					writef("}")
 				} else if ft.Elem().String() == "encoding/json.RawMessage" {
 					writef("\tdst.%s[i] = append(src.%s[i][:0:0], src.%s[i]...)", fname, fname, fname)
+				} else if _, isIface := ft.Elem().Underlying().(*types.Interface); isIface {
+					writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
 				} else {
 					writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname)
 				}
@@ -145,14 +159,19 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("dst.%s = append(src.%s[:0:0], src.%s...)", fname, fname, fname)
 			}
 		case *types.Pointer:
-			if named, _ := ft.Elem().(*types.Named); named != nil && codegen.ContainsPointers(ft.Elem()) {
+			base := ft.Elem()
+			hasPtrs := codegen.ContainsPointers(base)
+			if named, _ := base.(*types.Named); named != nil && hasPtrs {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}
 			it.Import("tailscale.com/types/ptr")
 			writef("if dst.%s != nil {", fname)
-			writef("\tdst.%s = ptr.To(*src.%s)", fname, fname)
-			if codegen.ContainsPointers(ft.Elem()) {
+			if _, isIface := base.Underlying().(*types.Interface); isIface && hasPtrs {
+				writef("\tdst.%s = ptr.To((*src.%s).Clone())", fname, fname)
+			} else if !hasPtrs {
+				writef("\tdst.%s = ptr.To(*src.%s)", fname, fname)
+			} else {
 				writef("\t" + `panic("TODO pointers in pointers")`)
 			}
 			writef("}")
@@ -172,18 +191,50 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("if dst.%s != nil {", fname)
 				writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem))
 				writef("\tfor k, v := range src.%s {", fname)
-				switch elem.(type) {
+
+				switch elem := elem.Underlying().(type) {
 				case *types.Pointer:
-					writef("\t\tdst.%s[k] = v.Clone()", fname)
+					writef("\t\tif v == nil { dst.%s[k] = nil } else {", fname)
+					if base := elem.Elem().Underlying(); codegen.ContainsPointers(base) {
+						if _, isIface := base.(*types.Interface); isIface {
+							it.Import("tailscale.com/types/ptr")
+							writef("\t\t\tdst.%s[k] = ptr.To((*v).Clone())", fname)
+						} else {
+							writef("\t\t\tdst.%s[k] = v.Clone()", fname)
+						}
+					} else {
+						it.Import("tailscale.com/types/ptr")
+						writef("\t\t\tdst.%s[k] = ptr.To(*v)", fname)
+					}
+					writef("}")
+				case *types.Interface:
+					if cloneResultType := methodResultType(elem, "Clone"); cloneResultType != nil {
+						if _, isPtr := cloneResultType.(*types.Pointer); isPtr {
+							writef("\t\tdst.%s[k] = *(v.Clone())", fname)
+						} else {
+							writef("\t\tdst.%s[k] = v.Clone()", fname)
+						}
+					} else {
+						writef(`panic("%s (%v) does not have a Clone method")`, fname, elem)
+					}
 				default:
 					writef("\t\tdst.%s[k] = *(v.Clone())", fname)
 				}
+
 				writef("\t}")
 				writef("}")
 			} else {
 				it.Import("maps")
 				writef("\tdst.%s = maps.Clone(src.%s)", fname, fname)
 			}
+		case *types.Interface:
+			// If ft is an interface with a "Clone() ft" method, it can be used to clone the field.
+			// This includes scenarios where ft is a constrained type parameter.
+			if cloneResultType := methodResultType(ft, "Clone"); cloneResultType.Underlying() == ft {
+				writef("dst.%s = src.%s.Clone()", fname, fname)
+				continue
+			}
+			writef(`panic("%s (%v) does not have a compatible Clone method")`, fname, ft)
 		default:
 			writef(`panic("TODO: %s (%T)")`, fname, ft)
 		}
@@ -191,7 +242,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 	writef("return dst")
 	fmt.Fprintf(buf, "}\n\n")
 
-	buf.Write(codegen.AssertStructUnchanged(t, name, "Clone", it))
+	buf.Write(codegen.AssertStructUnchanged(t, name, typeParams, "Clone", it))
 }
 
 // hasBasicUnderlying reports true when typ.Underlying() is a slice or a map.
@@ -203,3 +254,15 @@ func hasBasicUnderlying(typ types.Type) bool {
 		return false
 	}
 }
+
+func methodResultType(typ types.Type, method string) types.Type {
+	viewMethod := codegen.LookupMethod(typ, method)
+	if viewMethod == nil {
+		return nil
+	}
+	sig, ok := viewMethod.Type().(*types.Signature)
+	if !ok || sig.Results().Len() != 1 {
+		return nil
+	}
+	return sig.Results().At(0).Type()
+}

cmd/cloner/clonerex/clonerex.go

@@ -3,6 +3,7 @@
 
 //go:generate go run tailscale.com/cmd/cloner  -clonefunc=true -type SliceContainer
 
+// Package clonerex is an example package for the cloner tool.
 package clonerex
 
 type SliceContainer struct {

cmd/containerboot/kube.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -18,36 +19,20 @@ import (
 	"tailscale.com/tailcfg"
 )
 
-// findKeyInKubeSecret inspects the kube secret secretName for a data
-// field called "authkey", and returns its value if present.
-func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
-	s, err := kc.GetSecret(ctx, secretName)
-	if err != nil {
-		return "", err
+// storeDeviceID writes deviceID to 'device_id' data field of the named
+// Kubernetes Secret.
+func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
+	s := &kube.Secret{
+		Data: map[string][]byte{
+			"device_id": []byte(deviceID),
+		},
 	}
-	ak, ok := s.Data["authkey"]
-	if !ok {
-		return "", nil
-	}
-	return string(ak), nil
+	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
 }
 
-// storeDeviceInfo writes deviceID into the "device_id" data field of the kube
-// secret secretName.
-func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
-	// First check if the secret exists at all. Even if running on
-	// kubernetes, we do not necessarily store state in a k8s secret.
-	if _, err := kc.GetSecret(ctx, secretName); err != nil {
-		if s, ok := err.(*kube.Status); ok {
-			if s.Code >= 400 && s.Code <= 499 {
-				// Assume the secret doesn't exist, or we don't have
-				// permission to access it.
-				return nil
-			}
-		}
-		return err
-	}
-
+// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields
+// 'device_ips', 'device_fqdn' of the named Kubernetes Secret.
+func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, addresses []netip.Prefix) error {
 	var ips []string
 	for _, addr := range addresses {
 		ips = append(ips, addr.Addr().String())
@@ -57,14 +42,13 @@ func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.St
 		return err
 	}
 
-	m := &kube.Secret{
+	s := &kube.Secret{
 		Data: map[string][]byte{
-			"device_id":   []byte(deviceID),
 			"device_fqdn": []byte(fqdn),
 			"device_ips":  deviceIPs,
 		},
 	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, m, "tailscale-container")
+	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
 }
 
 // deleteAuthKey deletes the 'authkey' field of the given kube
@@ -88,9 +72,59 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc *kube.Client
+var kc kube.Client
 
-func initKube(root string) {
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kube.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
+
+func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.
@@ -101,9 +135,9 @@ func initKube(root string) {
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}
-	if root != "/" {
-		// If we are running in a test, we need to set the URL to the
-		// httptest server.
+	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		// Used to set http server in tests, or optionally enabled by flag
 		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
 	}
 }

cmd/containerboot/kube_test.go

@@ -0,0 +1,206 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/kube"
+)
+
+func TestSetupKube(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *settings
+		wantErr bool
+		wantCfg *settings
+		kc      kube.Client
+	}{
+		{
+			name: "TS_AUTHKEY set, state Secret exists",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, nil
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+		},
+		{
+			name: "TS_AUTHKEY set, state Secret does not exist, we have permissions to create it",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, true, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+		},
+		{
+			name: "TS_AUTHKEY set, state Secret does not exist, we do not have permissions to create it",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY set, we encounter a non-404 error when trying to retrieve the state Secret",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 403}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY set, we encounter a non-404 error when trying to check Secret permissions",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, errors.New("broken")
+				},
+			},
+			wantErr: true,
+		},
+		{
+			// Interactive login using URL in Pod logs
+			name: "TS_AUTHKEY not set, state Secret does not exist, we have permissions to create it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, true, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+		},
+		{
+			// Interactive login using URL in Pod logs
+			name: "TS_AUTHKEY not set, state Secret exists, but does not contain auth key",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{}, nil
+				},
+			},
+		},
+		{
+			name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				},
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY not set, state Secret contains auth key, we have RBAC to patch it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return true, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				},
+			},
+			wantCfg: &settings{
+				KubeSecret:         "foo",
+				AuthKey:            "foo",
+				KubernetesCanPatch: true,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		kc = tt.kc
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
+				t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {
+				t.Errorf("unexpected contents of settings after running settings.setupKube()\n(-got +want):\n%s", diff)
+			}
+		})
+	}
+}

cmd/containerboot/main.go

@@ -18,7 +18,11 @@
 //     previously advertised routes. To accept routes, use TS_EXTRA_ARGS to pass
 //     in --accept-routes.
 //   - TS_DEST_IP: proxy all incoming Tailscale traffic to the given
-//     destination.
+//     destination defined by an IP address.
+//   - TS_EXPERIMENTAL_DEST_DNS_NAME: proxy all incoming Tailscale traffic to the given
+//     destination defined by a DNS name. The DNS name will be periodically resolved and firewall rules updated accordingly.
+//     This is currently intended to be used by the Kubernetes operator (ExternalName Services).
+//     This is an experimental env var and will likely change in the future.
 //   - TS_TAILNET_TARGET_IP: proxy all incoming non-Tailscale traffic to the given
 //     destination defined by an IP.
 //   - TS_TAILNET_TARGET_FQDN: proxy all incoming non-Tailscale traffic to the given
@@ -48,13 +52,20 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
-//   - EXPERIMENTAL_TS_CONFIGFILE_PATH: if specified, a path to tailscaled
-//     config. If this is set, TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY,
+//   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
+//     directory that containers tailscaled config in file. The config file needs to be
+//     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
+//     TS_EXTRA_ARGS, TS_AUTHKEY,
 //     TS_ROUTES, TS_ACCEPT_DNS env vars must not be set. If this is set,
 //     containerboot only runs `tailscaled --config <path-to-this-configfile>`
 //     and not `tailscale up` or `tailscale set`.
 //     The config file contents are currently read once on container start.
 //     NB: This env var is currently experimental and the logic will likely change!
+//     TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS: set to true to
+//     autoconfigure the default network interface for optimal performance for
+//     Tailscale subnet router/exit node.
+//     https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+//     NB: This env var is currently experimental and the logic will likely change!
 //   - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true
 //     and if this containerboot instance is an L7 ingress proxy (created by
 //     the Kubernetes operator), set up rules to allow proxying cluster traffic,
@@ -82,12 +93,16 @@ import (
 	"fmt"
 	"io/fs"
 	"log"
+	"math"
+	"net"
 	"net/netip"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -100,6 +115,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/conffile"
+	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/ptr"
@@ -122,7 +138,8 @@ func main() {
 		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
 		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
 		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
-		ProxyTo:                               defaultEnv("TS_DEST_IP", ""),
+		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
+		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
 		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
 		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
 		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
@@ -137,9 +154,10 @@ func main() {
 		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
 		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
 		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
-		TailscaledConfigFilePath:              defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""),
+		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -150,8 +168,8 @@ func main() {
 		if err := ensureTunFile(cfg.Root); err != nil {
 			log.Fatalf("Unable to create tuntap device file: %v", err)
 		}
-		if cfg.ProxyTo != "" || cfg.Routes != nil || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" {
-			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.TailnetTargetIP, cfg.TailnetTargetFQDN, cfg.Routes); err != nil {
+		if cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.Routes != nil || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" {
+			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTargetIP, cfg.TailnetTargetIP, cfg.TailnetTargetFQDN, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
@@ -163,44 +181,16 @@ func main() {
 		}
 	}
 
-	if cfg.InKubernetes {
-		initKube(cfg.Root)
-	}
-
 	// Context is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
 	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" {
-		canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret)
-		if err != nil {
-			log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-		}
-		cfg.KubernetesCanPatch = canPatch
-
-		if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-			key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret)
-			if err != nil {
-				log.Fatalf("Getting authkey from kube secret: %v", err)
-			}
-			if key != "" {
-				// This behavior of pulling authkeys from kube secrets was added
-				// at the same time as the patch permission, so we can enforce
-				// that we must be able to patch out the authkey after
-				// authenticating if you want to use this feature. This avoids
-				// us having to deal with the case where we might leave behind
-				// an unnecessary reusable authkey in a secret, like a rake in
-				// the grass.
-				if !cfg.KubernetesCanPatch {
-					log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-				}
-				log.Print("Using authkey found in kube secret")
-				cfg.AuthKey = key
-			} else {
-				log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-			}
+	if cfg.InKubernetes {
+		initKubeClient(cfg.Root)
+		if err := cfg.setupKube(bootCtx); err != nil {
+			log.Fatalf("error setting up for running on Kubernetes: %v", err)
 		}
 	}
 
@@ -215,6 +205,12 @@ func main() {
 	}
 	defer killTailscaled()
 
+	if cfg.EnableForwardingOptimizations {
+		if err := client.SetUDPGROForwarding(bootCtx); err != nil {
+			log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
+		}
+	}
+
 	w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
 	if err != nil {
 		log.Fatalf("failed to watch tailscaled for updates: %v", err)
@@ -325,7 +321,7 @@ authLoop:
 		}
 	}
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch && isTwoStepConfigAuthOnce(cfg) {
+	if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) {
 		// We were told to only auth once, so any secret-bound
 		// authkey is no longer needed. We don't strictly need to
 		// wipe it, but it's good hygiene.
@@ -341,14 +337,16 @@ authLoop:
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
-		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
-		startupTasksDone  = false
-		currentIPs        deephash.Sum // tailscale IPs assigned to device
-		currentDeviceInfo deephash.Sum // device ID and fqdn
+		startupTasksDone       = false
+		currentIPs             deephash.Sum // tailscale IPs assigned to device
+		currentDeviceID        deephash.Sum // device ID
+		currentDeviceEndpoints deephash.Sum // device FQDN and IPs
 
 		currentEgressIPs deephash.Sum
 
+		addrs        []netip.Prefix
+		backendAddrs []net.IP
+
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
 	)
@@ -356,12 +354,50 @@ authLoop:
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
 	}
 	var nfr linuxfw.NetfilterRunner
-	if wantProxy {
+	if isL3Proxy(cfg) {
 		nfr, err = newNetfilterRunner(log.Printf)
 		if err != nil {
 			log.Fatalf("error creating new netfilter runner: %v", err)
 		}
 	}
+
+	// Setup for proxies that are configured to proxy to a target specified
+	// by a DNS name (TS_EXPERIMENTAL_DEST_DNS_NAME).
+	const defaultCheckPeriod = time.Minute * 10 // how often to check what IPs the DNS name resolves to
+	var (
+		tc                    = make(chan string, 1)
+		failedResolveAttempts int
+		t                     *time.Timer = time.AfterFunc(defaultCheckPeriod, func() {
+			if cfg.ProxyTargetDNSName != "" {
+				tc <- "recheck"
+			}
+		})
+	)
+	defer t.Stop()
+	// resetTimer resets timer for when to next attempt to resolve the DNS
+	// name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The
+	// timer gets reset to 10 minutes from now unless the last resolution
+	// attempt failed. If one or more consecutive previous resolution
+	// attempts failed, the next resolution attempt will happen after the smallest
+	// of (10 minutes, 2 ^ number-of-consecutive-failed-resolution-attempts
+	// seconds) i.e 2s, 4s, 8s ... 10 minutes.
+	resetTimer := func(lastResolveFailed bool) {
+		if !lastResolveFailed {
+			log.Printf("reconfigureTimer: next DNS resolution attempt in %s", defaultCheckPeriod)
+			t.Reset(defaultCheckPeriod)
+			failedResolveAttempts = 0
+			return
+		}
+		minDelay := 2 // 2 seconds
+		nextTick := time.Second * time.Duration(math.Pow(float64(minDelay), float64(failedResolveAttempts)))
+		if nextTick > defaultCheckPeriod {
+			nextTick = defaultCheckPeriod // cap at 10 minutes
+		}
+		log.Printf("reconfigureTimer: last DNS resolution attempt failed, next DNS resolution attempt in %v", nextTick)
+		t.Reset(nextTick)
+		failedResolveAttempts++
+	}
+
 	notifyChan := make(chan ipn.Notify)
 	errChan := make(chan error)
 	go func() {
@@ -399,10 +435,24 @@ runLoop:
 				log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
 			}
 			if n.NetMap != nil {
-				addrs := n.NetMap.SelfNode.Addresses().AsSlice()
+				addrs = n.NetMap.SelfNode.Addresses().AsSlice()
 				newCurrentIPs := deephash.Hash(&addrs)
 				ipsHaveChanged := newCurrentIPs != currentIPs
 
+				// Store device ID in a Kubernetes Secret before
+				// setting up any routing rules. This ensures
+				// that, for containerboot instances that are
+				// Kubernetes operator proxies, the operator is
+				// able to retrieve the device ID from the
+				// Kubernetes Secret to clean up tailnet nodes
+				// for proxies whose route setup continuously
+				// fails.
+				deviceID := n.NetMap.SelfNode.StableID()
+				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceID, &deviceID) {
+					if err := storeDeviceID(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID()); err != nil {
+						log.Fatalf("storing device ID in Kubernetes Secret: %v", err)
+					}
+				}
 				if cfg.TailnetTargetFQDN != "" {
 					var (
 						egressAddrs          []netip.Prefix
@@ -425,29 +475,50 @@ runLoop:
 					egressAddrs = node.Addresses().AsSlice()
 					newCurentEgressIPs = deephash.Hash(&egressAddrs)
 					egressIPsHaveChanged = newCurentEgressIPs != currentEgressIPs
-					if egressIPsHaveChanged && len(egressAddrs) > 0 {
+					if egressIPsHaveChanged && len(egressAddrs) != 0 {
+						var rulesInstalled bool
 						for _, egressAddr := range egressAddrs {
 							ea := egressAddr.Addr()
-							// TODO (irbekrm): make it work for IPv6 too.
-							if ea.Is6() {
-								log.Println("Not installing egress forwarding rules for IPv6 as this is currently not supported")
-								continue
-							}
-							log.Printf("Installing forwarding rules for destination %v", ea.String())
-							if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil {
-								log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err)
+							if ea.Is4() || (ea.Is6() && nfr.HasIPV6NAT()) {
+								rulesInstalled = true
+								log.Printf("Installing forwarding rules for destination %v", ea.String())
+								if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil {
+									log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err)
+								}
 							}
 						}
+						if !rulesInstalled {
+							log.Fatalf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT())
+						}
 					}
 					currentEgressIPs = newCurentEgressIPs
 				}
-				if cfg.ProxyTo != "" && len(addrs) > 0 && ipsHaveChanged {
+				if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged {
 					log.Printf("Installing proxy rules")
-					if err := installIngressForwardingRule(ctx, cfg.ProxyTo, addrs, nfr); err != nil {
+					if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil {
 						log.Fatalf("installing ingress proxy rules: %v", err)
 					}
 				}
-				if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) > 0 {
+				if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged {
+					newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName)
+					if err != nil {
+						log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err)
+						resetTimer(true)
+						continue
+					}
+					backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool {
+						return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) })
+					}))
+					if backendsHaveChanged {
+						log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs)
+						if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil {
+							log.Fatalf("error installing ingress proxy rules: %v", err)
+						}
+					}
+					resetTimer(false)
+					backendAddrs = newBackendAddrs
+				}
+				if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) != 0 {
 					cd := n.NetMap.DNS.CertDomains[0]
 					prev := certDomain.Swap(ptr.To(cd))
 					if prev == nil || *prev != cd {
@@ -457,7 +528,7 @@ runLoop:
 						}
 					}
 				}
-				if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) > 0 {
+				if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 {
 					log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP)
 					if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil {
 						log.Fatalf("installing egress proxy rules: %v", err)
@@ -469,7 +540,7 @@ runLoop:
 				// enabled, set up proxy rule each time the
 				// tailnet IPs of this node change (including
 				// the first time they become available).
-				if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) > 0 {
+				if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 {
 					log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP)
 					if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil {
 						log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err)
@@ -477,46 +548,85 @@ runLoop:
 				}
 				currentIPs = newCurrentIPs
 
-				deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()}
-				if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
-					if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
-						log.Fatalf("storing device ID in kube secret: %v", err)
+				// Only store device FQDN and IP addresses to
+				// Kubernetes Secret when any required proxy
+				// route setup has succeeded. IPs and FQDN are
+				// read from the Secret by the Tailscale
+				// Kubernetes operator and, for some proxy
+				// types, such as Tailscale Ingress, advertized
+				// on the Ingress status. Writing them to the
+				// Secret only after the proxy routing has been
+				// set up ensures that the operator does not
+				// advertize endpoints of broken proxies.
+				// TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'.
+				deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()}
+				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceEndpoints, &deviceEndpoints) {
+					if err := storeDeviceEndpoints(ctx, cfg.KubeSecret, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
+						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
 			}
 			if !startupTasksDone {
-				if (!wantProxy || currentIPs != deephash.Sum{}) && (!wantDeviceInfo || currentDeviceInfo != deephash.Sum{}) {
+				// For containerboot instances that act as TCP
+				// proxies (proxying traffic to an endpoint
+				// passed via one of the env vars that
+				// containerbot reads) and store state in a
+				// Kubernetes Secret, we consider startup tasks
+				// done at the point when device info has been
+				// successfully stored to state Secret.
+				// For all other containerboot instances, if we
+				// just get to this point the startup tasks can
+				// be considered done.
+				if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) {
 					// This log message is used in tests to detect when all
 					// post-auth configuration is done.
 					log.Println("Startup complete, waiting for shutdown signal")
 					startupTasksDone = true
 
-					// Reap all processes, since we are PID1 and need to collect zombies. We can
-					// only start doing this once we've stopped shelling out to things
-					// `tailscale up`, otherwise this goroutine can reap the CLI subprocesses
-					// and wedge bringup.
+					// Wait on tailscaled process. It won't
+					// be cleaned up by default when the
+					// container exits as it is not PID1.
+					// TODO (irbekrm): perhaps we can
+					// replace the reaper by a running
+					// cmd.Wait in a goroutine immediately
+					// after starting tailscaled?
 					reaper := func() {
 						defer wg.Done()
 						for {
 							var status unix.WaitStatus
-							pid, err := unix.Wait4(-1, &status, 0, nil)
+							_, err := unix.Wait4(daemonProcess.Pid, &status, 0, nil)
 							if errors.Is(err, unix.EINTR) {
 								continue
 							}
 							if err != nil {
-								log.Fatalf("Waiting for exited processes: %v", err)
-							}
-							if pid == daemonProcess.Pid {
-								log.Printf("Tailscaled exited")
-								os.Exit(0)
+								log.Fatalf("Waiting for tailscaled to exit: %v", err)
 							}
+							log.Print("tailscaled exited")
+							os.Exit(0)
 						}
-
 					}
 					wg.Add(1)
 					go reaper()
 				}
 			}
+		case <-tc:
+			newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName)
+			if err != nil {
+				log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err)
+				resetTimer(true)
+				continue
+			}
+			backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool {
+				return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) })
+			}))
+			if backendsHaveChanged && len(addrs) != 0 {
+				log.Printf("Backend address change detected, installing proxy rules for backends %v", newBackendAddrs)
+				if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil {
+					log.Fatalf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err)
+				}
+			}
+			backendAddrs = newBackendAddrs
+			resetTimer(false)
 		}
 	}
 	wg.Wait()
@@ -757,12 +867,12 @@ func ensureTunFile(root string) error {
 }
 
 // ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, clusterProxyTarget, tailnetTargetiP, tailnetTargetFQDN string, routes *string) error {
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
 	var (
 		v4Forwarding, v6Forwarding bool
 	)
-	if clusterProxyTarget != "" {
-		proxyIP, err := netip.ParseAddr(clusterProxyTarget)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
 		if err != nil {
 			return fmt.Errorf("invalid cluster destination IP: %v", err)
 		}
@@ -772,8 +882,8 @@ func ensureIPForwarding(root, clusterProxyTarget, tailnetTargetiP, tailnetTarget
 			v6Forwarding = true
 		}
 	}
-	if tailnetTargetiP != "" {
-		proxyIP, err := netip.ParseAddr(tailnetTargetiP)
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
 		if err != nil {
 			return fmt.Errorf("invalid tailnet destination IP: %v", err)
 		}
@@ -801,7 +911,10 @@ func ensureIPForwarding(root, clusterProxyTarget, tailnetTargetiP, tailnetTarget
 			}
 		}
 	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
 
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
 	var paths []string
 	if v4Forwarding {
 		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
@@ -830,7 +943,7 @@ func ensureIPForwarding(root, clusterProxyTarget, tailnetTargetiP, tailnetTarget
 	return nil
 }
 
-func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
 	dst, err := netip.ParseAddr(dstStr)
 	if err != nil {
 		return err
@@ -896,16 +1009,23 @@ func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []ne
 		return err
 	}
 	var local netip.Addr
+	proxyHasIPv4Address := false
 	for _, pfx := range tsIPs {
 		if !pfx.IsSingleIP() {
 			continue
 		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
 		if pfx.Addr().Is4() != dst.Is4() {
 			continue
 		}
 		local = pfx.Addr()
 		break
 	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
 	if !local.IsValid() {
 		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
 	}
@@ -918,15 +1038,89 @@ func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []ne
 	return nil
 }
 
+func installIngressForwardingRuleForDNSTarget(ctx context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}
+
 // settings is all the configuration for containerboot.
 type settings struct {
 	AuthKey  string
 	Hostname string
 	Routes   *string
-	// ProxyTo is the destination IP to which all incoming
+	// ProxyTargetIP is the destination IP to which all incoming
 	// Tailscale traffic should be proxied. If empty, no proxying
 	// is done. This is typically a locally reachable IP.
-	ProxyTo string
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
 	// TailnetTargetIP is the destination IP to which all incoming
 	// non-Tailscale traffic should be proxied. This is typically a
 	// Tailscale IP.
@@ -934,22 +1128,23 @@ type settings struct {
 	// TailnetTargetFQDN is an MagicDNS name to which all incoming
 	// non-Tailscale traffic should be proxied. This must be a full Tailnet
 	// node FQDN.
-	TailnetTargetFQDN        string
-	ServeConfigPath          string
-	DaemonExtraArgs          string
-	ExtraArgs                string
-	InKubernetes             bool
-	UserspaceMode            bool
-	StateDir                 string
-	AcceptDNS                *bool
-	KubeSecret               string
-	SOCKSProxyAddr           string
-	HTTPProxyAddr            string
-	Socket                   string
-	AuthOnce                 bool
-	Root                     string
-	KubernetesCanPatch       bool
-	TailscaledConfigFilePath string
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
 	// If set to true and, if this containerboot instance is a Kubernetes
 	// ingress proxy, set up rules to forward incoming cluster traffic to be
 	// forwarded to the ingress target in cluster.
@@ -962,13 +1157,26 @@ type settings struct {
 
 func (s *settings) validate() error {
 	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
 		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
 			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
 		}
 	}
-	if s.ProxyTo != "" && s.UserspaceMode {
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
 		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
 	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
 	if s.TailnetTargetIP != "" && s.UserspaceMode {
 		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
 	}
@@ -979,7 +1187,7 @@ func (s *settings) validate() error {
 		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
 	}
 	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("EXPERIMENTAL_TS_CONFIGFILE_PATH cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
 	}
 	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
 		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
@@ -990,9 +1198,34 @@ func (s *settings) validate() error {
 	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
 		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
 	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
 	return nil
 }
 
+func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
+	// TODO (irbekrm): look at using recursive.Resolver instead to resolve
+	// the DNS names as well as retrieve TTLs. It looks though that this
+	// seems to return very short TTLs (shorter than on the actual records).
+	ip4s, err := net.DefaultResolver.LookupIP(ctx, "ip4", name)
+	if err != nil {
+		if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) {
+			return nil, fmt.Errorf("error looking up IPv4 addresses: %v", err)
+		}
+	}
+	ip6s, err := net.DefaultResolver.LookupIP(ctx, "ip6", name)
+	if err != nil {
+		if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) {
+			return nil, fmt.Errorf("error looking up IPv6 addresses: %v", err)
+		}
+	}
+	if len(ip4s) == 0 && len(ip6s) == 0 {
+		return nil, fmt.Errorf("no IPv4 or IPv6 addresses found for host: %s", name)
+	}
+	return append(ip4s, ip6s...), nil
+}
+
 // defaultEnv returns the value of the given envvar name, or defVal if
 // unset.
 func defaultEnv(name, defVal string) string {
@@ -1089,3 +1322,55 @@ func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
 func isOneStepConfig(cfg *settings) bool {
 	return cfg.TailscaledConfigFilePath != ""
 }
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
+// tailscaledConfigFilePath returns the path to the tailscaled config file that
+// should be used for the current capability version. It is determined by the
+// TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a
+// file named cap-<capability_version>.hujson in the directory. It searches for
+// the highest capability version that is less than or equal to the current
+// capability version.
+func tailscaledConfigFilePath() string {
+	dir := os.Getenv("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR")
+	if dir == "" {
+		return ""
+	}
+	fe, err := os.ReadDir(dir)
+	if err != nil {
+		log.Fatalf("error reading tailscaled config directory %q: %v", dir, err)
+	}
+	maxCompatVer := tailcfg.CapabilityVersion(-1)
+	for _, e := range fe {
+		// We don't check if type if file as in most cases this will
+		// come from a mounted kube Secret, where the directory contents
+		// will be various symlinks.
+		if e.Type().IsDir() {
+			continue
+		}
+		cv, err := kubeutils.CapVerFromFileName(e.Name())
+		if err != nil {
+			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
+			continue
+		}
+		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
+			maxCompatVer = cv
+		}
+	}
+	if maxCompatVer == -1 {
+		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
+	}
+	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
+	return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer))
+}

cmd/containerboot/main_test.go

@@ -52,7 +52,7 @@ func TestContainerBoot(t *testing.T) {
 	}
 	defer kube.Close()
 
-	tailscaledConf := &ipn.ConfigVAlpha{AuthKey: func(s string) *string { return &s }("foo"), Version: "alpha0"}
+	tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"}
 	tailscaledConfBytes, err := json.Marshal(tailscaledConf)
 	if err != nil {
 		t.Fatalf("error unmarshaling tailscaled config: %v", err)
@@ -65,7 +65,7 @@ func TestContainerBoot(t *testing.T) {
 		"dev/net",
 		"proc/sys/net/ipv4",
 		"proc/sys/net/ipv6/conf/all",
-		"etc",
+		"etc/tailscaled",
 	}
 	for _, path := range dirs {
 		if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil {
@@ -80,7 +80,7 @@ func TestContainerBoot(t *testing.T) {
 		"dev/net/tun":                           []byte(""),
 		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
 		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
-		"etc/tailscaled":                        tailscaledConfBytes,
+		"etc/tailscaled/cap-95.hujson":          tailscaledConfBytes,
 	}
 	resetFiles := func() {
 		for path, content := range files {
@@ -116,6 +116,9 @@ func TestContainerBoot(t *testing.T) {
 		// WantFiles files that should exist in the container and their
 		// contents.
 		WantFiles map[string]string
+		// WantFatalLog is the fatal log message we expect from containerboot.
+		// If set for a phase, the test will finish on that phase.
+		WantFatalLog string
 	}
 	runningNotify := &ipn.Notify{
 		State: ptr.To(ipn.Running),
@@ -349,12 +352,57 @@ func TestContainerBoot(t *testing.T) {
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
 				},
 				{
 					Notify: runningNotify,
 				},
 			},
 		},
+		{
+			Name: "egress_proxy_fqdn_ipv6_target_on_ipv4_host",
+			Env: map[string]string{
+				"TS_AUTHKEY":               "tskey-key",
+				"TS_TAILNET_TARGET_FQDN":   "ipv6-node.test.ts.net", // resolves to IPv6 address
+				"TS_USERSPACE":             "false",
+				"TS_TEST_FAKE_NETFILTER_6": "false",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
+				},
+				{
+					Notify: &ipn.Notify{
+						State: ptr.To(ipn.Running),
+						NetMap: &netmap.NetworkMap{
+							SelfNode: (&tailcfg.Node{
+								StableID:  tailcfg.StableNodeID("myID"),
+								Name:      "test-node.test.ts.net",
+								Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
+							}).View(),
+							Peers: []tailcfg.NodeView{
+								(&tailcfg.Node{
+									StableID:  tailcfg.StableNodeID("ipv6ID"),
+									Name:      "ipv6-node.test.ts.net",
+									Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")},
+								}).View(),
+							},
+						},
+					},
+					WantFatalLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false",
+				},
+			},
+		},
 		{
 			Name: "authkey_once",
 			Env: map[string]string{
@@ -638,14 +686,14 @@ func TestContainerBoot(t *testing.T) {
 			},
 		},
 		{
-			Name: "experimental tailscaled configfile",
+			Name: "experimental tailscaled config path",
 			Env: map[string]string{
-				"EXPERIMENTAL_TS_CONFIGFILE_PATH": filepath.Join(d, "etc/tailscaled"),
+				"TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(d, "etc/tailscaled/"),
 			},
 			Phases: []phase{
 				{
 					WantCmds: []string{
-						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled",
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson",
 					},
 				}, {
 					Notify: runningNotify,
@@ -697,6 +745,25 @@ func TestContainerBoot(t *testing.T) {
 			var wantCmds []string
 			for i, p := range test.Phases {
 				lapi.Notify(p.Notify)
+				if p.WantFatalLog != "" {
+					err := tstest.WaitFor(2*time.Second, func() error {
+						state, err := cmd.Process.Wait()
+						if err != nil {
+							return err
+						}
+						if state.ExitCode() != 1 {
+							return fmt.Errorf("process exited with code %d but wanted %d", state.ExitCode(), 1)
+						}
+						waitLogLine(t, time.Second, cbOut, p.WantFatalLog)
+						return nil
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					// Early test return, we don't expect the successful startup log message.
+					return
+				}
 				wantCmds = append(wantCmds, p.WantCmds...)
 				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
 				err := tstest.WaitFor(2*time.Second, func() error {

cmd/derper/README.md

@@ -0,0 +1,109 @@
+# DERP
+
+This is the code for the [Tailscale DERP server](https://tailscale.com/kb/1232/derp-servers).
+
+In general, you should not need to or want to run this code. The overwhelming
+majority of Tailscale users (both individuals and companies) do not.
+
+In the happy path, Tailscale establishes direct connections between peers and
+data plane traffic flows directly between them, without using DERP for more than
+acting as a low bandwidth side channel to bootstrap the NAT traversal. If you
+find yourself wanting DERP for more bandwidth, the real problem is usually the
+network configuration of your Tailscale node(s), making sure that Tailscale can
+get direction connections via some mechanism.
+
+If you've decided or been advised to run your own `derper`, then read on.
+
+## Caveats
+
+* Node sharing and other cross-Tailnet features don't work when using custom
+  DERP servers.
+
+* DERP servers only see encrypted WireGuard packets and thus are not useful for
+  network-level debugging.
+
+* The Tailscale control plane does certain geo-level steering features and
+  optimizations that are not available when using custom DERP servers.
+
+## Guide to running `cmd/derper`
+
+* You must build and update the `cmd/derper` binary yourself. There are no
+  packages. Use `go install tailscale.com/cmd/derper@latest` with the latest
+  version of Go. You should update this binary approximately as regularly as
+  you update Tailscale nodes. If using `--verify-clients`, the `derper` binary
+  and `tailscaled` binary on the machine must be built from the same git revision.
+  (It might work otherwise, but they're developed and only tested together.)
+
+* The DERP protocol does a protocol switch inside TLS from HTTP to a custom
+  bidirectional binary protocol. It is thus incompatible with many HTTP proxies.
+  Do not put `derper` behind another HTTP proxy.
+
+* The `tailscaled` client does its own selection of the fastest/nearest DERP
+  server based on latency measurements. Do not put `derper` behind a global load
+  balancer.
+
+* DERP servers should ideally have both a static IPv4 and static IPv6 address.
+Both of those should be listed in the DERP map so the client doesn't need to
+rely on its DNS which might be broken and dependent on DERP to get back up.
+
+* A DERP server should not share an IP address with any other DERP server.
+
+* Avoid having multiple DERP nodes in a region. If you must, they all need to be
+  meshed with each other and monitored. Having two one-node "regions" in the
+  same datacenter is usually easier and more reliable than meshing, at the cost
+  of more required connections from clients in some cases. If your clients
+  aren't mobile (battery constrained), one node regions are definitely
+  preferred. If you really need multiple nodes in a region for HA reasons, two
+  is sufficient.
+
+* Monitor your DERP servers with [`cmd/derpprobe`](../derpprobe/).
+
+* If using `--verify-clients`, a `tailscaled` must be running alongside the
+  `derper`, and all clients must be visible to the derper tailscaled in the ACL.
+
+* If using `--verify-clients`, a `tailscaled` must also be running alongside
+  your `derpprobe`, and `derpprobe` needs to use `--derp-map=local`.
+
+* The firewall on the `derper` should permit TCP ports 80 and 443 and UDP port
+  3478.
+
+* Only LetsEncrypt certs are rotated automatically. Other cert updates require a
+  restart.
+
+* Don't use a firewall in front of `derper` that suppresses `RST`s upon
+  receiving traffic to a dead or unknown connection.
+
+* Don't rate-limit UDP STUN packets.
+
+* Don't rate-limit outbound TCP traffic (only inbound).
+
+## Diagnostics
+
+This is not a complete guide on DERP diagnostics.
+
+Running your own DERP services requires exeprtise in multi-layer network and
+application diagnostics. As the DERP runs multiple protocols at multiple layers
+and is not a regular HTTP(s) server you will need expertise in correlative
+analysis to diagnose the most tricky problems. There is no "plain text" or
+"open" mode of operation for DERP.
+
+* The debug handler is accessible at URL path `/debug/`. It is only accessible
+  over localhost or from a Tailscale IP address.
+
+* Go pprof can be accessed via the debug handler at `/debug/pprof/`
+
+* Prometheus compatible metrics can be gathered from the debug handler at
+  `/debug/varz`.
+
+* `cmd/stunc` in the Tailscale repository provides a basic tool for diagnosing
+  issues with STUN.
+
+* `cmd/derpprobe` provides a service for monitoring DERP cluster health.
+
+* `tailscale debug derp` and `tailscale netcheck` provide additional client
+  driven diagnostic information for DERP communications.
+
+* Tailscale logs may provide insight for certain problems, such as if DERPs are
+  unreachable or peers are regularly not reachable in their DERP home regions.
+  There are many possible misconfiguration causes for these problems, but
+  regular log entries are a good first indicator that there is a problem.

cmd/derper/bootstrap_dns.go

@@ -5,35 +5,45 @@ package main
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"expvar"
 	"log"
+	"math/rand/v2"
 	"net"
 	"net/http"
+	"net/netip"
+	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/syncs"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/slicesx"
 )
 
 const refreshTimeout = time.Minute
 
-type dnsEntryMap map[string][]net.IP
+type dnsEntryMap struct {
+	IPs     map[string][]net.IP
+	Percent map[string]float64 // "foo.com" => 0.5 for 50%
+}
 
 var (
-	dnsCache            syncs.AtomicValue[dnsEntryMap]
+	dnsCache            atomic.Pointer[dnsEntryMap]
 	dnsCacheBytes       syncs.AtomicValue[[]byte] // of JSON
-	unpublishedDNSCache syncs.AtomicValue[dnsEntryMap]
+	unpublishedDNSCache atomic.Pointer[dnsEntryMap]
 	bootstrapLookupMap  syncs.Map[string, bool]
 )
 
 var (
-	bootstrapDNSRequests = expvar.NewInt("counter_bootstrap_dns_requests")
-	publishedDNSHits     = expvar.NewInt("counter_bootstrap_dns_published_hits")
-	publishedDNSMisses   = expvar.NewInt("counter_bootstrap_dns_published_misses")
-	unpublishedDNSHits   = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
-	unpublishedDNSMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+	bootstrapDNSRequests        = expvar.NewInt("counter_bootstrap_dns_requests")
+	publishedDNSHits            = expvar.NewInt("counter_bootstrap_dns_published_hits")
+	publishedDNSMisses          = expvar.NewInt("counter_bootstrap_dns_published_misses")
+	unpublishedDNSHits          = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
+	unpublishedDNSMisses        = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+	unpublishedDNSPercentMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_percent_misses")
 )
 
 func init() {
@@ -59,15 +69,13 @@ func refreshBootstrapDNS() {
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-	dnsEntries := resolveList(ctx, strings.Split(*bootstrapDNS, ","))
+	dnsEntries := resolveList(ctx, *bootstrapDNS)
 	// Randomize the order of the IPs for each name to avoid the client biasing
 	// to IPv6
-	for k := range dnsEntries {
-		ips := dnsEntries[k]
-		slicesx.Shuffle(ips)
-		dnsEntries[k] = ips
+	for _, vv := range dnsEntries.IPs {
+		slicesx.Shuffle(vv)
 	}
-	j, err := json.MarshalIndent(dnsEntries, "", "\t")
+	j, err := json.MarshalIndent(dnsEntries.IPs, "", "\t")
 	if err != nil {
 		// leave the old values in place
 		return
@@ -81,27 +89,50 @@ func refreshUnpublishedDNS() {
 	if *unpublishedDNS == "" {
 		return
 	}
-
 	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-
-	dnsEntries := resolveList(ctx, strings.Split(*unpublishedDNS, ","))
+	dnsEntries := resolveList(ctx, *unpublishedDNS)
 	unpublishedDNSCache.Store(dnsEntries)
 }
 
-func resolveList(ctx context.Context, names []string) dnsEntryMap {
-	dnsEntries := make(dnsEntryMap)
+// resolveList takes a comma-separated list of DNS names to resolve.
+//
+// If an entry contains a slash, it's two DNS names: the first is the one to
+// resolve and the second is that of a TXT recording containing the rollout
+// percentage in range "0".."100". If the TXT record doesn't exist or is
+// malformed, the percentage is 0. If the TXT record is not provided (there's no
+// slash), then the percentage is 100.
+func resolveList(ctx context.Context, list string) *dnsEntryMap {
+	ents := strings.Split(list, ",")
+
+	ret := &dnsEntryMap{}
 
 	var r net.Resolver
-	for _, name := range names {
+	for _, ent := range ents {
+		name, txtName, _ := strings.Cut(ent, "/")
 		addrs, err := r.LookupIP(ctx, "ip", name)
 		if err != nil {
 			log.Printf("bootstrap DNS lookup %q: %v", name, err)
 			continue
 		}
-		dnsEntries[name] = addrs
+		mak.Set(&ret.IPs, name, addrs)
+
+		if txtName == "" {
+			mak.Set(&ret.Percent, name, 1.0)
+			continue
+		}
+		vals, err := r.LookupTXT(ctx, txtName)
+		if err != nil {
+			log.Printf("bootstrap DNS lookup %q: %v", txtName, err)
+			continue
+		}
+		for _, v := range vals {
+			if v, err := strconv.Atoi(v); err == nil && v >= 0 && v <= 100 {
+				mak.Set(&ret.Percent, name, float64(v)/100)
+			}
+		}
 	}
-	return dnsEntries
+	return ret
 }
 
 func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
@@ -115,22 +146,36 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	// Try answering a query from our hidden map first
 	if q := r.URL.Query().Get("q"); q != "" {
 		bootstrapLookupMap.Store(q, true)
-		if ips, ok := unpublishedDNSCache.Load()[q]; ok && len(ips) > 0 {
+		if bootstrapLookupMap.Len() > 500 { // defensive
+			bootstrapLookupMap.Clear()
+		}
+		if m := unpublishedDNSCache.Load(); m != nil && len(m.IPs[q]) > 0 {
 			unpublishedDNSHits.Add(1)
 
-			// Only return the specific query, not everything.
-			m := dnsEntryMap{q: ips}
-			j, err := json.MarshalIndent(m, "", "\t")
-			if err == nil {
-				w.Write(j)
-				return
+			percent := m.Percent[q]
+			if remoteAddrMatchesPercent(r.RemoteAddr, percent) {
+				// Only return the specific query, not everything.
+				m := map[string][]net.IP{q: m.IPs[q]}
+				j, err := json.MarshalIndent(m, "", "\t")
+				if err == nil {
+					w.Write(j)
+					return
+				}
+			} else {
+				unpublishedDNSPercentMisses.Add(1)
 			}
 		}
 
 		// If we have a "q" query for a name in the published cache
 		// list, then track whether that's a hit/miss.
-		if m, ok := dnsCache.Load()[q]; ok {
-			if len(m) > 0 {
+		m := dnsCache.Load()
+		var inPub bool
+		var ips []net.IP
+		if m != nil {
+			ips, inPub = m.IPs[q]
+		}
+		if inPub {
+			if len(ips) > 0 {
 				publishedDNSHits.Add(1)
 			} else {
 				publishedDNSMisses.Add(1)
@@ -146,3 +191,29 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	j := dnsCacheBytes.Load()
 	w.Write(j)
 }
+
+// percent is [0.0, 1.0].
+func remoteAddrMatchesPercent(remoteAddr string, percent float64) bool {
+	if percent == 0 {
+		return false
+	}
+	if percent == 1 {
+		return true
+	}
+	reqIPStr, _, err := net.SplitHostPort(remoteAddr)
+	if err != nil {
+		return false
+	}
+	reqIP, err := netip.ParseAddr(reqIPStr)
+	if err != nil {
+		return false
+	}
+	if reqIP.IsLoopback() {
+		// For local testing.
+		return rand.Float64() < 0.5
+	}
+	reqIP16 := reqIP.As16()
+	rndSrc := rand.NewPCG(binary.LittleEndian.Uint64(reqIP16[:8]), binary.LittleEndian.Uint64(reqIP16[8:]))
+	rnd := rand.New(rndSrc)
+	return percent > rnd.Float64()
+}

cmd/derper/bootstrap_dns_test.go

@@ -4,15 +4,19 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/netip"
 	"net/url"
 	"reflect"
 	"testing"
 
 	"tailscale.com/tstest"
+	"tailscale.com/tstest/nettest"
 )
 
 func BenchmarkHandleBootstrapDNS(b *testing.B) {
@@ -37,7 +41,7 @@ func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p),
 
 func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {}
 
-func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
+func getBootstrapDNS(t *testing.T, q string) map[string][]net.IP {
 	t.Helper()
 	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape(q), nil)
 	w := httptest.NewRecorder()
@@ -47,14 +51,17 @@ func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
 	if res.StatusCode != 200 {
 		t.Fatalf("got status=%d; want %d", res.StatusCode, 200)
 	}
-	var ips dnsEntryMap
-	if err := json.NewDecoder(res.Body).Decode(&ips); err != nil {
-		t.Fatalf("error decoding response body: %v", err)
+	var m map[string][]net.IP
+	var buf bytes.Buffer
+	if err := json.NewDecoder(io.TeeReader(res.Body, &buf)).Decode(&m); err != nil {
+		t.Fatalf("error decoding response body %q: %v", buf.Bytes(), err)
 	}
-	return ips
+	return m
 }
 
 func TestUnpublishedDNS(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
+
 	const published = "login.tailscale.com"
 	const unpublished = "log.tailscale.io"
 
@@ -104,15 +111,21 @@ func resetMetrics() {
 // Verify that we don't count an empty list in the unpublishedDNSCache as a
 // cache hit in our metrics.
 func TestUnpublishedDNSEmptyList(t *testing.T) {
-	pub := dnsEntryMap{
-		"tailscale.com": {net.IPv4(10, 10, 10, 10)},
+	pub := &dnsEntryMap{
+		IPs: map[string][]net.IP{"tailscale.com": {net.IPv4(10, 10, 10, 10)}},
 	}
 	dnsCache.Store(pub)
 	dnsCacheBytes.Store([]byte(`{"tailscale.com":["10.10.10.10"]}`))
 
-	unpublishedDNSCache.Store(dnsEntryMap{
-		"log.tailscale.io":           {},
-		"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+	unpublishedDNSCache.Store(&dnsEntryMap{
+		IPs: map[string][]net.IP{
+			"log.tailscale.io":           {},
+			"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+		},
+		Percent: map[string]float64{
+			"log.tailscale.io":           1.0,
+			"controlplane.tailscale.com": 1.0,
+		},
 	})
 
 	t.Run("CacheMiss", func(t *testing.T) {
@@ -122,8 +135,8 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 			ips := getBootstrapDNS(t, q)
 
 			// Expected our public map to be returned on a cache miss
-			if !reflect.DeepEqual(ips, pub) {
-				t.Errorf("got ips=%+v; want %+v", ips, pub)
+			if !reflect.DeepEqual(ips, pub.IPs) {
+				t.Errorf("got ips=%+v; want %+v", ips, pub.IPs)
 			}
 			if v := unpublishedDNSHits.Value(); v != 0 {
 				t.Errorf("got hits=%d; want 0", v)
@@ -138,7 +151,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 	t.Run("CacheHit", func(t *testing.T) {
 		resetMetrics()
 		ips := getBootstrapDNS(t, "controlplane.tailscale.com")
-		want := dnsEntryMap{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
+		want := map[string][]net.IP{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
 		if !reflect.DeepEqual(ips, want) {
 			t.Errorf("got ips=%+v; want %+v", ips, want)
 		}
@@ -163,3 +176,54 @@ func TestLookupMetric(t *testing.T) {
 		t.Errorf("bootstrapLookupMap.Len() want=5, got %v", bootstrapLookupMap.Len())
 	}
 }
+
+func TestRemoteAddrMatchesPercent(t *testing.T) {
+	tests := []struct {
+		remoteAddr string
+		percent    float64
+		want       bool
+	}{
+		// 0% and 100%.
+		{"10.0.0.1:1234", 0.0, false},
+		{"10.0.0.1:1234", 1.0, true},
+
+		// Invalid IP.
+		{"", 1.0, true},
+		{"", 0.0, false},
+		{"", 0.5, false},
+
+		// Small manual sample at 50%. The func uses a deterministic PRNG seed.
+		{"1.2.3.4:567", 0.5, true},
+		{"1.2.3.5:567", 0.5, true},
+		{"1.2.3.6:567", 0.5, false},
+		{"1.2.3.7:567", 0.5, true},
+		{"1.2.3.8:567", 0.5, false},
+		{"1.2.3.9:567", 0.5, true},
+		{"1.2.3.10:567", 0.5, true},
+	}
+	for _, tt := range tests {
+		got := remoteAddrMatchesPercent(tt.remoteAddr, tt.percent)
+		if got != tt.want {
+			t.Errorf("remoteAddrMatchesPercent(%q, %v) = %v; want %v", tt.remoteAddr, tt.percent, got, tt.want)
+		}
+	}
+
+	var match, all int
+	const wantPercent = 0.5
+	for a := range 256 {
+		for b := range 256 {
+			all++
+			if remoteAddrMatchesPercent(
+				netip.AddrPortFrom(netip.AddrFrom4([4]byte{1, 2, byte(a), byte(b)}), 12345).String(),
+				wantPercent) {
+				match++
+			}
+		}
+	}
+	gotPercent := float64(match) / float64(all)
+	const tolerance = 0.005
+	t.Logf("got percent %v (goal %v)", gotPercent, wantPercent)
+	if gotPercent < wantPercent-tolerance || gotPercent > wantPercent+tolerance {
+		t.Errorf("got %v; want %v ± %v", gotPercent, wantPercent, tolerance)
+	}
+}

cmd/derper/depaware.txt

@@ -10,6 +10,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
    L    github.com/google/nftables                                   from tailscale.com/util/linuxfw
    L 💣 github.com/google/nftables/alignedbuff                       from github.com/google/nftables/xt
@@ -17,10 +23,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-        github.com/google/uuid                                       from tailscale.com/tsweb
+        github.com/google/uuid                                       from tailscale.com/util/fastuuid
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
-   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/interfaces+
+   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
    L 💣 github.com/mdlayher/netlink                                  from github.com/google/nftables+
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
@@ -46,14 +52,15 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
-        go4.org/netipx                                               from tailscale.com/net/tsaddr+
-   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
+        go4.org/netipx                                               from tailscale.com/net/tsaddr
+   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         google.golang.org/protobuf/encoding/protodelim               from github.com/prometheus/common/expfmt
         google.golang.org/protobuf/encoding/prototext                from github.com/prometheus/common/expfmt+
         google.golang.org/protobuf/encoding/protowire                from google.golang.org/protobuf/encoding/protodelim+
         google.golang.org/protobuf/internal/descfmt                  from google.golang.org/protobuf/internal/filedesc
         google.golang.org/protobuf/internal/descopts                 from google.golang.org/protobuf/internal/filedesc+
         google.golang.org/protobuf/internal/detrand                  from google.golang.org/protobuf/internal/descfmt+
+        google.golang.org/protobuf/internal/editiondefaults          from google.golang.org/protobuf/internal/filedesc
         google.golang.org/protobuf/internal/encoding/defval          from google.golang.org/protobuf/internal/encoding/tag+
         google.golang.org/protobuf/internal/encoding/messageset      from google.golang.org/protobuf/encoding/prototext+
         google.golang.org/protobuf/internal/encoding/tag             from google.golang.org/protobuf/internal/impl
@@ -86,22 +93,20 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/derp                                           from tailscale.com/cmd/derper+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/derper
         tailscale.com/disco                                          from tailscale.com/derp
+        tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/health                                         from tailscale.com/net/tlsdial
-        tailscale.com/hostinfo                                       from tailscale.com/net/interfaces+
+        tailscale.com/health                                         from tailscale.com/net/tlsdial+
+        tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
-        tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
-     💣 tailscale.com/net/interfaces                                 from tailscale.com/net/netmon+
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
-        tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
-        tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
+     💣 tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
+     💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale
-        tailscale.com/net/packet                                     from tailscale.com/wgengine/filter
         tailscale.com/net/sockstats                                  from tailscale.com/derp/derphttp
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
@@ -114,18 +119,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
-        tailscale.com/tailfs                                         from tailscale.com/client/tailscale+
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
-        tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tstime/rate                                    from tailscale.com/derp
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
-        tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
+        tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
         tailscale.com/types/lazy                                     from tailscale.com/version+
         tailscale.com/types/logger                                   from tailscale.com/cmd/derper+
@@ -137,16 +141,18 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/ipn+
+        tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/net/netmon+
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
+        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
-        tailscale.com/util/mak                                       from tailscale.com/net/interfaces+
+        tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
         tailscale.com/util/set                                       from tailscale.com/derp+
@@ -155,9 +161,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
+   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
-        tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
+        tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap
         golang.org/x/crypto/acme                                     from golang.org/x/crypto/acme/autocert
         golang.org/x/crypto/acme/autocert                            from tailscale.com/cmd/derper
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
@@ -172,6 +179,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+   W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -232,7 +240,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper+
+        flag                                                         from tailscale.com/cmd/derper
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -250,6 +258,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
+        math/rand/v2                                                 from tailscale.com/util/fastuuid+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -273,7 +282,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         runtime/debug                                                from github.com/prometheus/client_golang/prometheus+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof
-        runtime/trace                                                from net/http/pprof+
+        runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/ipn/ipnstate+
         sort                                                         from compress/flate+
         strconv                                                      from compress/flate+
@@ -281,7 +290,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync                                                         from compress/flate+
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from runtime/pprof
         time                                                         from compress/gzip+
         unicode                                                      from bytes+

cmd/derper/derper.go

@@ -2,6 +2,12 @@
 // SPDX-License-Identifier: BSD-3-Clause
 
 // The derper binary is a simple DERP server.
+//
+// For more information, see:
+//
+//   - About: https://tailscale.com/kb/1232/derp-servers
+//   - Protocol & Go docs: https://pkg.go.dev/tailscale.com/derp
+//   - Running a DERP server: https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp
 package main // import "tailscale.com/cmd/derper"
 
 import (
@@ -22,6 +28,9 @@ import (
 	"os/signal"
 	"path/filepath"
 	"regexp"
+	"runtime"
+	runtimemetrics "runtime/metrics"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -36,24 +45,26 @@ import (
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/version"
 )
 
 var (
-	dev        = flag.Bool("dev", false, "run in localhost development mode (overrides -a)")
-	addr       = flag.String("a", ":443", "server HTTP/HTTPS listen address, in form \":port\", \"ip:port\", or for IPv6 \"[ip]:port\". If the IP is omitted, it defaults to all interfaces. Serves HTTPS if the port is 443 and/or -certmode is manual, otherwise HTTP.")
-	httpPort   = flag.Int("http-port", 80, "The port on which to serve HTTP. Set to -1 to disable. The listener is bound to the same IP (if any) as specified in the -a flag.")
-	stunPort   = flag.Int("stun-port", 3478, "The UDP port on which to serve STUN. The listener is bound to the same IP (if any) as specified in the -a flag.")
-	configPath = flag.String("c", "", "config file path")
-	certMode   = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
-	certDir    = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
-	hostname   = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
-	runSTUN    = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
-	runDERP    = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
+	dev         = flag.Bool("dev", false, "run in localhost development mode (overrides -a)")
+	versionFlag = flag.Bool("version", false, "print version and exit")
+	addr        = flag.String("a", ":443", "server HTTP/HTTPS listen address, in form \":port\", \"ip:port\", or for IPv6 \"[ip]:port\". If the IP is omitted, it defaults to all interfaces. Serves HTTPS if the port is 443 and/or -certmode is manual, otherwise HTTP.")
+	httpPort    = flag.Int("http-port", 80, "The port on which to serve HTTP. Set to -1 to disable. The listener is bound to the same IP (if any) as specified in the -a flag.")
+	stunPort    = flag.Int("stun-port", 3478, "The UDP port on which to serve STUN. The listener is bound to the same IP (if any) as specified in the -a flag.")
+	configPath  = flag.String("c", "", "config file path")
+	certMode    = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
+	certDir     = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
+	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
+	runSTUN     = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
+	runDERP     = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
 
 	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 	bootstrapDNS    = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
-	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list")
+	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
 	verifyClients   = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 	verifyClientURL = flag.String("verify-client-url", "", "if non-empty, an admission controller URL for permitting client connections; see tailcfg.DERPAdmitClientRequest")
 	verifyFailOpen  = flag.Bool("verify-client-url-fail-open", true, "whether we fail open if --verify-client-url is unreachable")
@@ -129,6 +140,10 @@ func writeNewConfig() config {
 
 func main() {
 	flag.Parse()
+	if *versionFlag {
+		fmt.Println(version.Long())
+		return
+	}
 
 	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer cancel()
@@ -185,7 +200,12 @@ func main() {
 			http.Error(w, "derp server disabled", http.StatusNotFound)
 		}))
 	}
-	mux.HandleFunc("/derp/probe", probeHandler)
+
+	// These two endpoints are the same. Different versions of the clients
+	// have assumes different paths over time so we support both.
+	mux.HandleFunc("/derp/probe", derphttp.ProbeHandler)
+	mux.HandleFunc("/derp/latency-check", derphttp.ProbeHandler)
+
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", tsweb.BrowserHeaderHandlerFunc(handleBootstrapDNS))
 	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -195,11 +215,16 @@ func main() {
 		io.WriteString(w, `<html><body>
 <h1>DERP</h1>
 <p>
-  This is a
-  <a href="https://tailscale.com/">Tailscale</a>
-  <a href="https://pkg.go.dev/tailscale.com/derp">DERP</a>
-  server.
+  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
 </p>
+<p>
+  Documentation:
+</p>
+<ul>
+  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
+  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
+  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
+</ul>
 `)
 		if !*runDERP {
 			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
@@ -225,6 +250,20 @@ func main() {
 		}
 	}))
 	debug.Handle("traffic", "Traffic check", http.HandlerFunc(s.ServeDebugTraffic))
+	debug.Handle("set-mutex-profile-fraction", "SetMutexProfileFraction", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		s := r.FormValue("rate")
+		if s == "" || r.Header.Get("Sec-Debug") != "derp" {
+			http.Error(w, "To set, use: curl -HSec-Debug:derp 'http://derp/debug/set-mutex-profile-fraction?rate=100'", http.StatusBadRequest)
+			return
+		}
+		v, err := strconv.Atoi(s)
+		if err != nil {
+			http.Error(w, "bad rate value", http.StatusBadRequest)
+			return
+		}
+		old := runtime.SetMutexProfileFraction(v)
+		fmt.Fprintf(w, "mutex changed from %v to %v\n", old, v)
+	}))
 
 	// Longer lived DERP connections send an application layer keepalive. Note
 	// if the keepalive is hit, the user timeout will take precedence over the
@@ -364,17 +403,6 @@ func isChallengeChar(c rune) bool {
 		c == '.' || c == '-' || c == '_'
 }
 
-// probeHandler is the endpoint that js/wasm clients hit to measure
-// DERP latency, since they can't do UDP STUN queries.
-func probeHandler(w http.ResponseWriter, r *http.Request) {
-	switch r.Method {
-	case "HEAD", "GET":
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-	default:
-		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
-	}
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {
@@ -452,3 +480,16 @@ func (l *rateLimitedListener) Accept() (net.Conn, error) {
 	l.numAccepts.Add(1)
 	return cn, nil
 }
+
+func init() {
+	expvar.Publish("go_sync_mutex_wait_seconds", expvar.Func(func() any {
+		const name = "/sync/mutex/wait/total:seconds" // Go 1.20+
+		var s [1]runtimemetrics.Sample
+		s[0].Name = name
+		runtimemetrics.Read(s[:])
+		if v := s[0].Value; v.Kind() == runtimemetrics.KindFloat64 {
+			return v.Float64()
+		}
+		return 0
+	}))
+}

cmd/derper/derper_test.go

@@ -99,10 +99,13 @@ func TestNoContent(t *testing.T) {
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
+			"testing":                            "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/buffer":       "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
+			"tailscale.com/net/packet":           "not needed in derper",
+			"github.com/gaissmai/bart":           "not needed in derper",
 		},
 	}.Check(t)
 }

cmd/derper/mesh.go

@@ -9,13 +9,12 @@ import (
 	"fmt"
 	"log"
 	"net"
-	"net/netip"
 	"strings"
 	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
-	"tailscale.com/types/key"
+	"tailscale.com/net/netmon"
 	"tailscale.com/types/logger"
 )
 
@@ -36,7 +35,8 @@ func startMesh(s *derp.Server) error {
 
 func startMeshWithHost(s *derp.Server, host string) error {
 	logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
-	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf)
+	netMon := netmon.NewStatic() // good enough for cmd/derper; no need for netns fanciness
+	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf, netMon)
 	if err != nil {
 		return err
 	}
@@ -69,8 +69,8 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return d.DialContext(ctx, network, addr)
 	})
 
-	add := func(k key.NodePublic, _ netip.AddrPort) { s.AddPacketForwarder(k, c) }
-	remove := func(k key.NodePublic) { s.RemovePacketForwarder(k, c) }
+	add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) }
+	remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)
 	return nil
 }

cmd/derpprobe/derpprobe.go

@@ -7,8 +7,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"html"
-	"io"
 	"log"
 	"net/http"
 	"sort"
@@ -16,10 +14,12 @@ import (
 
 	"tailscale.com/prober"
 	"tailscale.com/tsweb"
+	"tailscale.com/version"
 )
 
 var (
-	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")
+	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
+	versionFlag  = flag.Bool("version", false, "print version and exit")
 	listen       = flag.String("listen", ":8030", "HTTP listen address")
 	probeOnce    = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
 	spread       = flag.Bool("spread", true, "whether to spread probing over time")
@@ -33,6 +33,10 @@ var (
 
 func main() {
 	flag.Parse()
+	if *versionFlag {
+		fmt.Println(version.Long())
+		return
+	}
 
 	p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe")
 	opts := []prober.DERPOpt{
@@ -64,8 +68,13 @@ func main() {
 	}
 
 	mux := http.NewServeMux()
-	tsweb.Debugger(mux)
-	mux.HandleFunc("/", http.HandlerFunc(serveFunc(p)))
+	d := tsweb.Debugger(mux)
+	d.Handle("probe-run", "Run a probe", tsweb.StdHandler(tsweb.ReturnHandlerFunc(p.RunHandler), tsweb.HandlerOptions{Logf: log.Printf}))
+	mux.Handle("/", tsweb.StdHandler(p.StatusHandler(
+		prober.WithTitle("DERP Prober"),
+		prober.WithPageLink("Prober metrics", "/debug/varz"),
+		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
+	), tsweb.HandlerOptions{Logf: log.Printf}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }
@@ -99,26 +108,3 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 	sort.Strings(o.good)
 	return
 }
-
-func serveFunc(p *prober.Prober) func(w http.ResponseWriter, r *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		st := getOverallStatus(p)
-		summary := "All good"
-		if (float64(len(st.bad)) / float64(len(st.bad)+len(st.good))) > 0.25 {
-			// Returning a 500 allows monitoring this server externally and configuring
-			// an alert on HTTP response code.
-			w.WriteHeader(500)
-			summary = fmt.Sprintf("%d problems", len(st.bad))
-		}
-
-		io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
-		fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
-		for _, s := range st.bad {
-			fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
-		}
-		for _, s := range st.good {
-			fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
-		}
-		io.WriteString(w, "</ul></body></html>\n")
-	}
-}

cmd/dist/dist.go

@@ -13,11 +13,16 @@ import (
 
 	"tailscale.com/release/dist"
 	"tailscale.com/release/dist/cli"
+	"tailscale.com/release/dist/qnap"
 	"tailscale.com/release/dist/synology"
 	"tailscale.com/release/dist/unixpkgs"
 )
 
-var synologyPackageCenter bool
+var (
+	synologyPackageCenter bool
+	qnapPrivateKeyPath    string
+	qnapCertificatePath   string
+)
 
 func getTargets() ([]dist.Target, error) {
 	var ret []dist.Target
@@ -37,6 +42,10 @@ func getTargets() ([]dist.Target, error) {
 	// To build for package center, run
 	// ./tool/go run ./cmd/dist build --synology-package-center synology
 	ret = append(ret, synology.Targets(synologyPackageCenter, nil)...)
+	if (qnapPrivateKeyPath == "") != (qnapCertificatePath == "") {
+		return nil, errors.New("both --qnap-private-key-path and --qnap-certificate-path must be set")
+	}
+	ret = append(ret, qnap.Targets(qnapPrivateKeyPath, qnapCertificatePath)...)
 	return ret, nil
 }
 
@@ -45,6 +54,8 @@ func main() {
 	for _, subcmd := range cmd.Subcommands {
 		if subcmd.Name == "build" {
 			subcmd.FlagSet.BoolVar(&synologyPackageCenter, "synology-package-center", false, "build synology packages with extra metadata for the official package center")
+			subcmd.FlagSet.StringVar(&qnapPrivateKeyPath, "qnap-private-key-path", "", "sign qnap packages with given key (must also provide --qnap-certificate-path)")
+			subcmd.FlagSet.StringVar(&qnapCertificatePath, "qnap-certificate-path", "", "sign qnap packages with given certificate (must also provide --qnap-private-key-path)")
 		}
 	}
 

cmd/k8s-nameserver/main.go

@@ -0,0 +1,379 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// k8s-nameserver is a simple nameserver implementation meant to be used with
+// k8s-operator to allow to resolve magicDNS names associated with tailnet
+// proxies in cluster.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/miekg/dns"
+	operatorutils "tailscale.com/k8s-operator"
+	"tailscale.com/util/dnsname"
+)
+
+const (
+	// tsNetDomain is the domain that this DNS nameserver has registered a handler for.
+	tsNetDomain = "ts.net"
+	// addr is the the address that the UDP and TCP listeners will listen on.
+	addr = ":1053"
+
+	// The following constants are specific to the nameserver configuration
+	// provided by a mounted Kubernetes Configmap. The Configmap mounted at
+	// /config is the only supported way for configuring this nameserver.
+	defaultDNSConfigDir    = "/config"
+	kubeletMountedConfigLn = "..data"
+)
+
+// nameserver is a simple nameserver that responds to DNS queries for A records
+// for ts.net domain names over UDP or TCP. It serves DNS responses from
+// in-memory IPv4 host records. It is intended to be deployed on Kubernetes with
+// a ConfigMap mounted at /config that should contain the host records. It
+// dynamically reconfigures its in-memory mappings as the contents of the
+// mounted ConfigMap changes.
+type nameserver struct {
+	// configReader returns the latest desired configuration (host records)
+	// for the nameserver. By default it gets set to a reader that reads
+	// from a Kubernetes ConfigMap mounted at /config, but this can be
+	// overridden in tests.
+	configReader configReaderFunc
+	// configWatcher is a watcher that returns an event when the desired
+	// configuration has changed and the nameserver should update the
+	// in-memory records.
+	configWatcher <-chan string
+
+	mu sync.Mutex // protects following
+	// ip4 are the in-memory hostname -> IP4 mappings that the nameserver
+	// uses to respond to A record queries.
+	ip4 map[dnsname.FQDN][]net.IP
+}
+
+func main() {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Ensure that we watch the kube Configmap mounted at /config for
+	// nameserver configuration updates and send events when updates happen.
+	c := ensureWatcherForKubeConfigMap(ctx)
+
+	ns := &nameserver{
+		configReader:  configMapConfigReader,
+		configWatcher: c,
+	}
+
+	// Ensure that in-memory records get set up to date now and will get
+	// reset when the configuration changes.
+	ns.runRecordsReconciler(ctx)
+
+	// Register a DNS server handle for ts.net domain names. Not having a
+	// handle registered for any other domain names is how we enforce that
+	// this nameserver can only be used for ts.net domains - querying any
+	// other domain names returns Rcode Refused.
+	dns.HandleFunc(tsNetDomain, ns.handleFunc())
+
+	// Listen for DNS queries over UDP and TCP.
+	udpSig := make(chan os.Signal)
+	tcpSig := make(chan os.Signal)
+	go listenAndServe("udp", addr, udpSig)
+	go listenAndServe("tcp", addr, tcpSig)
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
+	s := <-sig
+	log.Printf("OS signal (%s) received, shutting down", s)
+	cancel()    // exit the records reconciler and configmap watcher goroutines
+	udpSig <- s // stop the UDP listener
+	tcpSig <- s // stop the TCP listener
+}
+
+// handleFunc is a DNS query handler that can respond to A record queries from
+// the nameserver's in-memory records.
+// - If an A record query is received and the
+// nameserver's in-memory records contain records for the queried domain name,
+// return a success response.
+// - If an A record query is received, but the
+// nameserver's in-memory records do not contain records for the queried domain name,
+// return NXDOMAIN.
+// - If an A record query is received, but the queried domain name is not valid, return Format Error.
+// - If a query is received for any other record type than A, return Not Implemented.
+func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) {
+	h := func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		defer func() {
+			w.WriteMsg(m)
+		}()
+		if len(r.Question) < 1 {
+			log.Print("[unexpected] nameserver received a request with no questions")
+			m = r.SetRcodeFormatError(r)
+			return
+		}
+		// TODO (irbekrm): maybe set message compression
+		switch r.Question[0].Qtype {
+		case dns.TypeA:
+			q := r.Question[0].Name
+			fqdn, err := dnsname.ToFQDN(q)
+			if err != nil {
+				m = r.SetRcodeFormatError(r)
+				return
+			}
+			// The only supported use of this nameserver is as a
+			// single source of truth for MagicDNS names by
+			// non-tailnet Kubernetes workloads.
+			m.Authoritative = true
+			m.RecursionAvailable = false
+
+			ips := n.lookupIP4(fqdn)
+			if ips == nil || len(ips) == 0 {
+				// As we are the authoritative nameserver for MagicDNS
+				// names, if we do not have a record for this MagicDNS
+				// name, it does not exist.
+				m = m.SetRcode(r, dns.RcodeNameError)
+				return
+			}
+			// TODO (irbekrm): TTL is currently set to 0, meaning
+			// that cluster workloads will not cache the DNS
+			// records. Revisit this in future when we understand
+			// the usage patterns better- is it putting too much
+			// load on kube DNS server or is this fine?
+			for _, ip := range ips {
+				rr := &dns.A{Hdr: dns.RR_Header{Name: q, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0}, A: ip}
+				m.SetRcode(r, dns.RcodeSuccess)
+				m.Answer = append(m.Answer, rr)
+			}
+		case dns.TypeAAAA:
+			// TODO (irbekrm): add IPv6 support.
+			// The nameserver currently does not support IPv6
+			// (records are not being created for IPv6 Pod addresses).
+			// However, we can expect that some callers will
+			// nevertheless send AAAA queries.
+			// We have to return NOERROR if a query is received for
+			// an AAAA record for a DNS name that we have an A
+			// record for- else the caller might not follow with an
+			// A record query.
+			// https://github.com/tailscale/tailscale/issues/12321
+			// https://datatracker.ietf.org/doc/html/rfc4074
+			q := r.Question[0].Name
+			fqdn, err := dnsname.ToFQDN(q)
+			if err != nil {
+				m = r.SetRcodeFormatError(r)
+				return
+			}
+			// The only supported use of this nameserver is as a
+			// single source of truth for MagicDNS names by
+			// non-tailnet Kubernetes workloads.
+			m.Authoritative = true
+			ips := n.lookupIP4(fqdn)
+			if len(ips) == 0 {
+				// As we are the authoritative nameserver for MagicDNS
+				// names, if we do not have a record for this MagicDNS
+				// name, it does not exist.
+				m = m.SetRcode(r, dns.RcodeNameError)
+				return
+			}
+			m.SetRcode(r, dns.RcodeSuccess)
+		default:
+			log.Printf("[unexpected] nameserver received a query for an unsupported record type: %s", r.Question[0].String())
+			m.SetRcode(r, dns.RcodeNotImplemented)
+		}
+	}
+	return h
+}
+
+// runRecordsReconciler ensures that nameserver's in-memory records are
+// reset when the provided configuration changes.
+func (n *nameserver) runRecordsReconciler(ctx context.Context) {
+	log.Print("updating nameserver's records from the provided configuration...")
+	if err := n.resetRecords(); err != nil { // ensure records are up to date before the nameserver starts
+		log.Fatalf("error setting nameserver's records: %v", err)
+	}
+	log.Print("nameserver's records were updated")
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				log.Printf("context cancelled, exiting records reconciler")
+				return
+			case <-n.configWatcher:
+				log.Print("configuration update detected, resetting records")
+				if err := n.resetRecords(); err != nil {
+					// TODO (irbekrm): this runs in a
+					// container that will be thrown away,
+					// so this should be ok. But maybe still
+					// need to ensure that the DNS server
+					// terminates connections more
+					// gracefully.
+					log.Fatalf("error resetting records: %v", err)
+				}
+				log.Print("nameserver records were reset")
+			}
+		}
+	}()
+}
+
+// resetRecords sets the in-memory DNS records of this nameserver from the
+// provided configuration. It does not check for the diff, so the caller is
+// expected to ensure that this is only called when reset is needed.
+func (n *nameserver) resetRecords() error {
+	dnsCfgBytes, err := n.configReader()
+	if err != nil {
+		log.Printf("error reading nameserver's configuration: %v", err)
+		return err
+	}
+	if dnsCfgBytes == nil || len(dnsCfgBytes) < 1 {
+		log.Print("nameserver's configuration is empty, any in-memory records will be unset")
+		n.mu.Lock()
+		n.ip4 = make(map[dnsname.FQDN][]net.IP)
+		n.mu.Unlock()
+		return nil
+	}
+	dnsCfg := &operatorutils.Records{}
+	err = json.Unmarshal(dnsCfgBytes, dnsCfg)
+	if err != nil {
+		return fmt.Errorf("error unmarshalling nameserver configuration: %v\n", err)
+	}
+
+	if dnsCfg.Version != operatorutils.Alpha1Version {
+		return fmt.Errorf("unsupported configuration version %s, supported versions are %s\n", dnsCfg.Version, operatorutils.Alpha1Version)
+	}
+
+	ip4 := make(map[dnsname.FQDN][]net.IP)
+	defer func() {
+		n.mu.Lock()
+		defer n.mu.Unlock()
+		n.ip4 = ip4
+	}()
+
+	if len(dnsCfg.IP4) == 0 {
+		log.Print("nameserver's configuration contains no records, any in-memory records will be unset")
+		return nil
+	}
+
+	for fqdn, ips := range dnsCfg.IP4 {
+		fqdn, err := dnsname.ToFQDN(fqdn)
+		if err != nil {
+			log.Printf("invalid nameserver's configuration: %s is not a valid FQDN: %v; skipping this record", fqdn, err)
+			continue // one invalid hostname should not break the whole nameserver
+		}
+		for _, ipS := range ips {
+			ip := net.ParseIP(ipS).To4()
+			if ip == nil { // To4 returns nil if IP is not a IPv4 address
+				log.Printf("invalid nameserver's configuration: %v does not appear to be an IPv4 address; skipping this record", ipS)
+				continue // one invalid IP address should not break the whole nameserver
+			}
+			ip4[fqdn] = []net.IP{ip}
+		}
+	}
+	return nil
+}
+
+// listenAndServe starts a DNS server for the provided network and address.
+func listenAndServe(net, addr string, shutdown chan os.Signal) {
+	s := &dns.Server{Addr: addr, Net: net}
+	go func() {
+		<-shutdown
+		log.Printf("shutting down server for %s", net)
+		s.Shutdown()
+	}()
+	log.Printf("listening for %s queries on %s", net, addr)
+	if err := s.ListenAndServe(); err != nil {
+		log.Fatalf("error running %s server: %v", net, err)
+	}
+}
+
+// ensureWatcherForKubeConfigMap sets up a new file watcher for the ConfigMap
+// that's expected to be mounted at /config. Returns a channel that receives an
+// event every time the contents get updated.
+func ensureWatcherForKubeConfigMap(ctx context.Context) chan string {
+	c := make(chan string)
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatalf("error creating a new watcher for the mounted ConfigMap: %v", err)
+	}
+	// kubelet mounts configmap to a Pod using a series of symlinks, one of
+	// which is <mount-dir>/..data that Kubernetes recommends consumers to
+	// use if they need to monitor changes
+	// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
+	toWatch := filepath.Join(defaultDNSConfigDir, kubeletMountedConfigLn)
+	go func() {
+		defer watcher.Close()
+		log.Printf("starting file watch for %s", defaultDNSConfigDir)
+		for {
+			select {
+			case <-ctx.Done():
+				log.Print("context cancelled, exiting ConfigMap watcher")
+				return
+			case event, ok := <-watcher.Events:
+				if !ok {
+					log.Fatal("watcher finished; exiting")
+				}
+				if event.Name == toWatch {
+					msg := fmt.Sprintf("ConfigMap update received: %s", event)
+					log.Print(msg)
+					c <- msg
+				}
+			case err, ok := <-watcher.Errors:
+				if err != nil {
+					// TODO (irbekrm): this runs in a
+					// container that will be thrown away,
+					// so this should be ok. But maybe still
+					// need to ensure that the DNS server
+					// terminates connections more
+					// gracefully.
+					log.Fatalf("[unexpected] error watching configuration: %v", err)
+				}
+				if !ok {
+					// TODO (irbekrm): this runs in a
+					// container that will be thrown away,
+					// so this should be ok. But maybe still
+					// need to ensure that the DNS server
+					// terminates connections more
+					// gracefully.
+					log.Fatalf("[unexpected] errors watcher exited")
+				}
+			}
+		}
+	}()
+	if err = watcher.Add(defaultDNSConfigDir); err != nil {
+		log.Fatalf("failed setting up a watcher for the mounted ConfigMap: %v", err)
+	}
+	return c
+}
+
+// configReaderFunc is a function that returns the desired nameserver configuration.
+type configReaderFunc func() ([]byte, error)
+
+// configMapConfigReader reads the desired nameserver configuration from a
+// records.json file in a ConfigMap mounted at /config.
+var configMapConfigReader configReaderFunc = func() ([]byte, error) {
+	if contents, err := os.ReadFile(filepath.Join(defaultDNSConfigDir, operatorutils.DNSRecordsCMKey)); err == nil {
+		return contents, nil
+	} else if os.IsNotExist(err) {
+		return nil, nil
+	} else {
+		return nil, err
+	}
+}
+
+// lookupIP4 returns any IPv4 addresses for the given FQDN from nameserver's
+// in-memory records.
+func (n *nameserver) lookupIP4(fqdn dnsname.FQDN) []net.IP {
+	if n.ip4 == nil {
+		return nil
+	}
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	f := n.ip4[fqdn]
+	return f
+}

cmd/k8s-nameserver/main_test.go

@@ -0,0 +1,229 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"net"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/miekg/dns"
+	"tailscale.com/util/dnsname"
+)
+
+func TestNameserver(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		ip4      map[dnsname.FQDN][]net.IP
+		query    *dns.Msg
+		wantResp *dns.Msg
+	}{
+		{
+			name: "A record query, record exists",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}},
+				MsgHdr:   dns.MsgHdr{Id: 1, RecursionDesired: true},
+			},
+			wantResp: &dns.Msg{
+				Answer: []dns.RR{&dns.A{Hdr: dns.RR_Header{
+					Name: "foo.bar.com", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0},
+					A: net.IP{1, 2, 3, 4}}},
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}},
+				MsgHdr: dns.MsgHdr{
+					Id:                 1,
+					Rcode:              dns.RcodeSuccess,
+					RecursionAvailable: false,
+					RecursionDesired:   true,
+					Response:           true,
+					Opcode:             dns.OpcodeQuery,
+					Authoritative:      true,
+				}},
+		},
+		{
+			name: "A record query, record does not exist",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}},
+				MsgHdr: dns.MsgHdr{
+					Id:                 1,
+					Rcode:              dns.RcodeNameError,
+					RecursionAvailable: false,
+					Response:           true,
+					Opcode:             dns.OpcodeQuery,
+					Authoritative:      true,
+				}},
+		},
+		{
+			name: "A record query, but the name is not a valid FQDN",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}},
+				MsgHdr: dns.MsgHdr{
+					Id:       1,
+					Rcode:    dns.RcodeFormatError,
+					Response: true,
+					Opcode:   dns.OpcodeQuery,
+				}},
+		},
+		{
+			name: "AAAA record query, A record exists",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr: dns.MsgHdr{
+					Id:            1,
+					Rcode:         dns.RcodeSuccess,
+					Response:      true,
+					Opcode:        dns.OpcodeQuery,
+					Authoritative: true,
+				}},
+		},
+		{
+			name: "AAAA record query, A record does not exist",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr: dns.MsgHdr{
+					Id:            1,
+					Rcode:         dns.RcodeNameError,
+					Response:      true,
+					Opcode:        dns.OpcodeQuery,
+					Authoritative: true,
+				}},
+		},
+		{
+			name: "CNAME record query",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}},
+				MsgHdr: dns.MsgHdr{
+					Id:       1,
+					Rcode:    dns.RcodeNotImplemented,
+					Response: true,
+					Opcode:   dns.OpcodeQuery,
+				}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ns := &nameserver{
+				ip4: tt.ip4,
+			}
+			handler := ns.handleFunc()
+			fakeRespW := &fakeResponseWriter{}
+			handler(fakeRespW, tt.query)
+			if diff := cmp.Diff(*fakeRespW.msg, *tt.wantResp); diff != "" {
+				t.Fatalf("unexpected response (-got +want): \n%s", diff)
+			}
+		})
+	}
+}
+
+func TestResetRecords(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   []byte
+		hasIp4   map[dnsname.FQDN][]net.IP
+		wantsIp4 map[dnsname.FQDN][]net.IP
+		wantsErr bool
+	}{
+		{
+			name:     "previously empty nameserver.ip4 gets set",
+			config:   []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
+			wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}},
+		},
+		{
+			name:     "nameserver.ip4 gets reset",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			config:   []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
+			wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}},
+		},
+		{
+			name:     "configuration with incompatible version",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			config:   []byte(`{"version": "v1beta1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
+			wantsIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			wantsErr: true,
+		},
+		{
+			name:     "nameserver.ip4 gets reset to empty config when no configuration is provided",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			wantsIp4: make(map[dnsname.FQDN][]net.IP),
+		},
+		{
+			name:     "nameserver.ip4 gets reset to empty config when the provided configuration is empty",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			config:   []byte(`{"version": "v1alpha1", "ip4": {}}`),
+			wantsIp4: make(map[dnsname.FQDN][]net.IP),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ns := &nameserver{
+				ip4:          tt.hasIp4,
+				configReader: func() ([]byte, error) { return tt.config, nil },
+			}
+			if err := ns.resetRecords(); err == nil == tt.wantsErr {
+				t.Errorf("resetRecords() returned err: %v, wantsErr: %v", err, tt.wantsErr)
+			}
+			if diff := cmp.Diff(ns.ip4, tt.wantsIp4); diff != "" {
+				t.Fatalf("unexpected nameserver.ip4 contents (-got +want): \n%s", diff)
+			}
+		})
+	}
+}
+
+// fakeResponseWriter is a faked out dns.ResponseWriter that can be used in
+// tests that need to read the response message that was written.
+type fakeResponseWriter struct {
+	msg *dns.Msg
+}
+
+var _ dns.ResponseWriter = &fakeResponseWriter{}
+
+func (fr *fakeResponseWriter) WriteMsg(msg *dns.Msg) error {
+	fr.msg = msg
+	return nil
+}
+func (fr *fakeResponseWriter) LocalAddr() net.Addr {
+	return nil
+}
+func (fr *fakeResponseWriter) RemoteAddr() net.Addr {
+	return nil
+}
+func (fr *fakeResponseWriter) Write([]byte) (int, error) {
+	return 0, nil
+}
+func (fr *fakeResponseWriter) Close() error {
+	return nil
+}
+func (fr *fakeResponseWriter) TsigStatus() error {
+	return nil
+}
+func (fr *fakeResponseWriter) TsigTimersOnly(bool) {}
+func (fr *fakeResponseWriter) Hijack()             {}

cmd/k8s-operator/connector.go

@@ -33,11 +33,8 @@ import (
 
 const (
 	reasonConnectorCreationFailed = "ConnectorCreationFailed"
-
-	reasonConnectorCreated           = "ConnectorCreated"
-	reasonConnectorCleanupFailed     = "ConnectorCleanupFailed"
-	reasonConnectorCleanupInProgress = "ConnectorCleanupInProgress"
-	reasonConnectorInvalid           = "ConnectorInvalid"
+	reasonConnectorCreated        = "ConnectorCreated"
+	reasonConnectorInvalid        = "ConnectorInvalid"
 
 	messageConnectorCreationFailed = "Failed creating Connector: %v"
 	messageConnectorInvalid        = "Connector is invalid: %v"
@@ -108,7 +105,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	}
 
 	oldCnStatus := cn.Status.DeepCopy()
-	setStatus := func(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
@@ -184,7 +181,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		Connector: &connector{
 			isExitNode: cn.Spec.ExitNode,
 		},
-		ProxyClass: proxyClass,
+		ProxyClassName: proxyClass,
 	}
 
 	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
@@ -211,7 +208,27 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
-	return err
+	if err != nil {
+		return err
+	}
+
+	_, tsHost, ips, err := a.ssr.DeviceInfo(ctx, crl)
+	if err != nil {
+		return err
+	}
+
+	if tsHost == "" {
+		logger.Debugf("no Tailscale hostname known yet, waiting for connector pod to finish auth")
+		// No hostname yet. Wait for the connector pod to auth.
+		cn.Status.TailnetIPs = nil
+		cn.Status.Hostname = ""
+		return nil
+	}
+
+	cn.Status.TailnetIPs = ips
+	cn.Status.Hostname = tsHost
+
+	return nil
 }
 
 func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {

cmd/k8s-operator/connector_test.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/tstest"
+	"tailscale.com/util/mak"
 )
 
 func TestConnector(t *testing.T) {
@@ -29,7 +30,7 @@ func TestConnector(t *testing.T) {
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind:       tsapi.ConnectorKind,
-			APIVersion: "tailscale.io/v1alpha1",
+			APIVersion: "tailscale.com/v1alpha1",
 		},
 		Spec: tsapi.ConnectorSpec{
 			SubnetRouter: &tsapi.SubnetRouter{
@@ -74,9 +75,26 @@ func TestConnector(t *testing.T) {
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
+	// Connector status should get updated with the IP/hostname info when available.
+	const hostname = "foo.tailnetxyz.ts.net"
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "device_fqdn", []byte(hostname))
+		mak.Set(&secret.Data, "device_ips", []byte(`["127.0.0.1", "::1"]`))
+	})
+	expectReconciled(t, cr, "", "test")
+	cn.Finalizers = append(cn.Finalizers, "tailscale.com/finalizer")
+	cn.Status.IsExitNode = cn.Spec.ExitNode
+	cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
+	cn.Status.Hostname = hostname
+	cn.Status.TailnetIPs = []string{"127.0.0.1", "::1"}
+	expectEqual(t, fc, cn, func(o *tsapi.Connector) {
+		o.Status.Conditions = nil
+	})
+
 	// Add another route to be advertised.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
 		conn.Spec.SubnetRouter.AdvertiseRoutes = []tsapi.Route{"10.40.0.0/14", "10.44.0.0/20"}
@@ -152,7 +170,7 @@ func TestConnector(t *testing.T) {
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
 	// Add an exit node.
@@ -237,7 +255,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
 	// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet
@@ -254,9 +272,9 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	// its resources.
 	mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
 		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []tsapi.ConnectorCondition{{
+			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               tsapi.ProxyClassready,
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/depaware.txt

@@ -0,0 +1,997 @@
+tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/depaware)
+
+        filippo.io/edwards25519                                      from github.com/hdevalence/ed25519consensus
+        filippo.io/edwards25519/field                                from filippo.io/edwards25519
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+   L    github.com/aws/aws-sdk-go-v2/aws                             from github.com/aws/aws-sdk-go-v2/aws/defaults+
+   L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/awsstore
+   L    github.com/aws/aws-sdk-go-v2/aws/defaults                    from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/aws/middleware                  from github.com/aws/aws-sdk-go-v2/aws/retry+
+   L    github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics  from github.com/aws/aws-sdk-go-v2/aws/retry+
+   L    github.com/aws/aws-sdk-go-v2/aws/protocol/query              from github.com/aws/aws-sdk-go-v2/service/sts
+   L    github.com/aws/aws-sdk-go-v2/aws/protocol/restjson           from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/aws/protocol/xml                from github.com/aws/aws-sdk-go-v2/service/sts
+   L    github.com/aws/aws-sdk-go-v2/aws/ratelimit                   from github.com/aws/aws-sdk-go-v2/aws/retry
+   L    github.com/aws/aws-sdk-go-v2/aws/retry                       from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client+
+   L    github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4          from github.com/aws/aws-sdk-go-v2/aws/signer/v4
+   L    github.com/aws/aws-sdk-go-v2/aws/signer/v4                   from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+
+   L    github.com/aws/aws-sdk-go-v2/aws/transport/http              from github.com/aws/aws-sdk-go-v2/config+
+   L    github.com/aws/aws-sdk-go-v2/config                          from tailscale.com/ipn/store/awsstore
+   L    github.com/aws/aws-sdk-go-v2/credentials                     from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds        from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/credentials/endpointcreds       from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds
+   L    github.com/aws/aws-sdk-go-v2/credentials/processcreds        from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/credentials/ssocreds            from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/credentials/stscreds            from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds                from github.com/aws/aws-sdk-go-v2/config+
+   L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds
+   L    github.com/aws/aws-sdk-go-v2/internal/auth                   from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
+   L    github.com/aws/aws-sdk-go-v2/internal/auth/smithy            from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/configsources          from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/endpoints              from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn   from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2           from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
+   L    github.com/aws/aws-sdk-go-v2/internal/ini                    from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/internal/rand                   from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/aws-sdk-go-v2/internal/sdk                    from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/aws-sdk-go-v2/internal/sdkio                  from github.com/aws/aws-sdk-go-v2/credentials/processcreds
+   L    github.com/aws/aws-sdk-go-v2/internal/shareddefaults         from github.com/aws/aws-sdk-go-v2/config+
+   L    github.com/aws/aws-sdk-go-v2/internal/strings                from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4
+   L    github.com/aws/aws-sdk-go-v2/internal/sync/singleflight      from github.com/aws/aws-sdk-go-v2/aws
+   L    github.com/aws/aws-sdk-go-v2/internal/timeconv               from github.com/aws/aws-sdk-go-v2/aws/retry
+   L    github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts
+   L    github.com/aws/aws-sdk-go-v2/service/internal/presigned-url  from github.com/aws/aws-sdk-go-v2/service/sts
+   L    github.com/aws/aws-sdk-go-v2/service/ssm                     from tailscale.com/ipn/store/awsstore
+   L    github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints  from github.com/aws/aws-sdk-go-v2/service/ssm
+   L    github.com/aws/aws-sdk-go-v2/service/ssm/types               from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/service/sso                     from github.com/aws/aws-sdk-go-v2/config+
+   L    github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints  from github.com/aws/aws-sdk-go-v2/service/sso
+   L    github.com/aws/aws-sdk-go-v2/service/sso/types               from github.com/aws/aws-sdk-go-v2/service/sso
+   L    github.com/aws/aws-sdk-go-v2/service/ssooidc                 from github.com/aws/aws-sdk-go-v2/config+
+   L    github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssooidc
+   L    github.com/aws/aws-sdk-go-v2/service/ssooidc/types           from github.com/aws/aws-sdk-go-v2/service/ssooidc
+   L    github.com/aws/aws-sdk-go-v2/service/sts                     from github.com/aws/aws-sdk-go-v2/config+
+   L    github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints  from github.com/aws/aws-sdk-go-v2/service/sts
+   L    github.com/aws/aws-sdk-go-v2/service/sts/types               from github.com/aws/aws-sdk-go-v2/credentials/stscreds+
+   L    github.com/aws/smithy-go                                     from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+
+   L    github.com/aws/smithy-go/auth                                from github.com/aws/aws-sdk-go-v2/internal/auth+
+   L    github.com/aws/smithy-go/auth/bearer                         from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/smithy-go/context                             from github.com/aws/smithy-go/auth/bearer
+   L    github.com/aws/smithy-go/document                            from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/smithy-go/encoding                            from github.com/aws/smithy-go/encoding/json+
+   L    github.com/aws/smithy-go/encoding/httpbinding                from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
+   L    github.com/aws/smithy-go/encoding/json                       from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/smithy-go/encoding/xml                        from github.com/aws/aws-sdk-go-v2/service/sts
+   L    github.com/aws/smithy-go/endpoints                           from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/smithy-go/internal/sync/singleflight          from github.com/aws/smithy-go/auth/bearer
+   L    github.com/aws/smithy-go/io                                  from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+
+   L    github.com/aws/smithy-go/logging                             from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/smithy-go/middleware                          from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/smithy-go/private/requestcompression          from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/smithy-go/ptr                                 from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/smithy-go/rand                                from github.com/aws/aws-sdk-go-v2/aws/middleware+
+   L    github.com/aws/smithy-go/time                                from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/smithy-go/transport/http                      from github.com/aws/aws-sdk-go-v2/aws/middleware+
+   L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
+   L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
+        github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
+        github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
+     💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+   L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
+     💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
+   W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
+   W 💣 github.com/dblohm7/wingoes/com                               from tailscale.com/util/osdiag+
+   W 💣 github.com/dblohm7/wingoes/com/automation                    from tailscale.com/util/osdiag/internal/wsc
+   W    github.com/dblohm7/wingoes/internal                          from github.com/dblohm7/wingoes/com
+   W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/osdiag+
+  LW 💣 github.com/digitalocean/go-smbios/smbios                     from tailscale.com/posture
+        github.com/distribution/reference                            from tailscale.com/cmd/k8s-operator
+        github.com/emicklei/go-restful/v3                            from k8s.io/kube-openapi/pkg/common
+        github.com/emicklei/go-restful/v3/log                        from github.com/emicklei/go-restful/v3
+        github.com/evanphx/json-patch/v5                             from sigs.k8s.io/controller-runtime/pkg/client
+        github.com/evanphx/json-patch/v5/internal/json               from github.com/evanphx/json-patch/v5
+     💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
+        github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
+        github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
+        github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
+        github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
+        github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json/jsontext+
+        github.com/go-json-experiment/json/jsontext                  from tailscale.com/logtail+
+        github.com/go-logr/logr                                      from github.com/go-logr/logr/slogr+
+        github.com/go-logr/logr/slogr                                from github.com/go-logr/zapr
+        github.com/go-logr/zapr                                      from sigs.k8s.io/controller-runtime/pkg/log/zap+
+   W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
+   W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
+        github.com/go-openapi/jsonpointer                            from github.com/go-openapi/jsonreference
+        github.com/go-openapi/jsonreference                          from k8s.io/kube-openapi/pkg/internal+
+        github.com/go-openapi/jsonreference/internal                 from github.com/go-openapi/jsonreference
+        github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
+   L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+     💣 github.com/gogo/protobuf/proto                               from k8s.io/api/admission/v1+
+        github.com/gogo/protobuf/sortkeys                            from k8s.io/api/admission/v1+
+        github.com/golang/groupcache/lru                             from k8s.io/client-go/tools/record+
+        github.com/golang/protobuf/proto                             from k8s.io/client-go/discovery+
+        github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header+
+        github.com/google/gnostic-models/compiler                    from github.com/google/gnostic-models/openapiv2+
+        github.com/google/gnostic-models/extensions                  from github.com/google/gnostic-models/compiler
+        github.com/google/gnostic-models/jsonschema                  from github.com/google/gnostic-models/compiler
+        github.com/google/gnostic-models/openapiv2                   from k8s.io/client-go/discovery+
+        github.com/google/gnostic-models/openapiv3                   from k8s.io/kube-openapi/pkg/handler3+
+     💣 github.com/google/go-cmp/cmp                                 from k8s.io/apimachinery/pkg/util/diff+
+        github.com/google/go-cmp/cmp/internal/diff                   from github.com/google/go-cmp/cmp
+        github.com/google/go-cmp/cmp/internal/flags                  from github.com/google/go-cmp/cmp+
+        github.com/google/go-cmp/cmp/internal/function               from github.com/google/go-cmp/cmp
+     💣 github.com/google/go-cmp/cmp/internal/value                  from github.com/google/go-cmp/cmp
+        github.com/google/gofuzz                                     from k8s.io/apimachinery/pkg/apis/meta/v1+
+        github.com/google/gofuzz/bytesource                          from github.com/google/gofuzz
+   L    github.com/google/nftables                                   from tailscale.com/util/linuxfw
+   L 💣 github.com/google/nftables/alignedbuff                       from github.com/google/nftables/xt
+   L 💣 github.com/google/nftables/binaryutil                        from github.com/google/nftables+
+   L    github.com/google/nftables/expr                              from github.com/google/nftables+
+   L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
+   L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
+        github.com/google/uuid                                       from github.com/prometheus-community/pro-bing+
+        github.com/gorilla/csrf                                      from tailscale.com/client/web
+        github.com/gorilla/securecookie                              from github.com/gorilla/csrf
+        github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
+   L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
+        github.com/imdario/mergo                                     from k8s.io/client-go/tools/clientcmd
+   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
+   L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
+   L    github.com/insomniacslk/dhcp/interfaces                      from github.com/insomniacslk/dhcp/dhcpv4
+   L    github.com/insomniacslk/dhcp/rfc1035label                    from github.com/insomniacslk/dhcp/dhcpv4
+   L    github.com/jmespath/go-jmespath                              from github.com/aws/aws-sdk-go-v2/service/ssm
+        github.com/josharian/intern                                  from github.com/mailru/easyjson/jlexer
+   L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
+   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
+   L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
+     💣 github.com/json-iterator/go                                  from sigs.k8s.io/structured-merge-diff/v4/fieldpath+
+        github.com/klauspost/compress                                from github.com/klauspost/compress/zstd
+        github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
+        github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
+        github.com/klauspost/compress/internal/cpuinfo               from github.com/klauspost/compress/huff0+
+        github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
+        github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
+        github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
+        github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
+        github.com/mailru/easyjson/buffer                            from github.com/mailru/easyjson/jwriter
+     💣 github.com/mailru/easyjson/jlexer                            from github.com/go-openapi/swag
+        github.com/mailru/easyjson/jwriter                           from github.com/go-openapi/swag
+   L    github.com/mdlayher/genetlink                                from tailscale.com/net/tstun
+   L 💣 github.com/mdlayher/netlink                                  from github.com/google/nftables+
+   L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
+   L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
+   L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+        github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
+     💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
+        github.com/modern-go/concurrent                              from github.com/json-iterator/go
+     💣 github.com/modern-go/reflect2                                from github.com/json-iterator/go
+        github.com/munnerz/goautoneg                                 from k8s.io/kube-openapi/pkg/handler3
+        github.com/opencontainers/go-digest                          from github.com/distribution/reference
+   L    github.com/pierrec/lz4/v4                                    from github.com/u-root/uio/uio
+   L    github.com/pierrec/lz4/v4/internal/lz4block                  from github.com/pierrec/lz4/v4+
+   L    github.com/pierrec/lz4/v4/internal/lz4errors                 from github.com/pierrec/lz4/v4+
+   L    github.com/pierrec/lz4/v4/internal/lz4stream                 from github.com/pierrec/lz4/v4
+   L    github.com/pierrec/lz4/v4/internal/xxh32                     from github.com/pierrec/lz4/v4/internal/lz4stream
+        github.com/pkg/errors                                        from github.com/evanphx/json-patch/v5+
+   D    github.com/prometheus-community/pro-bing                     from tailscale.com/wgengine/netstack
+     💣 github.com/prometheus/client_golang/prometheus               from github.com/prometheus/client_golang/prometheus/collectors+
+        github.com/prometheus/client_golang/prometheus/collectors    from sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics
+        github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus+
+        github.com/prometheus/client_golang/prometheus/promhttp      from sigs.k8s.io/controller-runtime/pkg/metrics/server+
+        github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
+        github.com/prometheus/common/expfmt                          from github.com/prometheus/client_golang/prometheus+
+        github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
+        github.com/prometheus/common/model                           from github.com/prometheus/client_golang/prometheus+
+  LD    github.com/prometheus/procfs                                 from github.com/prometheus/client_golang/prometheus
+  LD    github.com/prometheus/procfs/internal/fs                     from github.com/prometheus/procfs
+  LD    github.com/prometheus/procfs/internal/util                   from github.com/prometheus/procfs
+   L 💣 github.com/safchain/ethtool                                  from tailscale.com/doctor/ethtool+
+        github.com/spf13/pflag                                       from k8s.io/client-go/tools/clientcmd
+   W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
+   W 💣 github.com/tailscale/go-winio                                from tailscale.com/safesocket
+   W 💣 github.com/tailscale/go-winio/internal/fs                    from github.com/tailscale/go-winio
+   W 💣 github.com/tailscale/go-winio/internal/socket                from github.com/tailscale/go-winio
+   W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
+   W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
+        github.com/tailscale/golang-x-crypto/acme                    from tailscale.com/ipn/ipnlocal
+  LD    github.com/tailscale/golang-x-crypto/internal/poly1305       from github.com/tailscale/golang-x-crypto/ssh
+  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal
+  LD    github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2+
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+        github.com/tailscale/hujson                                  from tailscale.com/ipn/conffile
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/net/routetable+
+        github.com/tailscale/peercred                                from tailscale.com/ipn/ipnauth
+        github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
+     💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
+   W 💣 github.com/tailscale/wireguard-go/conn/winrio                from github.com/tailscale/wireguard-go/conn
+     💣 github.com/tailscale/wireguard-go/device                     from tailscale.com/net/tstun+
+     💣 github.com/tailscale/wireguard-go/ipc                        from github.com/tailscale/wireguard-go/device
+   W 💣 github.com/tailscale/wireguard-go/ipc/namedpipe              from github.com/tailscale/wireguard-go/ipc
+        github.com/tailscale/wireguard-go/ratelimiter                from github.com/tailscale/wireguard-go/device
+        github.com/tailscale/wireguard-go/replay                     from github.com/tailscale/wireguard-go/device
+        github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
+        github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device
+     💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
+        github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
+   L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
+   L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
+        github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
+        go.uber.org/multierr                                         from go.uber.org/zap+
+        go.uber.org/zap                                              from github.com/go-logr/zapr+
+        go.uber.org/zap/buffer                                       from go.uber.org/zap/internal/bufferpool+
+        go.uber.org/zap/internal                                     from go.uber.org/zap
+        go.uber.org/zap/internal/bufferpool                          from go.uber.org/zap+
+        go.uber.org/zap/internal/color                               from go.uber.org/zap/zapcore
+        go.uber.org/zap/internal/exit                                from go.uber.org/zap/zapcore
+        go.uber.org/zap/internal/pool                                from go.uber.org/zap+
+        go.uber.org/zap/internal/stacktrace                          from go.uber.org/zap
+        go.uber.org/zap/zapcore                                      from github.com/go-logr/zapr+
+     💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
+        go4.org/netipx                                               from tailscale.com/ipn/ipnlocal+
+   W 💣 golang.zx2c4.com/wintun                                      from github.com/tailscale/wireguard-go/tun
+   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/dns+
+        gomodules.xyz/jsonpatch/v2                                   from sigs.k8s.io/controller-runtime/pkg/webhook+
+        google.golang.org/protobuf/encoding/protodelim               from github.com/prometheus/common/expfmt
+        google.golang.org/protobuf/encoding/prototext                from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/encoding/protowire                from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/internal/descfmt                  from google.golang.org/protobuf/internal/filedesc
+        google.golang.org/protobuf/internal/descopts                 from google.golang.org/protobuf/internal/filedesc+
+        google.golang.org/protobuf/internal/detrand                  from google.golang.org/protobuf/internal/descfmt+
+        google.golang.org/protobuf/internal/editiondefaults          from google.golang.org/protobuf/internal/filedesc+
+        google.golang.org/protobuf/internal/encoding/defval          from google.golang.org/protobuf/internal/encoding/tag+
+        google.golang.org/protobuf/internal/encoding/messageset      from google.golang.org/protobuf/encoding/prototext+
+        google.golang.org/protobuf/internal/encoding/tag             from google.golang.org/protobuf/internal/impl
+        google.golang.org/protobuf/internal/encoding/text            from google.golang.org/protobuf/encoding/prototext+
+        google.golang.org/protobuf/internal/errors                   from google.golang.org/protobuf/encoding/protodelim+
+        google.golang.org/protobuf/internal/filedesc                 from google.golang.org/protobuf/internal/encoding/tag+
+        google.golang.org/protobuf/internal/filetype                 from google.golang.org/protobuf/runtime/protoimpl
+        google.golang.org/protobuf/internal/flags                    from google.golang.org/protobuf/encoding/prototext+
+        google.golang.org/protobuf/internal/genid                    from google.golang.org/protobuf/encoding/prototext+
+     💣 google.golang.org/protobuf/internal/impl                     from google.golang.org/protobuf/internal/filetype+
+        google.golang.org/protobuf/internal/order                    from google.golang.org/protobuf/encoding/prototext+
+        google.golang.org/protobuf/internal/pragma                   from google.golang.org/protobuf/encoding/prototext+
+        google.golang.org/protobuf/internal/set                      from google.golang.org/protobuf/encoding/prototext
+     💣 google.golang.org/protobuf/internal/strs                     from google.golang.org/protobuf/encoding/prototext+
+        google.golang.org/protobuf/internal/version                  from google.golang.org/protobuf/runtime/protoimpl
+        google.golang.org/protobuf/proto                             from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/reflect/protodesc                 from github.com/golang/protobuf/proto
+     💣 google.golang.org/protobuf/reflect/protoreflect              from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/reflect/protoregistry             from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/runtime/protoiface                from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/runtime/protoimpl                 from github.com/golang/protobuf/proto+
+        google.golang.org/protobuf/types/descriptorpb                from github.com/google/gnostic-models/openapiv3+
+        google.golang.org/protobuf/types/gofeaturespb                from google.golang.org/protobuf/reflect/protodesc
+        google.golang.org/protobuf/types/known/anypb                 from github.com/google/gnostic-models/compiler+
+        google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
+        gopkg.in/inf.v0                                              from k8s.io/apimachinery/pkg/api/resource
+        gopkg.in/yaml.v2                                             from k8s.io/kube-openapi/pkg/util/proto+
+        gopkg.in/yaml.v3                                             from github.com/go-openapi/swag+
+        gvisor.dev/gvisor/pkg/atomicbitops                           from gvisor.dev/gvisor/pkg/buffer+
+        gvisor.dev/gvisor/pkg/bits                                   from gvisor.dev/gvisor/pkg/buffer
+     💣 gvisor.dev/gvisor/pkg/buffer                                 from gvisor.dev/gvisor/pkg/tcpip+
+        gvisor.dev/gvisor/pkg/context                                from gvisor.dev/gvisor/pkg/refs
+     💣 gvisor.dev/gvisor/pkg/gohacks                                from gvisor.dev/gvisor/pkg/state/wire+
+        gvisor.dev/gvisor/pkg/linewriter                             from gvisor.dev/gvisor/pkg/log
+        gvisor.dev/gvisor/pkg/log                                    from gvisor.dev/gvisor/pkg/context+
+        gvisor.dev/gvisor/pkg/rand                                   from gvisor.dev/gvisor/pkg/tcpip/network/hash+
+        gvisor.dev/gvisor/pkg/refs                                   from gvisor.dev/gvisor/pkg/buffer+
+     💣 gvisor.dev/gvisor/pkg/sleep                                  from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
+     💣 gvisor.dev/gvisor/pkg/state                                  from gvisor.dev/gvisor/pkg/atomicbitops+
+        gvisor.dev/gvisor/pkg/state/wire                             from gvisor.dev/gvisor/pkg/state
+     💣 gvisor.dev/gvisor/pkg/sync                                   from gvisor.dev/gvisor/pkg/atomicbitops+
+     💣 gvisor.dev/gvisor/pkg/sync/locking                           from gvisor.dev/gvisor/pkg/tcpip/stack
+        gvisor.dev/gvisor/pkg/tcpip                                  from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/adapters/gonet                   from tailscale.com/wgengine/netstack
+     💣 gvisor.dev/gvisor/pkg/tcpip/checksum                         from gvisor.dev/gvisor/pkg/buffer+
+        gvisor.dev/gvisor/pkg/tcpip/hash/jenkins                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
+        gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
+        gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
+     💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
+        gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
+        gvisor.dev/gvisor/pkg/tcpip/transport/internal/noop          from gvisor.dev/gvisor/pkg/tcpip/transport/raw
+        gvisor.dev/gvisor/pkg/tcpip/transport/packet                 from gvisor.dev/gvisor/pkg/tcpip/transport/raw
+        gvisor.dev/gvisor/pkg/tcpip/transport/raw                    from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
+     💣 gvisor.dev/gvisor/pkg/tcpip/transport/tcp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack           from gvisor.dev/gvisor/pkg/tcpip/stack
+        gvisor.dev/gvisor/pkg/tcpip/transport/udp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/waiter                                 from gvisor.dev/gvisor/pkg/context+
+        k8s.io/api/admission/v1                                      from sigs.k8s.io/controller-runtime/pkg/webhook/admission
+        k8s.io/api/admission/v1beta1                                 from sigs.k8s.io/controller-runtime/pkg/webhook/admission
+        k8s.io/api/admissionregistration/v1                          from k8s.io/api/admissionregistration/v1alpha1+
+        k8s.io/api/admissionregistration/v1alpha1                    from k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1+
+        k8s.io/api/admissionregistration/v1beta1                     from k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1+
+        k8s.io/api/apidiscovery/v2                                   from k8s.io/client-go/discovery
+        k8s.io/api/apidiscovery/v2beta1                              from k8s.io/client-go/discovery
+        k8s.io/api/apiserverinternal/v1alpha1                        from k8s.io/client-go/applyconfigurations/apiserverinternal/v1alpha1+
+        k8s.io/api/apps/v1                                           from k8s.io/client-go/applyconfigurations/apps/v1+
+        k8s.io/api/apps/v1beta1                                      from k8s.io/api/extensions/v1beta1+
+        k8s.io/api/apps/v1beta2                                      from k8s.io/client-go/applyconfigurations/apps/v1beta2+
+        k8s.io/api/authentication/v1                                 from k8s.io/api/admission/v1+
+        k8s.io/api/authentication/v1alpha1                           from k8s.io/client-go/kubernetes/scheme+
+        k8s.io/api/authentication/v1beta1                            from k8s.io/client-go/kubernetes/scheme+
+        k8s.io/api/authorization/v1                                  from k8s.io/client-go/kubernetes/scheme+
+        k8s.io/api/authorization/v1beta1                             from k8s.io/client-go/kubernetes/scheme+
+        k8s.io/api/autoscaling/v1                                    from k8s.io/client-go/applyconfigurations/autoscaling/v1+
+        k8s.io/api/autoscaling/v2                                    from k8s.io/client-go/applyconfigurations/autoscaling/v2+
+        k8s.io/api/autoscaling/v2beta1                               from k8s.io/client-go/applyconfigurations/autoscaling/v2beta1+
+        k8s.io/api/autoscaling/v2beta2                               from k8s.io/client-go/applyconfigurations/autoscaling/v2beta2+
+        k8s.io/api/batch/v1                                          from k8s.io/api/batch/v1beta1+
+        k8s.io/api/batch/v1beta1                                     from k8s.io/client-go/applyconfigurations/batch/v1beta1+
+        k8s.io/api/certificates/v1                                   from k8s.io/client-go/applyconfigurations/certificates/v1+
+        k8s.io/api/certificates/v1alpha1                             from k8s.io/client-go/applyconfigurations/certificates/v1alpha1+
+        k8s.io/api/certificates/v1beta1                              from k8s.io/client-go/applyconfigurations/certificates/v1beta1+
+        k8s.io/api/coordination/v1                                   from k8s.io/client-go/applyconfigurations/coordination/v1+
+        k8s.io/api/coordination/v1beta1                              from k8s.io/client-go/applyconfigurations/coordination/v1beta1+
+        k8s.io/api/core/v1                                           from k8s.io/api/apps/v1+
+        k8s.io/api/discovery/v1                                      from k8s.io/client-go/applyconfigurations/discovery/v1+
+        k8s.io/api/discovery/v1beta1                                 from k8s.io/client-go/applyconfigurations/discovery/v1beta1+
+        k8s.io/api/events/v1                                         from k8s.io/client-go/applyconfigurations/events/v1+
+        k8s.io/api/events/v1beta1                                    from k8s.io/client-go/applyconfigurations/events/v1beta1+
+        k8s.io/api/extensions/v1beta1                                from k8s.io/client-go/applyconfigurations/extensions/v1beta1+
+        k8s.io/api/flowcontrol/v1                                    from k8s.io/client-go/applyconfigurations/flowcontrol/v1+
+        k8s.io/api/flowcontrol/v1beta1                               from k8s.io/client-go/applyconfigurations/flowcontrol/v1beta1+
+        k8s.io/api/flowcontrol/v1beta2                               from k8s.io/client-go/applyconfigurations/flowcontrol/v1beta2+
+        k8s.io/api/flowcontrol/v1beta3                               from k8s.io/client-go/applyconfigurations/flowcontrol/v1beta3+
+        k8s.io/api/networking/v1                                     from k8s.io/client-go/applyconfigurations/networking/v1+
+        k8s.io/api/networking/v1alpha1                               from k8s.io/client-go/applyconfigurations/networking/v1alpha1+
+        k8s.io/api/networking/v1beta1                                from k8s.io/client-go/applyconfigurations/networking/v1beta1+
+        k8s.io/api/node/v1                                           from k8s.io/client-go/applyconfigurations/node/v1+
+        k8s.io/api/node/v1alpha1                                     from k8s.io/client-go/applyconfigurations/node/v1alpha1+
+        k8s.io/api/node/v1beta1                                      from k8s.io/client-go/applyconfigurations/node/v1beta1+
+        k8s.io/api/policy/v1                                         from k8s.io/client-go/applyconfigurations/policy/v1+
+        k8s.io/api/policy/v1beta1                                    from k8s.io/client-go/applyconfigurations/policy/v1beta1+
+        k8s.io/api/rbac/v1                                           from k8s.io/client-go/applyconfigurations/rbac/v1+
+        k8s.io/api/rbac/v1alpha1                                     from k8s.io/client-go/applyconfigurations/rbac/v1alpha1+
+        k8s.io/api/rbac/v1beta1                                      from k8s.io/client-go/applyconfigurations/rbac/v1beta1+
+        k8s.io/api/resource/v1alpha2                                 from k8s.io/client-go/applyconfigurations/resource/v1alpha2+
+        k8s.io/api/scheduling/v1                                     from k8s.io/client-go/applyconfigurations/scheduling/v1+
+        k8s.io/api/scheduling/v1alpha1                               from k8s.io/client-go/applyconfigurations/scheduling/v1alpha1+
+        k8s.io/api/scheduling/v1beta1                                from k8s.io/client-go/applyconfigurations/scheduling/v1beta1+
+        k8s.io/api/storage/v1                                        from k8s.io/client-go/applyconfigurations/storage/v1+
+        k8s.io/api/storage/v1alpha1                                  from k8s.io/client-go/applyconfigurations/storage/v1alpha1+
+        k8s.io/api/storage/v1beta1                                   from k8s.io/client-go/applyconfigurations/storage/v1beta1+
+        k8s.io/api/storagemigration/v1alpha1                         from k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1+
+        k8s.io/apiextensions-apiserver/pkg/apis/apiextensions        from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
+     💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1     from sigs.k8s.io/controller-runtime/pkg/webhook/conversion
+        k8s.io/apimachinery/pkg/api/equality                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
+        k8s.io/apimachinery/pkg/api/errors                           from k8s.io/apimachinery/pkg/util/managedfields/internal+
+        k8s.io/apimachinery/pkg/api/meta                             from k8s.io/apimachinery/pkg/api/validation+
+        k8s.io/apimachinery/pkg/api/resource                         from k8s.io/api/autoscaling/v1+
+        k8s.io/apimachinery/pkg/api/validation                       from k8s.io/apimachinery/pkg/util/managedfields/internal+
+     💣 k8s.io/apimachinery/pkg/apis/meta/internalversion            from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
+        k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme     from k8s.io/client-go/metadata
+     💣 k8s.io/apimachinery/pkg/apis/meta/v1                         from k8s.io/api/admission/v1+
+        k8s.io/apimachinery/pkg/apis/meta/v1/unstructured            from k8s.io/apimachinery/pkg/runtime/serializer/versioning+
+        k8s.io/apimachinery/pkg/apis/meta/v1/validation              from k8s.io/apimachinery/pkg/api/validation+
+     💣 k8s.io/apimachinery/pkg/apis/meta/v1beta1                    from k8s.io/apimachinery/pkg/apis/meta/internalversion
+        k8s.io/apimachinery/pkg/conversion                           from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
+        k8s.io/apimachinery/pkg/conversion/queryparams               from k8s.io/apimachinery/pkg/runtime+
+        k8s.io/apimachinery/pkg/fields                               from k8s.io/apimachinery/pkg/api/equality+
+        k8s.io/apimachinery/pkg/labels                               from k8s.io/apimachinery/pkg/api/equality+
+        k8s.io/apimachinery/pkg/runtime                              from k8s.io/api/admission/v1+
+        k8s.io/apimachinery/pkg/runtime/schema                       from k8s.io/api/admission/v1+
+        k8s.io/apimachinery/pkg/runtime/serializer                   from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
+        k8s.io/apimachinery/pkg/runtime/serializer/json              from k8s.io/apimachinery/pkg/runtime/serializer+
+        k8s.io/apimachinery/pkg/runtime/serializer/protobuf          from k8s.io/apimachinery/pkg/runtime/serializer
+        k8s.io/apimachinery/pkg/runtime/serializer/recognizer        from k8s.io/apimachinery/pkg/runtime/serializer+
+        k8s.io/apimachinery/pkg/runtime/serializer/streaming         from k8s.io/client-go/rest+
+        k8s.io/apimachinery/pkg/runtime/serializer/versioning        from k8s.io/apimachinery/pkg/runtime/serializer+
+        k8s.io/apimachinery/pkg/selection                            from k8s.io/apimachinery/pkg/apis/meta/v1+
+        k8s.io/apimachinery/pkg/types                                from k8s.io/api/admission/v1+
+        k8s.io/apimachinery/pkg/util/cache                           from k8s.io/client-go/tools/cache
+        k8s.io/apimachinery/pkg/util/diff                            from k8s.io/client-go/tools/cache
+        k8s.io/apimachinery/pkg/util/dump                            from k8s.io/apimachinery/pkg/util/diff+
+        k8s.io/apimachinery/pkg/util/errors                          from k8s.io/apimachinery/pkg/api/meta+
+        k8s.io/apimachinery/pkg/util/framer                          from k8s.io/apimachinery/pkg/runtime/serializer/json+
+        k8s.io/apimachinery/pkg/util/intstr                          from k8s.io/api/apps/v1+
+        k8s.io/apimachinery/pkg/util/json                            from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
+        k8s.io/apimachinery/pkg/util/managedfields                   from k8s.io/client-go/applyconfigurations/admissionregistration/v1+
+        k8s.io/apimachinery/pkg/util/managedfields/internal          from k8s.io/apimachinery/pkg/util/managedfields
+        k8s.io/apimachinery/pkg/util/mergepatch                      from k8s.io/apimachinery/pkg/util/strategicpatch
+        k8s.io/apimachinery/pkg/util/naming                          from k8s.io/apimachinery/pkg/runtime+
+        k8s.io/apimachinery/pkg/util/net                             from k8s.io/apimachinery/pkg/watch+
+        k8s.io/apimachinery/pkg/util/rand                            from k8s.io/apiserver/pkg/storage/names
+        k8s.io/apimachinery/pkg/util/runtime                         from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
+        k8s.io/apimachinery/pkg/util/sets                            from k8s.io/apimachinery/pkg/api/meta+
+        k8s.io/apimachinery/pkg/util/strategicpatch                  from k8s.io/client-go/tools/record+
+        k8s.io/apimachinery/pkg/util/uuid                            from sigs.k8s.io/controller-runtime/pkg/internal/controller+
+        k8s.io/apimachinery/pkg/util/validation                      from k8s.io/apimachinery/pkg/api/validation+
+        k8s.io/apimachinery/pkg/util/validation/field                from k8s.io/apimachinery/pkg/api/errors+
+        k8s.io/apimachinery/pkg/util/wait                            from k8s.io/client-go/tools/cache+
+        k8s.io/apimachinery/pkg/util/yaml                            from k8s.io/apimachinery/pkg/runtime/serializer/json
+        k8s.io/apimachinery/pkg/version                              from k8s.io/client-go/discovery+
+        k8s.io/apimachinery/pkg/watch                                from k8s.io/apimachinery/pkg/apis/meta/v1+
+        k8s.io/apimachinery/third_party/forked/golang/json           from k8s.io/apimachinery/pkg/util/strategicpatch
+        k8s.io/apimachinery/third_party/forked/golang/reflect        from k8s.io/apimachinery/pkg/conversion
+        k8s.io/apiserver/pkg/storage/names                           from tailscale.com/cmd/k8s-operator
+        k8s.io/client-go/applyconfigurations/admissionregistration/v1 from k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1+
+        k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1
+        k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1 from k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1
+        k8s.io/client-go/applyconfigurations/apiserverinternal/v1alpha1 from k8s.io/client-go/kubernetes/typed/apiserverinternal/v1alpha1
+        k8s.io/client-go/applyconfigurations/apps/v1                 from k8s.io/client-go/kubernetes/typed/apps/v1
+        k8s.io/client-go/applyconfigurations/apps/v1beta1            from k8s.io/client-go/kubernetes/typed/apps/v1beta1
+        k8s.io/client-go/applyconfigurations/apps/v1beta2            from k8s.io/client-go/kubernetes/typed/apps/v1beta2
+        k8s.io/client-go/applyconfigurations/autoscaling/v1          from k8s.io/client-go/kubernetes/typed/apps/v1+
+        k8s.io/client-go/applyconfigurations/autoscaling/v2          from k8s.io/client-go/kubernetes/typed/autoscaling/v2
+        k8s.io/client-go/applyconfigurations/autoscaling/v2beta1     from k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1
+        k8s.io/client-go/applyconfigurations/autoscaling/v2beta2     from k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2
+        k8s.io/client-go/applyconfigurations/batch/v1                from k8s.io/client-go/applyconfigurations/batch/v1beta1+
+        k8s.io/client-go/applyconfigurations/batch/v1beta1           from k8s.io/client-go/kubernetes/typed/batch/v1beta1
+        k8s.io/client-go/applyconfigurations/certificates/v1         from k8s.io/client-go/kubernetes/typed/certificates/v1
+        k8s.io/client-go/applyconfigurations/certificates/v1alpha1   from k8s.io/client-go/kubernetes/typed/certificates/v1alpha1
+        k8s.io/client-go/applyconfigurations/certificates/v1beta1    from k8s.io/client-go/kubernetes/typed/certificates/v1beta1
+        k8s.io/client-go/applyconfigurations/coordination/v1         from k8s.io/client-go/kubernetes/typed/coordination/v1
+        k8s.io/client-go/applyconfigurations/coordination/v1beta1    from k8s.io/client-go/kubernetes/typed/coordination/v1beta1
+        k8s.io/client-go/applyconfigurations/core/v1                 from k8s.io/client-go/applyconfigurations/apps/v1+
+        k8s.io/client-go/applyconfigurations/discovery/v1            from k8s.io/client-go/kubernetes/typed/discovery/v1
+        k8s.io/client-go/applyconfigurations/discovery/v1beta1       from k8s.io/client-go/kubernetes/typed/discovery/v1beta1
+        k8s.io/client-go/applyconfigurations/events/v1               from k8s.io/client-go/kubernetes/typed/events/v1
+        k8s.io/client-go/applyconfigurations/events/v1beta1          from k8s.io/client-go/kubernetes/typed/events/v1beta1
+        k8s.io/client-go/applyconfigurations/extensions/v1beta1      from k8s.io/client-go/kubernetes/typed/extensions/v1beta1
+        k8s.io/client-go/applyconfigurations/flowcontrol/v1          from k8s.io/client-go/kubernetes/typed/flowcontrol/v1
+        k8s.io/client-go/applyconfigurations/flowcontrol/v1beta1     from k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1
+        k8s.io/client-go/applyconfigurations/flowcontrol/v1beta2     from k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2
+        k8s.io/client-go/applyconfigurations/flowcontrol/v1beta3     from k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3
+        k8s.io/client-go/applyconfigurations/internal                from k8s.io/client-go/applyconfigurations/admissionregistration/v1+
+        k8s.io/client-go/applyconfigurations/meta/v1                 from k8s.io/client-go/applyconfigurations/admissionregistration/v1+
+        k8s.io/client-go/applyconfigurations/networking/v1           from k8s.io/client-go/kubernetes/typed/networking/v1
+        k8s.io/client-go/applyconfigurations/networking/v1alpha1     from k8s.io/client-go/kubernetes/typed/networking/v1alpha1
+        k8s.io/client-go/applyconfigurations/networking/v1beta1      from k8s.io/client-go/kubernetes/typed/networking/v1beta1
+        k8s.io/client-go/applyconfigurations/node/v1                 from k8s.io/client-go/kubernetes/typed/node/v1
+        k8s.io/client-go/applyconfigurations/node/v1alpha1           from k8s.io/client-go/kubernetes/typed/node/v1alpha1
+        k8s.io/client-go/applyconfigurations/node/v1beta1            from k8s.io/client-go/kubernetes/typed/node/v1beta1
+        k8s.io/client-go/applyconfigurations/policy/v1               from k8s.io/client-go/kubernetes/typed/policy/v1
+        k8s.io/client-go/applyconfigurations/policy/v1beta1          from k8s.io/client-go/kubernetes/typed/policy/v1beta1
+        k8s.io/client-go/applyconfigurations/rbac/v1                 from k8s.io/client-go/kubernetes/typed/rbac/v1
+        k8s.io/client-go/applyconfigurations/rbac/v1alpha1           from k8s.io/client-go/kubernetes/typed/rbac/v1alpha1
+        k8s.io/client-go/applyconfigurations/rbac/v1beta1            from k8s.io/client-go/kubernetes/typed/rbac/v1beta1
+        k8s.io/client-go/applyconfigurations/resource/v1alpha2       from k8s.io/client-go/kubernetes/typed/resource/v1alpha2
+        k8s.io/client-go/applyconfigurations/scheduling/v1           from k8s.io/client-go/kubernetes/typed/scheduling/v1
+        k8s.io/client-go/applyconfigurations/scheduling/v1alpha1     from k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1
+        k8s.io/client-go/applyconfigurations/scheduling/v1beta1      from k8s.io/client-go/kubernetes/typed/scheduling/v1beta1
+        k8s.io/client-go/applyconfigurations/storage/v1              from k8s.io/client-go/kubernetes/typed/storage/v1
+        k8s.io/client-go/applyconfigurations/storage/v1alpha1        from k8s.io/client-go/kubernetes/typed/storage/v1alpha1
+        k8s.io/client-go/applyconfigurations/storage/v1beta1         from k8s.io/client-go/kubernetes/typed/storage/v1beta1
+        k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1
+        k8s.io/client-go/discovery                                   from k8s.io/client-go/applyconfigurations/meta/v1+
+        k8s.io/client-go/dynamic                                     from sigs.k8s.io/controller-runtime/pkg/cache/internal+
+        k8s.io/client-go/features                                    from k8s.io/client-go/tools/cache
+        k8s.io/client-go/kubernetes                                  from k8s.io/client-go/tools/leaderelection/resourcelock
+        k8s.io/client-go/kubernetes/scheme                           from k8s.io/client-go/discovery+
+        k8s.io/client-go/kubernetes/typed/admissionregistration/v1   from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/apiserverinternal/v1alpha1 from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/apps/v1                    from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/apps/v1beta1               from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/apps/v1beta2               from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/authentication/v1          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/authentication/v1alpha1    from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/authentication/v1beta1     from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/authorization/v1           from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/authorization/v1beta1      from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/autoscaling/v1             from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/autoscaling/v2             from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/batch/v1                   from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/batch/v1beta1              from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/certificates/v1            from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/certificates/v1alpha1      from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/certificates/v1beta1       from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/coordination/v1            from k8s.io/client-go/kubernetes+
+        k8s.io/client-go/kubernetes/typed/coordination/v1beta1       from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/core/v1                    from k8s.io/client-go/kubernetes+
+        k8s.io/client-go/kubernetes/typed/discovery/v1               from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/discovery/v1beta1          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/events/v1                  from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/events/v1beta1             from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/extensions/v1beta1         from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/flowcontrol/v1             from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/networking/v1              from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/networking/v1alpha1        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/networking/v1beta1         from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/node/v1                    from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/node/v1alpha1              from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/node/v1beta1               from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/policy/v1                  from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/policy/v1beta1             from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/rbac/v1                    from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/rbac/v1alpha1              from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/rbac/v1beta1               from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/resource/v1alpha2          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/scheduling/v1              from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1        from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/scheduling/v1beta1         from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/storage/v1                 from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/storage/v1alpha1           from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/storage/v1beta1            from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1  from k8s.io/client-go/kubernetes
+        k8s.io/client-go/metadata                                    from sigs.k8s.io/controller-runtime/pkg/cache/internal+
+        k8s.io/client-go/openapi                                     from k8s.io/client-go/discovery
+        k8s.io/client-go/pkg/apis/clientauthentication               from k8s.io/client-go/pkg/apis/clientauthentication/install+
+        k8s.io/client-go/pkg/apis/clientauthentication/install       from k8s.io/client-go/plugin/pkg/client/auth/exec
+     💣 k8s.io/client-go/pkg/apis/clientauthentication/v1            from k8s.io/client-go/pkg/apis/clientauthentication/install+
+     💣 k8s.io/client-go/pkg/apis/clientauthentication/v1beta1       from k8s.io/client-go/pkg/apis/clientauthentication/install+
+        k8s.io/client-go/pkg/version                                 from k8s.io/client-go/rest
+        k8s.io/client-go/plugin/pkg/client/auth/exec                 from k8s.io/client-go/rest
+        k8s.io/client-go/rest                                        from k8s.io/client-go/discovery+
+        k8s.io/client-go/rest/watch                                  from k8s.io/client-go/rest
+        k8s.io/client-go/restmapper                                  from sigs.k8s.io/controller-runtime/pkg/client/apiutil
+        k8s.io/client-go/tools/auth                                  from k8s.io/client-go/tools/clientcmd
+        k8s.io/client-go/tools/cache                                 from sigs.k8s.io/controller-runtime/pkg/cache+
+        k8s.io/client-go/tools/cache/synctrack                       from k8s.io/client-go/tools/cache
+        k8s.io/client-go/tools/clientcmd                             from sigs.k8s.io/controller-runtime/pkg/client/config
+        k8s.io/client-go/tools/clientcmd/api                         from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/tools/clientcmd/api/latest                  from k8s.io/client-go/tools/clientcmd
+     💣 k8s.io/client-go/tools/clientcmd/api/v1                      from k8s.io/client-go/tools/clientcmd/api/latest
+        k8s.io/client-go/tools/internal/events                       from k8s.io/client-go/tools/record
+        k8s.io/client-go/tools/leaderelection                        from sigs.k8s.io/controller-runtime/pkg/manager+
+        k8s.io/client-go/tools/leaderelection/resourcelock           from k8s.io/client-go/tools/leaderelection+
+        k8s.io/client-go/tools/metrics                               from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/tools/pager                                 from k8s.io/client-go/tools/cache
+        k8s.io/client-go/tools/record                                from sigs.k8s.io/controller-runtime/pkg/cluster+
+        k8s.io/client-go/tools/record/util                           from k8s.io/client-go/tools/record
+        k8s.io/client-go/tools/reference                             from k8s.io/client-go/kubernetes/typed/core/v1+
+        k8s.io/client-go/transport                                   from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/util/cert                                   from k8s.io/client-go/rest+
+        k8s.io/client-go/util/connrotation                           from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/util/flowcontrol                            from k8s.io/client-go/kubernetes+
+        k8s.io/client-go/util/homedir                                from k8s.io/client-go/tools/clientcmd
+        k8s.io/client-go/util/keyutil                                from k8s.io/client-go/util/cert
+        k8s.io/client-go/util/workqueue                              from k8s.io/client-go/transport+
+        k8s.io/klog/v2                                               from k8s.io/apimachinery/pkg/api/meta+
+        k8s.io/klog/v2/internal/buffer                               from k8s.io/klog/v2
+        k8s.io/klog/v2/internal/clock                                from k8s.io/klog/v2
+        k8s.io/klog/v2/internal/dbg                                  from k8s.io/klog/v2
+        k8s.io/klog/v2/internal/serialize                            from k8s.io/klog/v2
+        k8s.io/klog/v2/internal/severity                             from k8s.io/klog/v2+
+        k8s.io/klog/v2/internal/sloghandler                          from k8s.io/klog/v2
+        k8s.io/kube-openapi/pkg/cached                               from k8s.io/kube-openapi/pkg/handler3
+        k8s.io/kube-openapi/pkg/common                               from k8s.io/kube-openapi/pkg/handler3
+        k8s.io/kube-openapi/pkg/handler3                             from k8s.io/client-go/openapi
+        k8s.io/kube-openapi/pkg/internal                             from k8s.io/kube-openapi/pkg/spec3+
+        k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json from k8s.io/kube-openapi/pkg/internal+
+        k8s.io/kube-openapi/pkg/schemaconv                           from k8s.io/apimachinery/pkg/util/managedfields+
+        k8s.io/kube-openapi/pkg/spec3                                from k8s.io/client-go/openapi+
+        k8s.io/kube-openapi/pkg/util/proto                           from k8s.io/apimachinery/pkg/util/managedfields+
+        k8s.io/kube-openapi/pkg/validation/spec                      from k8s.io/apimachinery/pkg/util/managedfields+
+        k8s.io/utils/buffer                                          from k8s.io/client-go/tools/cache
+        k8s.io/utils/clock                                           from k8s.io/apimachinery/pkg/util/cache+
+        k8s.io/utils/clock/testing                                   from k8s.io/client-go/util/flowcontrol
+        k8s.io/utils/internal/third_party/forked/golang/net          from k8s.io/utils/net
+        k8s.io/utils/net                                             from k8s.io/apimachinery/pkg/util/net+
+        k8s.io/utils/pointer                                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
+        k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
+        k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
+        k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
+        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
+        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
+        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
+        sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
+        sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
+        sigs.k8s.io/controller-runtime/pkg/cache/internal            from sigs.k8s.io/controller-runtime/pkg/cache
+        sigs.k8s.io/controller-runtime/pkg/certwatcher               from sigs.k8s.io/controller-runtime/pkg/metrics/server+
+        sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics       from sigs.k8s.io/controller-runtime/pkg/certwatcher
+        sigs.k8s.io/controller-runtime/pkg/client                    from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/client/apiutil            from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/client/config             from tailscale.com/cmd/k8s-operator
+        sigs.k8s.io/controller-runtime/pkg/cluster                   from sigs.k8s.io/controller-runtime/pkg/manager
+        sigs.k8s.io/controller-runtime/pkg/config                    from sigs.k8s.io/controller-runtime/pkg/manager
+        sigs.k8s.io/controller-runtime/pkg/controller                from sigs.k8s.io/controller-runtime/pkg/builder
+        sigs.k8s.io/controller-runtime/pkg/conversion                from sigs.k8s.io/controller-runtime/pkg/webhook/conversion
+        sigs.k8s.io/controller-runtime/pkg/event                     from sigs.k8s.io/controller-runtime/pkg/handler+
+        sigs.k8s.io/controller-runtime/pkg/handler                   from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/healthz                   from sigs.k8s.io/controller-runtime/pkg/manager+
+        sigs.k8s.io/controller-runtime/pkg/internal/controller       from sigs.k8s.io/controller-runtime/pkg/controller
+        sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics from sigs.k8s.io/controller-runtime/pkg/internal/controller
+        sigs.k8s.io/controller-runtime/pkg/internal/field/selector   from sigs.k8s.io/controller-runtime/pkg/cache/internal
+        sigs.k8s.io/controller-runtime/pkg/internal/httpserver       from sigs.k8s.io/controller-runtime/pkg/manager+
+        sigs.k8s.io/controller-runtime/pkg/internal/log              from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/internal/recorder         from sigs.k8s.io/controller-runtime/pkg/cluster+
+        sigs.k8s.io/controller-runtime/pkg/internal/source           from sigs.k8s.io/controller-runtime/pkg/source
+        sigs.k8s.io/controller-runtime/pkg/internal/syncs            from sigs.k8s.io/controller-runtime/pkg/cache/internal
+        sigs.k8s.io/controller-runtime/pkg/leaderelection            from sigs.k8s.io/controller-runtime/pkg/manager
+        sigs.k8s.io/controller-runtime/pkg/log                       from sigs.k8s.io/controller-runtime/pkg/client+
+        sigs.k8s.io/controller-runtime/pkg/log/zap                   from tailscale.com/cmd/k8s-operator
+        sigs.k8s.io/controller-runtime/pkg/manager                   from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/manager/signals           from tailscale.com/cmd/k8s-operator
+        sigs.k8s.io/controller-runtime/pkg/metrics                   from sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics+
+        sigs.k8s.io/controller-runtime/pkg/metrics/server            from sigs.k8s.io/controller-runtime/pkg/manager
+        sigs.k8s.io/controller-runtime/pkg/predicate                 from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/ratelimiter               from sigs.k8s.io/controller-runtime/pkg/controller+
+        sigs.k8s.io/controller-runtime/pkg/reconcile                 from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/recorder                  from sigs.k8s.io/controller-runtime/pkg/leaderelection+
+        sigs.k8s.io/controller-runtime/pkg/source                    from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/webhook                   from sigs.k8s.io/controller-runtime/pkg/manager
+        sigs.k8s.io/controller-runtime/pkg/webhook/admission         from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/webhook/conversion        from sigs.k8s.io/controller-runtime/pkg/builder
+        sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics  from sigs.k8s.io/controller-runtime/pkg/webhook+
+        sigs.k8s.io/json                                             from k8s.io/apimachinery/pkg/runtime/serializer/json+
+        sigs.k8s.io/json/internal/golang/encoding/json               from sigs.k8s.io/json
+     💣 sigs.k8s.io/structured-merge-diff/v4/fieldpath               from k8s.io/apimachinery/pkg/util/managedfields+
+        sigs.k8s.io/structured-merge-diff/v4/merge                   from k8s.io/apimachinery/pkg/util/managedfields/internal
+        sigs.k8s.io/structured-merge-diff/v4/schema                  from k8s.io/apimachinery/pkg/util/managedfields+
+        sigs.k8s.io/structured-merge-diff/v4/typed                   from k8s.io/apimachinery/pkg/util/managedfields+
+        sigs.k8s.io/structured-merge-diff/v4/value                   from k8s.io/apimachinery/pkg/runtime+
+        sigs.k8s.io/yaml                                             from k8s.io/apimachinery/pkg/runtime/serializer/json+
+        sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml
+        tailscale.com                                                from tailscale.com/version
+        tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
+        tailscale.com/atomicfile                                     from tailscale.com/ipn+
+        tailscale.com/client/tailscale                               from tailscale.com/client/web+
+        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
+        tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
+        tailscale.com/clientupdate                                   from tailscale.com/client/web+
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+        tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
+        tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
+        tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
+        tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
+        tailscale.com/derp                                           from tailscale.com/derp/derphttp+
+        tailscale.com/derp/derphttp                                  from tailscale.com/ipn/localapi+
+        tailscale.com/disco                                          from tailscale.com/derp+
+        tailscale.com/doctor                                         from tailscale.com/ipn/ipnlocal
+        tailscale.com/doctor/ethtool                                 from tailscale.com/ipn/ipnlocal
+     💣 tailscale.com/doctor/permissions                             from tailscale.com/ipn/ipnlocal
+        tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
+        tailscale.com/drive                                          from tailscale.com/client/tailscale+
+        tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/health                                         from tailscale.com/control/controlclient+
+        tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
+        tailscale.com/hostinfo                                       from tailscale.com/client/web+
+        tailscale.com/internal/noiseconn                             from tailscale.com/control/controlclient
+        tailscale.com/ipn                                            from tailscale.com/client/tailscale+
+        tailscale.com/ipn/conffile                                   from tailscale.com/ipn/ipnlocal+
+     💣 tailscale.com/ipn/ipnauth                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/localapi+
+        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/ipn/localapi                                   from tailscale.com/tsnet
+        tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
+        tailscale.com/ipn/store                                      from tailscale.com/ipn/ipnlocal+
+   L    tailscale.com/ipn/store/awsstore                             from tailscale.com/ipn/store
+        tailscale.com/ipn/store/kubestore                            from tailscale.com/cmd/k8s-operator+
+        tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/ipnlocal+
+        tailscale.com/k8s-operator                                   from tailscale.com/cmd/k8s-operator
+        tailscale.com/k8s-operator/apis                              from tailscale.com/k8s-operator/apis/v1alpha1
+        tailscale.com/k8s-operator/apis/v1alpha1                     from tailscale.com/cmd/k8s-operator+
+        tailscale.com/k8s-operator/sessionrecording                  from tailscale.com/cmd/k8s-operator
+        tailscale.com/k8s-operator/sessionrecording/conn             from tailscale.com/k8s-operator/sessionrecording/spdy
+        tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
+        tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
+        tailscale.com/kube                                           from tailscale.com/cmd/k8s-operator+
+        tailscale.com/licenses                                       from tailscale.com/client/web
+        tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
+        tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
+        tailscale.com/logpolicy                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/logtail                                        from tailscale.com/control/controlclient+
+        tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
+        tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
+        tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
+        tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns+
+        tailscale.com/net/dns/recursive                              from tailscale.com/net/dnsfallback
+        tailscale.com/net/dns/resolvconffile                         from tailscale.com/cmd/k8s-operator+
+        tailscale.com/net/dns/resolver                               from tailscale.com/net/dns
+        tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient+
+        tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
+        tailscale.com/net/ipset                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/memnet                                     from tailscale.com/tsnet
+        tailscale.com/net/netaddr                                    from tailscale.com/ipn+
+        tailscale.com/net/netcheck                                   from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/neterror                                   from tailscale.com/net/dns/resolver+
+        tailscale.com/net/netkernelconf                              from tailscale.com/ipn/ipnlocal
+        tailscale.com/net/netknob                                    from tailscale.com/logpolicy+
+     💣 tailscale.com/net/netmon                                     from tailscale.com/control/controlclient+
+     💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
+   W 💣 tailscale.com/net/netstat                                    from tailscale.com/portlist
+        tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
+        tailscale.com/net/packet                                     from tailscale.com/net/connstats+
+        tailscale.com/net/packet/checksum                            from tailscale.com/net/tstun
+        tailscale.com/net/ping                                       from tailscale.com/net/netcheck+
+        tailscale.com/net/portmapper                                 from tailscale.com/ipn/localapi+
+        tailscale.com/net/proxymux                                   from tailscale.com/tsnet
+        tailscale.com/net/routetable                                 from tailscale.com/doctor/routetable
+        tailscale.com/net/socks5                                     from tailscale.com/tsnet
+        tailscale.com/net/sockstats                                  from tailscale.com/control/controlclient+
+        tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
+   L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
+        tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
+        tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
+        tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
+     💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
+        tailscale.com/net/tstun                                      from tailscale.com/tsd+
+        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
+        tailscale.com/omit                                           from tailscale.com/ipn/conffile
+        tailscale.com/paths                                          from tailscale.com/client/tailscale+
+     💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
+        tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
+        tailscale.com/proxymap                                       from tailscale.com/tsd+
+     💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
+        tailscale.com/sessionrecording                               from tailscale.com/cmd/k8s-operator+
+        tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
+        tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
+        tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
+        tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
+        tailscale.com/tka                                            from tailscale.com/client/tailscale+
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
+        tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
+        tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
+        tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
+        tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
+        tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
+        tailscale.com/types/empty                                    from tailscale.com/ipn+
+        tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
+        tailscale.com/types/key                                      from tailscale.com/client/tailscale+
+        tailscale.com/types/lazy                                     from tailscale.com/ipn/ipnlocal+
+        tailscale.com/types/logger                                   from tailscale.com/appc+
+        tailscale.com/types/logid                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/types/netlogtype                               from tailscale.com/net/connstats+
+        tailscale.com/types/netmap                                   from tailscale.com/control/controlclient+
+        tailscale.com/types/nettype                                  from tailscale.com/ipn/localapi+
+        tailscale.com/types/opt                                      from tailscale.com/client/tailscale+
+        tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
+        tailscale.com/types/preftype                                 from tailscale.com/ipn+
+        tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
+        tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
+        tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
+        tailscale.com/types/views                                    from tailscale.com/appc+
+        tailscale.com/util/cibuild                                   from tailscale.com/health
+        tailscale.com/util/clientmetric                              from tailscale.com/cmd/k8s-operator+
+        tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
+        tailscale.com/util/cmpver                                    from tailscale.com/clientupdate+
+        tailscale.com/util/ctxkey                                    from tailscale.com/cmd/k8s-operator+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
+   L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics+
+        tailscale.com/util/dnsname                                   from tailscale.com/appc+
+        tailscale.com/util/execqueue                                 from tailscale.com/appc+
+        tailscale.com/util/goroutines                                from tailscale.com/ipn/ipnlocal
+        tailscale.com/util/groupmember                               from tailscale.com/client/web+
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
+        tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
+        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+   L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
+        tailscale.com/util/mak                                       from tailscale.com/appc+
+        tailscale.com/util/multierr                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/must                                      from tailscale.com/clientupdate/distsign+
+        tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+     💣 tailscale.com/util/osdiag                                    from tailscale.com/ipn/localapi
+   W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
+        tailscale.com/util/osshare                                   from tailscale.com/ipn/ipnlocal
+        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/progresstracking                          from tailscale.com/ipn/localapi
+        tailscale.com/util/race                                      from tailscale.com/net/dns/resolver
+        tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
+        tailscale.com/util/rands                                     from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/ringbuffer                                from tailscale.com/wgengine/magicsock
+        tailscale.com/util/set                                       from tailscale.com/cmd/k8s-operator+
+        tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
+        tailscale.com/util/slicesx                                   from tailscale.com/appc+
+        tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
+        tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
+        tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
+        tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
+        tailscale.com/util/truncate                                  from tailscale.com/logtail
+        tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
+     💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
+   W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
+   W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
+   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
+        tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+
+        tailscale.com/version                                        from tailscale.com/client/web+
+        tailscale.com/version/distro                                 from tailscale.com/client/web+
+        tailscale.com/wgengine                                       from tailscale.com/ipn/ipnlocal+
+        tailscale.com/wgengine/capture                               from tailscale.com/ipn/ipnlocal+
+        tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
+        tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap+
+     💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
+        tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
+        tailscale.com/wgengine/netstack                              from tailscale.com/tsnet
+        tailscale.com/wgengine/router                                from tailscale.com/ipn/ipnlocal+
+        tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
+        tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
+     💣 tailscale.com/wgengine/wgint                                 from tailscale.com/wgengine+
+        tailscale.com/wgengine/wglog                                 from tailscale.com/wgengine
+   W 💣 tailscale.com/wgengine/winnet                                from tailscale.com/wgengine/router
+        golang.org/x/crypto/argon2                                   from tailscale.com/tka
+        golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
+        golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
+  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf
+        golang.org/x/crypto/chacha20                                 from github.com/tailscale/golang-x-crypto/ssh+
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
+        golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
+        golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
+        golang.org/x/crypto/curve25519                               from github.com/tailscale/golang-x-crypto/ssh+
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
+        golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
+        golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
+        golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
+        golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
+        golang.org/x/net/bpf                                         from github.com/mdlayher/genetlink+
+        golang.org/x/net/dns/dnsmessage                              from net+
+        golang.org/x/net/http/httpguts                               from golang.org/x/net/http2+
+        golang.org/x/net/http/httpproxy                              from net/http+
+        golang.org/x/net/http2                                       from golang.org/x/net/http2/h2c+
+        golang.org/x/net/http2/h2c                                   from tailscale.com/ipn/ipnlocal
+        golang.org/x/net/http2/hpack                                 from golang.org/x/net/http2+
+        golang.org/x/net/icmp                                        from github.com/prometheus-community/pro-bing+
+        golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
+        golang.org/x/net/ipv4                                        from github.com/miekg/dns+
+        golang.org/x/net/ipv6                                        from github.com/miekg/dns+
+        golang.org/x/net/proxy                                       from tailscale.com/net/netns
+   D    golang.org/x/net/route                                       from net+
+        golang.org/x/oauth2                                          from golang.org/x/oauth2/clientcredentials+
+        golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
+        golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+
+        golang.org/x/sync/errgroup                                   from github.com/mdlayher/socket+
+        golang.org/x/sys/cpu                                         from github.com/josharian/native+
+  LD    golang.org/x/sys/unix                                        from github.com/fsnotify/fsnotify+
+   W    golang.org/x/sys/windows                                     from github.com/dblohm7/wingoes+
+   W    golang.org/x/sys/windows/registry                            from github.com/dblohm7/wingoes+
+   W    golang.org/x/sys/windows/svc                                 from golang.org/x/sys/windows/svc/mgr+
+   W    golang.org/x/sys/windows/svc/mgr                             from tailscale.com/util/winutil
+        golang.org/x/term                                            from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        golang.org/x/text/secure/bidirule                            from golang.org/x/net/idna
+        golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
+        golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
+        golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
+        golang.org/x/time/rate                                       from gvisor.dev/gvisor/pkg/log+
+        archive/tar                                                  from tailscale.com/clientupdate
+        bufio                                                        from compress/flate+
+        bytes                                                        from archive/tar+
+        cmp                                                          from github.com/gaissmai/bart+
+        compress/flate                                               from compress/gzip+
+        compress/gzip                                                from github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding+
+        compress/zlib                                                from debug/pe+
+        container/heap                                               from gvisor.dev/gvisor/pkg/tcpip/transport/tcp+
+        container/list                                               from crypto/tls+
+        context                                                      from crypto/tls+
+        crypto                                                       from crypto/ecdh+
+        crypto/aes                                                   from crypto/ecdsa+
+        crypto/cipher                                                from crypto/aes+
+        crypto/des                                                   from crypto/tls+
+        crypto/dsa                                                   from crypto/x509+
+        crypto/ecdh                                                  from crypto/ecdsa+
+        crypto/ecdsa                                                 from crypto/tls+
+        crypto/ed25519                                               from crypto/tls+
+        crypto/elliptic                                              from crypto/ecdsa+
+        crypto/hmac                                                  from crypto/tls+
+        crypto/md5                                                   from crypto/tls+
+        crypto/rand                                                  from crypto/ed25519+
+        crypto/rc4                                                   from crypto/tls+
+        crypto/rsa                                                   from crypto/tls+
+        crypto/sha1                                                  from crypto/tls+
+        crypto/sha256                                                from crypto/tls+
+        crypto/sha512                                                from crypto/ecdsa+
+        crypto/subtle                                                from crypto/aes+
+        crypto/tls                                                   from github.com/aws/aws-sdk-go-v2/aws/transport/http+
+        crypto/x509                                                  from crypto/tls+
+        crypto/x509/pkix                                             from crypto/x509+
+        database/sql                                                 from github.com/prometheus/client_golang/prometheus/collectors
+        database/sql/driver                                          from database/sql+
+   W    debug/dwarf                                                  from debug/pe
+   W    debug/pe                                                     from github.com/dblohm7/wingoes/pe
+        embed                                                        from crypto/internal/nistec+
+        encoding                                                     from encoding/gob+
+        encoding/asn1                                                from crypto/x509+
+        encoding/base32                                              from github.com/fxamacker/cbor/v2+
+        encoding/base64                                              from encoding/json+
+        encoding/binary                                              from compress/gzip+
+        encoding/csv                                                 from github.com/spf13/pflag
+        encoding/gob                                                 from github.com/gorilla/securecookie
+        encoding/hex                                                 from crypto/x509+
+        encoding/json                                                from expvar+
+        encoding/pem                                                 from crypto/tls+
+        encoding/xml                                                 from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+
+        errors                                                       from archive/tar+
+        expvar                                                       from github.com/prometheus/client_golang/prometheus+
+        flag                                                         from github.com/spf13/pflag+
+        fmt                                                          from archive/tar+
+        go/ast                                                       from go/doc+
+        go/build/constraint                                          from go/parser
+        go/doc                                                       from k8s.io/apimachinery/pkg/runtime
+        go/doc/comment                                               from go/doc
+        go/parser                                                    from k8s.io/apimachinery/pkg/runtime
+        go/scanner                                                   from go/ast+
+        go/token                                                     from go/ast+
+        hash                                                         from compress/zlib+
+        hash/adler32                                                 from compress/zlib+
+        hash/crc32                                                   from compress/gzip+
+        hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
+        hash/maphash                                                 from go4.org/mem
+        html                                                         from html/template+
+        html/template                                                from github.com/gorilla/csrf
+        io                                                           from archive/tar+
+        io/fs                                                        from archive/tar+
+        io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
+        log                                                          from expvar+
+        log/internal                                                 from log+
+        log/slog                                                     from github.com/go-logr/logr+
+        log/slog/internal                                            from log/slog
+        maps                                                         from sigs.k8s.io/controller-runtime/pkg/predicate+
+        math                                                         from archive/tar+
+        math/big                                                     from crypto/dsa+
+        math/bits                                                    from compress/flate+
+        math/rand                                                    from github.com/google/go-cmp/cmp+
+        math/rand/v2                                                 from tailscale.com/derp+
+        mime                                                         from github.com/prometheus/common/expfmt+
+        mime/multipart                                               from github.com/go-openapi/swag+
+        mime/quotedprintable                                         from mime/multipart
+        net                                                          from crypto/tls+
+        net/http                                                     from expvar+
+        net/http/httptest                                            from tailscale.com/control/controlclient
+        net/http/httptrace                                           from github.com/prometheus-community/pro-bing+
+        net/http/httputil                                            from github.com/aws/smithy-go/transport/http+
+        net/http/internal                                            from net/http+
+        net/http/pprof                                               from sigs.k8s.io/controller-runtime/pkg/manager+
+        net/netip                                                    from github.com/gaissmai/bart+
+        net/textproto                                                from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
+        net/url                                                      from crypto/x509+
+        os                                                           from crypto/rand+
+        os/exec                                                      from github.com/aws/aws-sdk-go-v2/credentials/processcreds+
+        os/signal                                                    from sigs.k8s.io/controller-runtime/pkg/manager/signals
+        os/user                                                      from archive/tar+
+        path                                                         from archive/tar+
+        path/filepath                                                from archive/tar+
+        reflect                                                      from archive/tar+
+        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints+
+        regexp/syntax                                                from regexp
+        runtime/debug                                                from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+
+        runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
+        runtime/pprof                                                from net/http/pprof+
+        runtime/trace                                                from net/http/pprof
+        slices                                                       from encoding/base32+
+        sort                                                         from archive/tar+
+        strconv                                                      from archive/tar+
+        strings                                                      from archive/tar+
+        sync                                                         from archive/tar+
+        sync/atomic                                                  from context+
+        syscall                                                      from archive/tar+
+        text/tabwriter                                               from k8s.io/apimachinery/pkg/util/diff+
+        text/template                                                from html/template
+        text/template/parse                                          from html/template+
+        time                                                         from archive/tar+
+        unicode                                                      from bytes+
+        unicode/utf16                                                from crypto/x509+
+        unicode/utf8                                                 from bufio+

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -21,6 +21,9 @@ spec:
       {{- end }}
       labels:
         app: operator
+        {{- with .Values.operatorConfig.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -46,7 +49,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           {{- $operatorTag:= printf ":%s" ( .Values.operatorConfig.image.tag | default .Chart.AppVersion )}}
-          image: {{ .Values.operatorConfig.image.repo }}{{- if .Values.operatorConfig.image.digest -}}{{ printf "@%s" .Values.operatorConfig.image.digest}}{{- else -}}{{ printf "%s" $operatorTag }}{{- end }}
+          image: {{ coalesce .Values.operatorConfig.image.repo .Values.operatorConfig.image.repository }}{{- if .Values.operatorConfig.image.digest -}}{{ printf "@%s" .Values.operatorConfig.image.digest}}{{- else -}}{{ printf "%s" $operatorTag }}{{- end }}
           imagePullPolicy: {{ .Values.operatorConfig.image.pullPolicy }}
           env:
             - name: OPERATOR_INITIAL_TAGS
@@ -67,13 +70,16 @@ spec:
               value: /oauth/client_secret
             {{- $proxyTag := printf ":%s" ( .Values.proxyConfig.image.tag | default .Chart.AppVersion )}}
             - name: PROXY_IMAGE
-              value: {{ .Values.proxyConfig.image.repo }}{{- if .Values.proxyConfig.image.digest -}}{{ printf "@%s" .Values.proxyConfig.image.digest}}{{- else -}}{{ printf "%s" $proxyTag }}{{- end }}
+              value: {{ coalesce .Values.proxyConfig.image.repo .Values.proxyConfig.image.repository }}{{- if .Values.proxyConfig.image.digest -}}{{ printf "@%s" .Values.proxyConfig.image.digest}}{{- else -}}{{ printf "%s" $proxyTag }}{{- end }}
             - name: PROXY_TAGS
               value: {{ .Values.proxyConfig.defaultTags }}
             - name: APISERVER_PROXY
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            {{- with .Values.operatorConfig.extraEnv }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
           volumeMounts:
           - name: oauth
             mountPath: /oauth

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -24,6 +24,9 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["dnsconfigs", "dnsconfigs/status"]
+  verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -45,11 +48,14 @@ metadata:
   namespace: {{ .Release.Namespace }}
 rules:
 - apiGroups: [""]
-  resources: ["secrets"]
+  resources: ["secrets", "serviceaccounts", "configmaps"]
   verbs: ["*"]
 - apiGroups: ["apps"]
-  resources: ["statefulsets"]
+  resources: ["statefulsets", "deployments"]
   verbs: ["*"]
+- apiGroups: ["discovery.k8s.io"]
+  resources: ["endpointslices"]
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -23,7 +23,8 @@ operatorConfig:
     - "tag:k8s-operator"
 
   image:
-    repo: tailscale/k8s-operator
+    # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/k8s-operator.
+    repository: tailscale/k8s-operator
     # Digest will be prioritized over tag. If neither are set appVersion will be
     # used.
     tag: ""
@@ -37,6 +38,7 @@ operatorConfig:
   resources: {}
 
   podAnnotations: {}
+  podLabels: {}
 
   tolerations: []
 
@@ -46,13 +48,25 @@ operatorConfig:
 
   securityContext: {}
 
+  extraEnv: []
+  # - name: EXTRA_VAR1
+  #   value: "value1"
+  # - name: EXTRA_VAR2
+  #   value: "value2"
+
+
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.
 # https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
 # https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
+# Note that this section contains only a few global configuration options and
+# will not be updated with more configuration options in the future.
+# If you need more configuration options, take a look at ProxyClass:
+# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
-    repo: tailscale/tailscale
+    # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
+    repository: tailscale/tailscale
     # Digest will be prioritized over tag. If neither are set appVersion will be
     # used.
     tag: ""

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.13.0
+    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
   name: connectors.tailscale.com
 spec:
   group: tailscale.com
@@ -31,47 +31,95 @@ spec:
       name: v1alpha1
       schema:
         openAPIV3Schema:
+          description: |-
+            Connector defines a Tailscale node that will be deployed in the cluster. The
+            node can be configured to act as a Tailscale subnet router and/or a Tailscale
+            exit node.
+            Connector is a cluster-scoped resource.
+            More info:
+            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
           type: object
           required:
             - spec
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
             spec:
-              description: ConnectorSpec describes the desired Tailscale component.
+              description: |-
+                ConnectorSpec describes the desired Tailscale component.
+                More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
                 exitNode:
-                  description: ExitNode defines whether the Connector node should act as a Tailscale exit node. Defaults to false. https://tailscale.com/kb/1103/exit-nodes
+                  description: |-
+                    ExitNode defines whether the Connector node should act as a
+                    Tailscale exit node. Defaults to false.
+                    https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
-                  description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
+                  description: |-
+                    Hostname is the tailnet hostname that should be assigned to the
+                    Connector node. If unset, hostname defaults to <connector
+                    name>-connector. Hostname can contain lower case letters, numbers and
+                    dashes, it must not start or end with a dash and must be between 2
+                    and 63 characters long.
                   type: string
                   pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
                 proxyClass:
-                  description: ProxyClass is the name of the ProxyClass custom resource that contains configuration options that should be applied to the resources created for this Connector. If unset, the operator will create resources with the default configuration.
+                  description: |-
+                    ProxyClass is the name of the ProxyClass custom resource that
+                    contains configuration options that should be applied to the
+                    resources created for this Connector. If unset, the operator will
+                    create resources with the default configuration.
                   type: string
                 subnetRouter:
-                  description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
+                  description: |-
+                    SubnetRouter defines subnet routes that the Connector node should
+                    expose to tailnet. If unset, none are exposed.
+                    https://tailscale.com/kb/1019/subnets/
                   type: object
                   required:
                     - advertiseRoutes
                   properties:
                     advertiseRoutes:
-                      description: AdvertiseRoutes refer to CIDRs that the subnet router should make available. Route values must be strings that represent a valid IPv4 or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. https://tailscale.com/kb/1201/4via6-subnets/
+                      description: |-
+                        AdvertiseRoutes refer to CIDRs that the subnet router should make
+                        available. Route values must be strings that represent a valid IPv4
+                        or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes.
+                        https://tailscale.com/kb/1201/4via6-subnets/
                       type: array
                       minItems: 1
                       items:
                         type: string
                         format: cidr
                 tags:
-                  description: Tags that the Tailscale node will be tagged with. Defaults to [tag:k8s]. To autoapprove the subnet routes or exit node defined by a Connector, you can configure Tailscale ACLs to give these tags the necessary permissions. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes. If you specify custom tags here, you must also make the operator an owner of these tags. See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector node has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                  description: |-
+                    Tags that the Tailscale node will be tagged with.
+                    Defaults to [tag:k8s].
+                    To autoapprove the subnet routes or exit node defined by a Connector,
+                    you can configure Tailscale ACLs to give these tags the necessary
+                    permissions.
+                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
+                    If you specify custom tags here, you must also make the operator an owner of these tags.
+                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+                    Tags cannot be changed once a Connector node has been created.
+                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
                   type: array
                   items:
                     type: string
@@ -80,48 +128,93 @@ spec:
                 - rule: has(self.subnetRouter) || self.exitNode == true
                   message: A Connector needs to be either an exit node or a subnet router, or both.
             status:
-              description: ConnectorStatus describes the status of the Connector. This is set and managed by the Tailscale operator.
+              description: |-
+                ConnectorStatus describes the status of the Connector. This is set
+                and managed by the Tailscale operator.
               type: object
               properties:
                 conditions:
-                  description: List of status conditions to indicate the status of the Connector. Known condition types are `ConnectorReady`.
+                  description: |-
+                    List of status conditions to indicate the status of the Connector.
+                    Known condition types are `ConnectorReady`.
                   type: array
                   items:
-                    description: ConnectorCondition contains condition information for a Connector.
+                    description: Condition contains details for one aspect of the current state of this API Resource.
                     type: object
                     required:
+                      - lastTransitionTime
+                      - message
+                      - reason
                       - status
                       - type
                     properties:
                       lastTransitionTime:
-                        description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
                         type: string
                         format: date-time
                       message:
-                        description: Message is a human readable description of the details of the last transition, complementing reason.
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
                         type: string
+                        maxLength: 32768
                       observedGeneration:
-                        description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
                         type: integer
                         format: int64
+                        minimum: 0
                       reason:
-                        description: Reason is a brief machine readable explanation for the condition's last transition.
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
                         type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
                       status:
-                        description: Status of the condition, one of ('True', 'False', 'Unknown').
+                        description: status of the condition, one of True, False, Unknown.
                         type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
                       type:
-                        description: Type of the condition, known values are (`SubnetRouterReady`).
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
                         type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                   x-kubernetes-list-map-keys:
                     - type
                   x-kubernetes-list-type: map
+                hostname:
+                  description: |-
+                    Hostname is the fully qualified domain name of the Connector node.
+                    If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+                    node.
+                  type: string
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean
                 subnetRoutes:
-                  description: SubnetRoutes are the routes currently exposed to tailnet via this Connector instance.
+                  description: |-
+                    SubnetRoutes are the routes currently exposed to tailnet via this
+                    Connector instance.
                   type: string
+                tailnetIPs:
+                  description: |-
+                    TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+                    assigned to the Connector node.
+                  type: array
+                  items:
+                    type: string
       served: true
       storage: true
       subresources:

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -0,0 +1,181 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+  name: dnsconfigs.tailscale.com
+spec:
+  group: tailscale.com
+  names:
+    kind: DNSConfig
+    listKind: DNSConfigList
+    plural: dnsconfigs
+    shortNames:
+      - dc
+    singular: dnsconfig
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Service IP address of the nameserver
+          jsonPath: .status.nameserver.ip
+          name: NameserverIP
+          type: string
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: |-
+            DNSConfig can be deployed to cluster to make a subset of Tailscale MagicDNS
+            names resolvable by cluster workloads. Use this if: A) you need to refer to
+            tailnet services, exposed to cluster via Tailscale Kubernetes operator egress
+            proxies by the MagicDNS names of those tailnet services (usually because the
+            services run over HTTPS)
+            B) you have exposed a cluster workload to the tailnet using Tailscale Ingress
+            and you also want to refer to the workload from within the cluster over the
+            Ingress's MagicDNS name (usually because you have some callback component
+            that needs to use the same URL as that used by a non-cluster client on
+            tailnet).
+            When a DNSConfig is applied to a cluster, Tailscale Kubernetes operator will
+            deploy a nameserver for ts.net DNS names and automatically populate it with records
+            for any Tailscale egress or Ingress proxies deployed to that cluster.
+            Currently you must manually update your cluster DNS configuration to add the
+            IP address of the deployed nameserver as a ts.net stub nameserver.
+            Instructions for how to do it:
+            https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configuration-of-stub-domain-and-upstream-nameserver-using-coredns (for CoreDNS),
+            https://cloud.google.com/kubernetes-engine/docs/how-to/kube-dns (for kube-dns).
+            Tailscale Kubernetes operator will write the address of a Service fronting
+            the nameserver to dsnconfig.status.nameserver.ip.
+            DNSConfig is a singleton - you must not create more than one.
+            NB: if you want cluster workloads to be able to refer to Tailscale Ingress
+            using its MagicDNS name, you must also annotate the Ingress resource with
+            tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation to
+            ensure that the proxy created for the Ingress listens on its Pod IP address.
+            NB: Clusters where Pods get assigned IPv6 addresses only are currently not supported.
+          type: object
+          required:
+            - spec
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: |-
+                Spec describes the desired DNS configuration.
+                More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
+              type: object
+              required:
+                - nameserver
+              properties:
+                nameserver:
+                  description: |-
+                    Configuration for a nameserver that can resolve ts.net DNS names
+                    associated with in-cluster proxies for Tailscale egress Services and
+                    Tailscale Ingresses. The operator will always deploy this nameserver
+                    when a DNSConfig is applied.
+                  type: object
+                  properties:
+                    image:
+                      description: Nameserver image.
+                      type: object
+                      properties:
+                        repo:
+                          description: Repo defaults to tailscale/k8s-nameserver.
+                          type: string
+                        tag:
+                          description: Tag defaults to operator's own tag.
+                          type: string
+            status:
+              description: |-
+                Status describes the status of the DNSConfig. This is set
+                and managed by the Tailscale operator.
+              type: object
+              properties:
+                conditions:
+                  type: array
+                  items:
+                    description: Condition contains details for one aspect of the current state of this API Resource.
+                    type: object
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        type: string
+                        format: date-time
+                      message:
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
+                        type: string
+                        maxLength: 32768
+                      observedGeneration:
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
+                        type: integer
+                        format: int64
+                        minimum: 0
+                      reason:
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
+                        type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                  x-kubernetes-list-map-keys:
+                    - type
+                  x-kubernetes-list-type: map
+                nameserver:
+                  description: Nameserver describes the status of nameserver cluster resources.
+                  type: object
+                  properties:
+                    ip:
+                      description: |-
+                        IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
+                        Currently you must manually update your cluster DNS config to add
+                        this address as a stub nameserver for ts.net for cluster workloads to be
+                        able to resolve MagicDNS names associated with egress or Ingress
+                        proxies.
+                        The IP address will change if you delete and recreate the DNSConfig.
+                      type: string
+      served: true
+      storage: true
+      subresources:
+        status: {}

cmd/k8s-operator/deploy/examples/dnsconfig.yaml

@@ -0,0 +1,6 @@
+apiVersion: tailscale.com/v1alpha1
+kind: DNSConfig 
+metadata:
+  name: ts-dns
+spec:
+  nameserver: {}

cmd/k8s-operator/deploy/examples/proxyclass.yaml

@@ -3,13 +3,21 @@ kind: ProxyClass
 metadata:
   name: prod
 spec:
+  metrics:
+    enable: true
   statefulSet:
     annotations:
-      platform-component: infra 
+      platform-component: infra
     pod:
       labels:
         team: eng
       nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
       imagePullSecrets:
       - name: "foo"
+      tailscaleContainer:
+        image: "ghcr.io/tailscale/tailscale:v1.64"
+        imagePullPolicy: IfNotPresent
+      tailscaleInitContainer:
+        image: "ghcr.io/tailscale/tailscale:v1.64"
+        imagePullPolicy: IfNotPresent

cmd/k8s-operator/deploy/manifests/nameserver/cm.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dnsrecords

cmd/k8s-operator/deploy/manifests/nameserver/deploy.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nameserver 
+spec:
+  replicas: 1
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels:
+      app: nameserver
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: nameserver
+    spec:
+      containers:
+      - imagePullPolicy: IfNotPresent
+        name: nameserver
+        ports:
+        - name: tcp
+          protocol: TCP
+          containerPort: 1053
+        - name: udp
+          protocol: UDP
+          containerPort: 1053
+        volumeMounts:
+        - name: dnsrecords
+          mountPath: /config
+      restartPolicy: Always
+      serviceAccount: nameserver
+      serviceAccountName: nameserver
+      volumes:
+      - name: dnsrecords
+        configMap:
+          name: dnsrecords

cmd/k8s-operator/deploy/manifests/nameserver/sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nameserver

cmd/k8s-operator/deploy/manifests/nameserver/svc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nameserver
+spec:
+  selector:
+    app: nameserver
+  ports:
+  - name: udp
+    targetPort: 1053
+    port: 53
+    protocol: UDP
+  - name: tcp
+    targetPort: 1053
+    port: 53
+    protocol: TCP 

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -14,10 +14,8 @@ spec:
         - name: sysctler
           securityContext:
             privileged: true
-          command: ["/bin/sh"]
-          args:
-            - -c
-            - sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1
+          command: ["/bin/sh", "-c"]
+          args: [sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi]
       resources:
         requests:
           cpu: 1m

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -20,3 +20,7 @@ spec:
           env:
             - name: TS_USERSPACE
               value: "true"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP

cmd/k8s-operator/dnsrecords.go

@@ -0,0 +1,334 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/net"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	operatorutils "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/util/mak"
+)
+
+const (
+	dnsRecordsRecocilerFinalizer = "tailscale.com/dns-records-reconciler"
+	annotationTSMagicDNSName     = "tailscale.com/magic-dnsname"
+)
+
+// dnsRecordsReconciler knows how to update dnsrecords ConfigMap with DNS
+// records.
+// The records that it creates are:
+//   - For tailscale Ingress, a mapping of the Ingress's MagicDNSName to the IP address of
+//     the ingress proxy Pod.
+//   - For egress proxies configured via tailscale.com/tailnet-fqdn annotation, a
+//     mapping of the tailnet FQDN to the IP address of the egress proxy Pod.
+//
+// Records will only be created if there is exactly one ready
+// tailscale.com/v1alpha1.DNSConfig instance in the cluster (so that we know
+// that there is a ts.net nameserver deployed in the cluster).
+type dnsRecordsReconciler struct {
+	client.Client
+	tsNamespace           string // namespace in which we provision tailscale resources
+	logger                *zap.SugaredLogger
+	isDefaultLoadBalancer bool // true if operator is the default ingress controller in this cluster
+}
+
+// Reconcile takes a reconcile.Request for a headless Service fronting a
+// tailscale proxy and updates DNS Records in dnsrecords ConfigMap for the
+// in-cluster ts.net nameserver if required.
+func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	logger := dnsRR.logger.With("Service", req.NamespacedName)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	headlessSvc := new(corev1.Service)
+	err = dnsRR.Client.Get(ctx, req.NamespacedName, headlessSvc)
+	if apierrors.IsNotFound(err) {
+		logger.Debugf("Service not found")
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get Service: %w", err)
+	}
+	if !(isManagedByType(headlessSvc, "svc") || isManagedByType(headlessSvc, "ingress")) {
+		logger.Debugf("Service is not a headless Service for a tailscale ingress or egress proxy; do nothing")
+		return reconcile.Result{}, nil
+	}
+
+	if !headlessSvc.DeletionTimestamp.IsZero() {
+		logger.Debug("Service is being deleted, clean up resources")
+		return reconcile.Result{}, dnsRR.maybeCleanup(ctx, headlessSvc, logger)
+	}
+
+	// Check that there is a ts.net nameserver deployed to the cluster by
+	// checking that there is tailscale.com/v1alpha1.DNSConfig resource in a
+	// Ready state.
+	dnsCfgLst := new(tsapi.DNSConfigList)
+	if err = dnsRR.List(ctx, dnsCfgLst); err != nil {
+		return reconcile.Result{}, fmt.Errorf("error listing DNSConfigs: %w", err)
+	}
+	if len(dnsCfgLst.Items) == 0 {
+		logger.Debugf("DNSConfig does not exist, not creating DNS records")
+		return reconcile.Result{}, nil
+	}
+	if len(dnsCfgLst.Items) > 1 {
+		logger.Errorf("Invalid cluster state - more than one DNSConfig found in cluster. Please ensure no more than one exists")
+		return reconcile.Result{}, nil
+	}
+	dnsCfg := dnsCfgLst.Items[0]
+	if !operatorutils.DNSCfgIsReady(&dnsCfg) {
+		logger.Info("DNSConfig is not ready yet, waiting...")
+		return reconcile.Result{}, nil
+	}
+
+	return reconcile.Result{}, dnsRR.maybeProvision(ctx, headlessSvc, logger)
+}
+
+// maybeProvision ensures that dnsrecords ConfigMap contains a record for the
+// proxy associated with the headless Service.
+// The record is only provisioned if the proxy is for a tailscale Ingress or
+// egress configured via tailscale.com/tailnet-fqdn annotation.
+//
+// For Ingress, the record is a mapping between the MagicDNSName of the Ingress, retrieved from
+// ingress.status.loadBalancer.ingress.hostname field and the proxy Pod IP addresses
+// retrieved from the EndpoinSlice associated with this headless Service, i.e
+// Records{IP4: <MagicDNS name of the Ingress>: <[IPs of the ingress proxy Pods]>}
+//
+// For egress, the record is a mapping between tailscale.com/tailnet-fqdn
+// annotation and the proxy Pod IP addresses, retrieved from the EndpointSlice
+// associated with this headless Service, i.e
+// Records{IP4: {<tailscale.com/tailnet-fqdn>: <[IPs of the egress proxy Pods]>}
+//
+// If records need to be created for this proxy, maybeProvision will also:
+// - update the headless Service with a tailscale.com/magic-dnsname annotation
+// - update the headless Service with a finalizer
+func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error {
+	if headlessSvc == nil {
+		logger.Info("[unexpected] maybeProvision called with a nil Service")
+		return nil
+	}
+	isEgressFQDNSvc, err := dnsRR.isSvcForFQDNEgressProxy(ctx, headlessSvc)
+	if err != nil {
+		return fmt.Errorf("error checking whether the Service is for an egress proxy: %w", err)
+	}
+	if !(isEgressFQDNSvc || isManagedByType(headlessSvc, "ingress")) {
+		logger.Debug("Service is not fronting a proxy that we create DNS records for; do nothing")
+		return nil
+	}
+	fqdn, err := dnsRR.fqdnForDNSRecord(ctx, headlessSvc, logger)
+	if err != nil {
+		return fmt.Errorf("error determining DNS name for record: %w", err)
+	}
+	if fqdn == "" {
+		logger.Debugf("MagicDNS name does not (yet) exist, not provisioning DNS record")
+		return nil // a new reconcile will be triggered once it's added
+	}
+
+	oldHeadlessSvc := headlessSvc.DeepCopy()
+	// Ensure that headless Service is annotated with a finalizer to help
+	// with records cleanup when proxy resources are deleted.
+	if !slices.Contains(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) {
+		headlessSvc.Finalizers = append(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
+	}
+	// Ensure that headless Service is annotated with the current MagicDNS
+	// name to help with records cleanup when proxy resources are deleted or
+	// MagicDNS name changes.
+	oldFqdn := headlessSvc.Annotations[annotationTSMagicDNSName]
+	if oldFqdn != "" && oldFqdn != fqdn { // i.e user has changed the value of tailscale.com/tailnet-fqdn annotation
+		logger.Debugf("MagicDNS name has changed, remvoving record for %s", oldFqdn)
+		updateFunc := func(rec *operatorutils.Records) {
+			delete(rec.IP4, oldFqdn)
+		}
+		if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
+			return fmt.Errorf("error removing record for %s: %w", oldFqdn, err)
+		}
+	}
+	mak.Set(&headlessSvc.Annotations, annotationTSMagicDNSName, fqdn)
+	if !apiequality.Semantic.DeepEqual(oldHeadlessSvc, headlessSvc) {
+		logger.Infof("provisioning DNS record for MagicDNS name: %s", fqdn) // this will be printed exactly once
+		if err := dnsRR.Update(ctx, headlessSvc); err != nil {
+			return fmt.Errorf("error updating proxy headless Service metadata: %w", err)
+		}
+	}
+
+	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
+	// headless Service.
+	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
+	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
+	if err != nil {
+		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
+	}
+	if eps == nil {
+		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
+		return nil
+	}
+	// An EndpointSlice for a Service can have a list of endpoints that each
+	// can have multiple addresses - these are the IP addresses of any Pods
+	// selected by that Service. Pick all the IPv4 addresses.
+	ips := make([]string, 0)
+	for _, ep := range eps.Endpoints {
+		for _, ip := range ep.Addresses {
+			if !net.IsIPv4String(ip) {
+				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+			} else {
+				ips = append(ips, ip)
+			}
+		}
+	}
+	if len(ips) == 0 {
+		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
+		return nil
+	}
+	updateFunc := func(rec *operatorutils.Records) {
+		mak.Set(&rec.IP4, fqdn, ips)
+	}
+	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
+		return fmt.Errorf("error updating DNS records: %w", err)
+	}
+	return nil
+}
+
+// maybeCleanup ensures that the DNS record for the proxy has been removed from
+// dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
+// has been removed from the Service. If the record is not found in the
+// ConfigMap, the ConfigMap does not exist, or the Service does not have
+// tailscale.com/magic-dnsname annotation, just remove the finalizer.
+func (h *dnsRecordsReconciler) maybeCleanup(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error {
+	ix := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
+	if ix == -1 {
+		logger.Debugf("no finalizer, nothing to do")
+		return nil
+	}
+	cm := &corev1.ConfigMap{}
+	err := h.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: h.tsNamespace}, cm)
+	if apierrors.IsNotFound(err) {
+		logger.Debug("'dsnrecords' ConfigMap not found")
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	if err != nil {
+		return fmt.Errorf("error retrieving 'dnsrecords' ConfigMap: %w", err)
+	}
+	if cm.Data == nil {
+		logger.Debug("'dnsrecords' ConfigMap contains no records")
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	_, ok := cm.Data[operatorutils.DNSRecordsCMKey]
+	if !ok {
+		logger.Debug("'dnsrecords' ConfigMap contains no records")
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	fqdn, _ := headlessSvc.GetAnnotations()[annotationTSMagicDNSName]
+	if fqdn == "" {
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	logger.Infof("removing DNS record for MagicDNS name %s", fqdn)
+	updateFunc := func(rec *operatorutils.Records) {
+		delete(rec.IP4, fqdn)
+	}
+	if err = h.updateDNSConfig(ctx, updateFunc); err != nil {
+		return fmt.Errorf("error updating DNS config: %w", err)
+	}
+	return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+}
+
+func (dnsRR *dnsRecordsReconciler) removeHeadlessSvcFinalizer(ctx context.Context, headlessSvc *corev1.Service) error {
+	idx := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
+	if idx == -1 {
+		return nil
+	}
+	headlessSvc.Finalizers = append(headlessSvc.Finalizers[:idx], headlessSvc.Finalizers[idx+1:]...)
+	return dnsRR.Update(ctx, headlessSvc)
+}
+
+// fqdnForDNSRecord returns MagicDNS name associated with a given headless Service.
+// If the headless Service is for a tailscale Ingress proxy, returns ingress.status.loadBalancer.ingress.hostname.
+// If the headless Service is for an tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value.
+// This function is not expected to be called with headless Services for other
+// proxy types, or any other Services, but it just returns an empty string if
+// that happens.
+func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) (string, error) {
+	parentName := parentFromObjectLabels(headlessSvc)
+	if isManagedByType(headlessSvc, "ingress") {
+		ing := new(networkingv1.Ingress)
+		if err := dnsRR.Get(ctx, parentName, ing); err != nil {
+			return "", err
+		}
+		if len(ing.Status.LoadBalancer.Ingress) == 0 {
+			return "", nil
+		}
+		return ing.Status.LoadBalancer.Ingress[0].Hostname, nil
+	}
+	if isManagedByType(headlessSvc, "svc") {
+		svc := new(corev1.Service)
+		if err := dnsRR.Get(ctx, parentName, svc); apierrors.IsNotFound(err) {
+			logger.Info("[unexpected] parent Service for egress proxy %s not found", headlessSvc.Name)
+			return "", nil
+		} else if err != nil {
+			return "", err
+		}
+		return svc.Annotations[AnnotationTailnetTargetFQDN], nil
+	}
+	return "", nil
+}
+
+// updateDNSConfig runs the provided update function against dnsrecords
+// ConfigMap. At this point the in-cluster ts.net nameserver is expected to be
+// successfully created together with the ConfigMap.
+func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error {
+	cm := &corev1.ConfigMap{}
+	err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm)
+	if apierrors.IsNotFound(err) {
+		dnsRR.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an isue and attach operator logs.")
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("error retrieving dnsrecords ConfigMap: %w", err)
+	}
+	dnsRecords := operatorutils.Records{Version: operatorutils.Alpha1Version, IP4: map[string][]string{}}
+	if cm.Data != nil && cm.Data[operatorutils.DNSRecordsCMKey] != "" {
+		if err := json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil {
+			return err
+		}
+	}
+	update(&dnsRecords)
+	dnsRecordsBs, err := json.Marshal(dnsRecords)
+	if err != nil {
+		return fmt.Errorf("error marshalling DNS records: %w", err)
+	}
+	mak.Set(&cm.Data, operatorutils.DNSRecordsCMKey, string(dnsRecordsBs))
+	return dnsRR.Update(ctx, cm)
+}
+
+// isSvcForFQDNEgressProxy returns true if the Service is a headless Service
+// created for a proxy for a tailscale egress Service configured via
+// tailscale.com/tailnet-fqdn annotation.
+func (dnsRR *dnsRecordsReconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Service) (bool, error) {
+	if !isManagedByType(svc, "svc") {
+		return false, nil
+	}
+	parentName := parentFromObjectLabels(svc)
+	parentSvc := new(corev1.Service)
+	if err := dnsRR.Get(ctx, parentName, parentSvc); apierrors.IsNotFound(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	annots := parentSvc.Annotations
+	return annots != nil && annots[AnnotationTailnetTargetFQDN] != "", nil
+}

cmd/k8s-operator/dnsrecords_test.go

@@ -0,0 +1,198 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	operatorutils "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
+)
+
+func TestDNSRecordsReconciler(t *testing.T) {
+	// Preconfigure a cluster with a DNSConfig
+	dnsConfig := &tsapi.DNSConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+		TypeMeta: metav1.TypeMeta{Kind: "DNSConfig"},
+		Spec: tsapi.DNSConfigSpec{
+			Nameserver: &tsapi.Nameserver{},
+		}}
+	ing := &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ts-ingress",
+			Namespace: "test",
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+		},
+		Status: networkingv1.IngressStatus{
+			LoadBalancer: networkingv1.IngressLoadBalancerStatus{
+				Ingress: []networkingv1.IngressLoadBalancerIngress{{
+					Hostname: "cluster.ingress.ts.net"}},
+			},
+		},
+	}
+	cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", Namespace: "tailscale"}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(cm).
+		WithObjects(dnsConfig).
+		WithObjects(ing).
+		WithStatusSubresource(dnsConfig, ing).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	// Set the ready condition of the DNSConfig
+	mustUpdateStatus[tsapi.DNSConfig](t, fc, "", "test", func(c *tsapi.DNSConfig) {
+		operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated, 0, cl, zl.Sugar())
+	})
+	dnsRR := &dnsRecordsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		tsNamespace: "tailscale",
+	}
+
+	// 1. DNS record is created for an egress proxy configured via
+	// tailscale.com/tailnet-fqdn annotation
+	egressSvcFQDN := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "egress-fqdn",
+			Namespace:   "test",
+			Annotations: map[string]string{"tailscale.com/tailnet-fqdn": "foo.bar.ts.net"},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "unused",
+			Type:         corev1.ServiceTypeExternalName,
+		},
+	}
+	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
+	mustCreate(t, fc, egressSvcFQDN)
+	mustCreate(t, fc, headlessForEgressSvcFQDN)
+	mustCreate(t, fc, ep)
+	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
+	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
+	// value changes
+	mustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) {
+		svc.Annotations["tailscale.com/tailnet-fqdn"] = "baz.bar.ts.net"
+	})
+	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
+	wantHosts = map[string][]string{"baz.bar.ts.net": {"10.9.8.7"}}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 3. DNS record is updated if the IP address of the proxy Pod changes.
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
+	})
+	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
+	wantHosts = map[string][]string{"baz.bar.ts.net": {"10.6.5.4"}}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 4. DNS record is created for an ingress proxy configured via Ingress
+	headlessForIngress := headlessSvcForParent(ing, "ingress")
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
+	mustCreate(t, fc, headlessForIngress)
+	mustCreate(t, fc, ep)
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
+	wantHosts["cluster.ingress.ts.net"] = []string{"10.9.8.7"}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 5. DNS records are updated if Ingress's MagicDNS name changes (i.e users changed spec.tls.hosts[0])
+	t.Log("test case 5")
+	mustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) {
+		ing.Status.LoadBalancer.Ingress[0].Hostname = "another.ingress.ts.net"
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
+	delete(wantHosts, "cluster.ingress.ts.net")
+	wantHosts["another.ingress.ts.net"] = []string{"10.9.8.7"}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 6. DNS records are updated if Ingress proxy's Pod IP changes
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Addresses = []string{"7.8.9.10"}
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
+	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
+	expectHostsRecords(t, fc, wantHosts)
+}
+
+func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      o.GetName(),
+			Namespace: "tailscale",
+			Labels: map[string]string{
+				LabelManaged:         "true",
+				LabelParentName:      o.GetName(),
+				LabelParentNamespace: o.GetNamespace(),
+				LabelParentType:      typ,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "None",
+			Type:      corev1.ServiceTypeClusterIP,
+			Selector:  map[string]string{"foo": "bar"},
+		},
+	}
+}
+
+func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
+	return &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      svc.Name,
+			Namespace: svc.Namespace,
+			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
+		},
+		Endpoints: []discoveryv1.Endpoint{{
+			Addresses: []string{ip},
+		}},
+	}
+}
+
+func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]string) {
+	t.Helper()
+	cm := new(corev1.ConfigMap)
+	if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
+		t.Fatalf("getting dnsconfig ConfigMap: %v", err)
+	}
+	if cm.Data == nil {
+		t.Fatal("dnsconfig ConfigMap has no data")
+	}
+	dnsConfigString, ok := cm.Data[operatorutils.DNSRecordsCMKey]
+	if !ok {
+		t.Fatal("dnsconfig ConfigMap does not contain dnsconfig")
+	}
+	dnsConfig := &operatorutils.Records{}
+	if err := json.Unmarshal([]byte(dnsConfigString), dnsConfig); err != nil {
+		t.Fatalf("unmarshaling dnsconfig: %v", err)
+	}
+	if diff := cmp.Diff(dnsConfig.IP4, wantsHosts); diff != "" {
+		t.Fatalf("unexpected dns config (-got +want):\n%s", diff)
+	}
+}

cmd/k8s-operator/generate/main.go

@@ -3,6 +3,7 @@
 
 //go:build !plan9
 
+// The generate command creates tailscale.com CRDs.
 package main
 
 import (
@@ -22,9 +23,11 @@ const (
 	operatorDeploymentFilesPath   = "cmd/k8s-operator/deploy"
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
+	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
+	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -36,10 +39,10 @@ func main() {
 	}
 	repoRoot := "../../"
 	switch os.Args[1] {
-	case "helmcrd": // insert CRD to Helm templates behind a installCRDs=true conditional check
-		log.Print("Adding Connector CRD to Helm templates")
+	case "helmcrd": // insert CRDs to Helm templates behind a installCRDs=true conditional check
+		log.Print("Adding CRDs to Helm templates")
 		if err := generate("./"); err != nil {
-			log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+			log.Fatalf("error adding CRDs to Helm templates: %v", err)
 		}
 		return
 	case "staticmanifests": // generate static manifests from Helm templates (including the CRD)
@@ -108,7 +111,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector and ProxyClass) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -140,6 +143,9 @@ func generate(baseDir string) error {
 	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
 		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
 	}
+	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
+		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
+	}
 	return nil
 }
 
@@ -151,5 +157,8 @@ func cleanup(baseDir string) error {
 	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
 	}
+	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
+	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -56,6 +56,9 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: proxyclasses.tailscale.com") {
 		t.Errorf("ProxyClass CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
+		t.Errorf("DNSConfig CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -71,4 +74,7 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") {
 		t.Errorf("ProxyClass CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
+		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/ingress.go

@@ -264,7 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		ServeConfig:         sc,
 		Tags:                tags,
 		ChildResourceLabels: crl,
-		ProxyClass:          proxyClass,
+		ProxyClassName:      proxyClass,
 	}
 
 	if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

cmd/k8s-operator/ingress_test.go

@@ -100,7 +100,7 @@ func TestTailscaleIngress(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
 
@@ -231,7 +231,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
 
@@ -248,9 +248,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	// created proxy resources.
 	mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
 		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []tsapi.ConnectorCondition{{
+			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               tsapi.ProxyClassready,
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/nameserver.go

@@ -0,0 +1,283 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sync"
+
+	_ "embed"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/yaml"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonNameserverCreationFailed  = "NameserverCreationFailed"
+	reasonMultipleDNSConfigsPresent = "MultipleDNSConfigsPresent"
+
+	reasonNameserverCreated = "NameserverCreated"
+
+	messageNameserverCreationFailed  = "Failed creating nameserver resources: %v"
+	messageMultipleDNSConfigsPresent = "Multiple DNSConfig resources found in cluster. Please ensure no more than one is present."
+
+	defaultNameserverImageRepo = "tailscale/k8s-nameserver"
+	// TODO (irbekrm): once we start publishing nameserver images for stable
+	// track, replace 'unstable' here with the version of this operator
+	// instance.
+	defaultNameserverImageTag = "unstable"
+)
+
+// NameserverReconciler knows how to create nameserver resources in cluster in
+// response to users applying DNSConfig.
+type NameserverReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+
+	mu                 sync.Mutex           // protects following
+	managedNameservers set.Slice[types.UID] // one or none
+}
+
+var (
+	gaugeNameserverResources = clientmetric.NewGauge("k8s_nameserver_resources")
+)
+
+func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	logger := a.logger.With("dnsConfig", req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	var dnsCfg tsapi.DNSConfig
+	err = a.Get(ctx, req.NamespacedName, &dnsCfg)
+	if apierrors.IsNotFound(err) {
+		// Request object not found, could have been deleted after reconcile request.
+		logger.Debugf("dnsconfig not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get dnsconfig: %w", err)
+	}
+	if !dnsCfg.DeletionTimestamp.IsZero() {
+		ix := xslices.Index(dnsCfg.Finalizers, FinalizerName)
+		if ix < 0 {
+			logger.Debugf("no finalizer, nothing to do")
+			return reconcile.Result{}, nil
+		}
+		logger.Info("Cleaning up DNSConfig resources")
+		if err := a.maybeCleanup(ctx, &dnsCfg, logger); err != nil {
+			logger.Errorf("error cleaning up reconciler resource: %v", err)
+			return res, err
+		}
+		dnsCfg.Finalizers = append(dnsCfg.Finalizers[:ix], dnsCfg.Finalizers[ix+1:]...)
+		if err := a.Update(ctx, &dnsCfg); err != nil {
+			logger.Errorf("error removing finalizer: %v", err)
+			return reconcile.Result{}, err
+		}
+		logger.Infof("Nameserver resources cleaned up")
+		return reconcile.Result{}, nil
+	}
+
+	oldCnStatus := dnsCfg.Status.DeepCopy()
+	setStatus := func(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+		tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, a.clock, logger)
+		if !apiequality.Semantic.DeepEqual(oldCnStatus, dnsCfg.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
+		}
+		return res, err
+	}
+	var dnsCfgs tsapi.DNSConfigList
+	if err := a.List(ctx, &dnsCfgs); err != nil {
+		return res, fmt.Errorf("error listing DNSConfigs: %w", err)
+	}
+	if len(dnsCfgs.Items) > 1 { // enforce DNSConfig to be a singleton
+		msg := "invalid cluster configuration: more than one tailscale.com/dnsconfigs found. Please ensure that no more than one is created."
+		logger.Error(msg)
+		a.recorder.Event(&dnsCfg, corev1.EventTypeWarning, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
+		setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
+	}
+
+	if !slices.Contains(dnsCfg.Finalizers, FinalizerName) {
+		logger.Infof("ensuring nameserver resources")
+		dnsCfg.Finalizers = append(dnsCfg.Finalizers, FinalizerName)
+		if err := a.Update(ctx, &dnsCfg); err != nil {
+			msg := fmt.Sprintf(messageNameserverCreationFailed, err)
+			logger.Error(msg)
+			return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
+		}
+	}
+	if err := a.maybeProvision(ctx, &dnsCfg, logger); err != nil {
+		return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
+	}
+
+	a.mu.Lock()
+	a.managedNameservers.Add(dnsCfg.UID)
+	a.mu.Unlock()
+	gaugeNameserverResources.Set(int64(a.managedNameservers.Len()))
+
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: a.tsNamespace},
+	}
+	if err := a.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil {
+		return res, fmt.Errorf("error getting Service: %w", err)
+	}
+	if ip := svc.Spec.ClusterIP; ip != "" && ip != "None" {
+		dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
+			IP: ip,
+		}
+		return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated)
+	}
+	logger.Info("nameserver Service does not have an IP address allocated, waiting...")
+	return reconcile.Result{}, nil
+}
+
+func nameserverResourceLabels(name, namespace string) map[string]string {
+	labels := childResourceLabels(name, namespace, "nameserver")
+	labels["app.kubernetes.io/name"] = "tailscale"
+	labels["app.kubernetes.io/component"] = "nameserver"
+	return labels
+}
+
+func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error {
+	labels := nameserverResourceLabels(tsDNSCfg.Name, a.tsNamespace)
+	dCfg := &deployConfig{
+		ownerRefs: []metav1.OwnerReference{*metav1.NewControllerRef(tsDNSCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))},
+		namespace: a.tsNamespace,
+		labels:    labels,
+		imageRepo: defaultNameserverImageRepo,
+		imageTag:  defaultNameserverImageTag,
+	}
+	if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Repo != "" {
+		dCfg.imageRepo = tsDNSCfg.Spec.Nameserver.Image.Repo
+	}
+	if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" {
+		dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag
+	}
+	for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} {
+		if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil {
+			return fmt.Errorf("error reconciling %s: %w", deployable.kind, err)
+		}
+	}
+	return nil
+}
+
+// maybeCleanup removes DNSConfig from being tracked. The cluster resources
+// created, will be automatically garbage collected as they are owned by the
+// DNSConfig.
+func (a *NameserverReconciler) maybeCleanup(ctx context.Context, dnsCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error {
+	a.mu.Lock()
+	a.managedNameservers.Remove(dnsCfg.UID)
+	a.mu.Unlock()
+	gaugeNameserverResources.Set(int64(a.managedNameservers.Len()))
+	return nil
+}
+
+type deployable struct {
+	kind      string
+	updateObj func(context.Context, *deployConfig, client.Client) error
+}
+
+type deployConfig struct {
+	imageRepo string
+	imageTag  string
+	labels    map[string]string
+	ownerRefs []metav1.OwnerReference
+	namespace string
+}
+
+var (
+	//go:embed deploy/manifests/nameserver/cm.yaml
+	cmYaml []byte
+	//go:embed deploy/manifests/nameserver/deploy.yaml
+	deployYaml []byte
+	//go:embed deploy/manifests/nameserver/sa.yaml
+	saYaml []byte
+	//go:embed deploy/manifests/nameserver/svc.yaml
+	svcYaml []byte
+
+	deployDeployable = deployable{
+		kind: "Deployment",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			d := new(appsv1.Deployment)
+			if err := yaml.Unmarshal(deployYaml, &d); err != nil {
+				return fmt.Errorf("error unmarshalling Deployment yaml: %w", err)
+			}
+			d.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", cfg.imageRepo, cfg.imageTag)
+			d.ObjectMeta.Namespace = cfg.namespace
+			d.ObjectMeta.Labels = cfg.labels
+			d.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			updateF := func(oldD *appsv1.Deployment) {
+				oldD.Spec = d.Spec
+			}
+			_, err := createOrUpdate[appsv1.Deployment](ctx, kubeClient, cfg.namespace, d, updateF)
+			return err
+		},
+	}
+	saDeployable = deployable{
+		kind: "ServiceAccount",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			sa := new(corev1.ServiceAccount)
+			if err := yaml.Unmarshal(saYaml, &sa); err != nil {
+				return fmt.Errorf("error unmarshalling ServiceAccount yaml: %w", err)
+			}
+			sa.ObjectMeta.Labels = cfg.labels
+			sa.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			sa.ObjectMeta.Namespace = cfg.namespace
+			_, err := createOrUpdate(ctx, kubeClient, cfg.namespace, sa, func(*corev1.ServiceAccount) {})
+			return err
+		},
+	}
+	svcDeployable = deployable{
+		kind: "Service",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			svc := new(corev1.Service)
+			if err := yaml.Unmarshal(svcYaml, &svc); err != nil {
+				return fmt.Errorf("error unmarshalling Service yaml: %w", err)
+			}
+			svc.ObjectMeta.Labels = cfg.labels
+			svc.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			svc.ObjectMeta.Namespace = cfg.namespace
+			_, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {})
+			return err
+		},
+	}
+	cmDeployable = deployable{
+		kind: "ConfigMap",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			cm := new(corev1.ConfigMap)
+			if err := yaml.Unmarshal(cmYaml, &cm); err != nil {
+				return fmt.Errorf("error unmarshalling ConfigMap yaml: %w", err)
+			}
+			cm.ObjectMeta.Labels = cfg.labels
+			cm.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			cm.ObjectMeta.Namespace = cfg.namespace
+			_, err := createOrUpdate[corev1.ConfigMap](ctx, kubeClient, cfg.namespace, cm, func(cm *corev1.ConfigMap) {})
+			return err
+		},
+	}
+)

cmd/k8s-operator/nameserver_test.go

@@ -0,0 +1,127 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// tailscale-operator provides a way to expose services running in a Kubernetes
+// cluster to your Tailnet and to make Tailscale nodes available to cluster
+// workloads
+package main
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/yaml"
+	operatorutils "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/util/mak"
+)
+
+func TestNameserverReconciler(t *testing.T) {
+	dnsCfg := &tsapi.DNSConfig{
+		TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+		Spec: tsapi.DNSConfigSpec{
+			Nameserver: &tsapi.Nameserver{
+				Image: &tsapi.Image{
+					Repo: "test",
+					Tag:  "v0.0.1",
+				},
+			},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(dnsCfg).
+		WithStatusSubresource(dnsCfg).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	nr := &NameserverReconciler{
+		Client:      fc,
+		clock:       cl,
+		logger:      zl.Sugar(),
+		tsNamespace: "tailscale",
+	}
+	expectReconciled(t, nr, "", "test")
+	// Verify that nameserver Deployment has been created and has the expected fields.
+	wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}}
+	if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil {
+		t.Fatalf("unmarshalling yaml: %v", err)
+	}
+	dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
+	wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef}
+	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1"
+	wantsDeploy.Namespace = "tailscale"
+	labels := nameserverResourceLabels("test", "tailscale")
+	wantsDeploy.ObjectMeta.Labels = labels
+	expectEqual(t, fc, wantsDeploy, nil)
+
+	// Verify that DNSConfig advertizes the nameserver's Service IP address,
+	// has the ready status condition and tailscale finalizer.
+	mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
+		svc.Spec.ClusterIP = "1.2.3.4"
+	})
+	expectReconciled(t, nr, "", "test")
+	dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
+		IP: "1.2.3.4",
+	}
+	dnsCfg.Finalizers = []string{FinalizerName}
+	dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{
+		Type:               string(tsapi.NameserverReady),
+		Status:             metav1.ConditionTrue,
+		Reason:             reasonNameserverCreated,
+		Message:            reasonNameserverCreated,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+	})
+	expectEqual(t, fc, dnsCfg, nil)
+
+	// // Verify that nameserver image gets updated to match DNSConfig spec.
+	mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
+		dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2"
+	})
+	expectReconciled(t, nr, "", "test")
+	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
+	expectEqual(t, fc, wantsDeploy, nil)
+
+	// Verify that when another actor sets ConfigMap data, it does not get
+	// overwritten by nameserver reconciler.
+	dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}}
+	bs, err := json.Marshal(dnsRecords)
+	if err != nil {
+		t.Fatalf("error marshalling ConfigMap contents: %v", err)
+	}
+	mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
+		mak.Set(&cm.Data, "records.json", string(bs))
+	})
+	expectReconciled(t, nr, "", "test")
+	wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords",
+		Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}},
+		TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+		Data:     map[string]string{"records.json": string(bs)},
+	}
+	expectEqual(t, fc, wantCm, nil)
+
+	// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
+	// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
+	mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
+		dnsCfg.Spec.Nameserver.Image = nil
+	})
+	expectReconciled(t, nr, "", "test")
+	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
+	expectEqual(t, fc, wantsDeploy, nil)
+}

cmd/k8s-operator/operator.go

@@ -20,6 +20,7 @@ import (
 	"golang.org/x/oauth2/clientcredentials"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
@@ -44,14 +45,14 @@ import (
 	"tailscale.com/version"
 )
 
-// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
-//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
-
 // Generate Connector and ProxyClass CustomResourceDefinition yamls from their Go types.
 //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen crd schemapatch:manifests=./deploy/crds output:dir=./deploy/crds paths=../../k8s-operator/apis/...
 
-// Generate CRD docs from the yamls
-//go:generate go run fybrik.io/crdoc --resources=./deploy/crds --output=../../k8s-operator/api.md
+// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
+//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
+
+// Generate CRD API docs.
+//go:generate go run github.com/elastic/crd-ref-docs --renderer=markdown --source-path=../../k8s-operator/apis/ --config=../../k8s-operator/api-docs-config.yaml --output-path=../../k8s-operator/api.md
 
 func main() {
 	// Required to use our client API. We're fine with the instability since the
@@ -59,12 +60,13 @@ func main() {
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	var (
-		tsNamespace       = defaultEnv("OPERATOR_NAMESPACE", "")
-		tslogging         = defaultEnv("OPERATOR_LOGGING", "info")
-		image             = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
-		priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
-		tags              = defaultEnv("PROXY_TAGS", "tag:k8s")
-		tsFirewallMode    = defaultEnv("PROXY_FIREWALL_MODE", "")
+		tsNamespace           = defaultEnv("OPERATOR_NAMESPACE", "")
+		tslogging             = defaultEnv("OPERATOR_LOGGING", "info")
+		image                 = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
+		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
+		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
+		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
 	var opts []kzap.Opts
@@ -93,9 +95,19 @@ func main() {
 	defer s.Close()
 	restConfig := config.GetConfigOrDie()
 	maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
-	// TODO (irbekrm): gather the reconciler options into an opts struct
-	// rather than passing a million of them in one by one.
-	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode)
+	rOpts := reconcilerOpts{
+		log:                           zlog,
+		tsServer:                      s,
+		tsClient:                      tsClient,
+		tailscaleNamespace:            tsNamespace,
+		restConfig:                    restConfig,
+		proxyImage:                    image,
+		proxyPriorityClassName:        priorityClassName,
+		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
+		proxyTags:                     tags,
+		proxyFirewallMode:             tsFirewallMode,
+	}
+	runReconcilers(rOpts)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -203,11 +215,8 @@ waitOnline:
 
 // runReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler. It blocks forever.
-func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) {
-	var (
-		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
-	)
-	startlog := zlog.Named("startReconcilers")
+func runReconcilers(opts reconcilerOpts) {
+	startlog := opts.log.Named("startReconcilers")
 	// For secrets and statefulsets, we only get permission to touch the objects
 	// in the controller's own namespace. This cannot be expressed by
 	// .Watches(...) below, instead you have to add a per-type field selector to
@@ -215,7 +224,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 	// implicitly filter what parts of the world the builder code gets to see at
 	// all.
 	nsFilter := cache.ByObject{
-		Field: client.InNamespace(tsNamespace).AsSelector(),
+		Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(),
 	}
 	mgrOpts := manager.Options{
 		// TODO (irbekrm): stricter filtering what we watch/cache/call
@@ -223,33 +232,37 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		// resources that we GET via the controller manager's client.
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
-				&corev1.Secret{}:      nsFilter,
-				&appsv1.StatefulSet{}: nsFilter,
+				&corev1.Secret{}:             nsFilter,
+				&corev1.ServiceAccount{}:     nsFilter,
+				&corev1.ConfigMap{}:          nsFilter,
+				&appsv1.StatefulSet{}:        nsFilter,
+				&appsv1.Deployment{}:         nsFilter,
+				&discoveryv1.EndpointSlice{}: nsFilter,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
 	}
-	mgr, err := manager.New(restConfig, mgrOpts)
+	mgr, err := manager.New(opts.restConfig, mgrOpts)
 	if err != nil {
 		startlog.Fatalf("could not create manager: %v", err)
 	}
 
 	svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler)
 	svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc"))
-	// If a ProxyClassChanges, enqueue all Services labeled with that
+	// If a ProxyClass changes, enqueue all Services labeled with that
 	// ProxyClass's name.
 	proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc(mgr.GetClient(), startlog))
 
 	eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
 	ssr := &tailscaleSTSReconciler{
 		Client:                 mgr.GetClient(),
-		tsnetServer:            s,
-		tsClient:               tsClient,
-		defaultTags:            strings.Split(tags, ","),
-		operatorNamespace:      tsNamespace,
-		proxyImage:             image,
-		proxyPriorityClassName: priorityClassName,
-		tsFirewallMode:         tsFirewallMode,
+		tsnetServer:            opts.tsServer,
+		tsClient:               opts.tsClient,
+		defaultTags:            strings.Split(opts.proxyTags, ","),
+		operatorNamespace:      opts.tailscaleNamespace,
+		proxyImage:             opts.proxyImage,
+		proxyPriorityClassName: opts.proxyPriorityClassName,
+		tsFirewallMode:         opts.proxyFirewallMode,
 	}
 	err = builder.
 		ControllerManagedBy(mgr).
@@ -261,9 +274,11 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Complete(&ServiceReconciler{
 			ssr:                   ssr,
 			Client:                mgr.GetClient(),
-			logger:                zlog.Named("service-reconciler"),
-			isDefaultLoadBalancer: isDefaultLoadBalancer,
+			logger:                opts.log.Named("service-reconciler"),
+			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
 			recorder:              eventRecorder,
+			tsNamespace:           opts.tailscaleNamespace,
+			clock:                 tstime.DefaultClock{},
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -285,7 +300,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			ssr:      ssr,
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
-			logger:   zlog.Named("ingress-reconciler"),
+			logger:   opts.log.Named("ingress-reconciler"),
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -304,29 +319,201 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			ssr:      ssr,
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
-			logger:   zlog.Named("connector-reconciler"),
+			logger:   opts.log.Named("connector-reconciler"),
 			clock:    tstime.DefaultClock{},
 		})
 	if err != nil {
-		startlog.Fatal("could not create connector reconciler: %v", err)
+		startlog.Fatalf("could not create connector reconciler: %v", err)
+	}
+	// TODO (irbekrm): switch to metadata-only watches for resources whose
+	// spec we don't need to inspect to reduce memory consumption.
+	// https://github.com/kubernetes-sigs/controller-runtime/issues/1159
+	nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver"))
+	err = builder.ControllerManagedBy(mgr).
+		For(&tsapi.DNSConfig{}).
+		Watches(&appsv1.Deployment{}, nameserverFilter).
+		Watches(&corev1.ConfigMap{}, nameserverFilter).
+		Watches(&corev1.Service{}, nameserverFilter).
+		Watches(&corev1.ServiceAccount{}, nameserverFilter).
+		Complete(&NameserverReconciler{
+			recorder:    eventRecorder,
+			tsNamespace: opts.tailscaleNamespace,
+			Client:      mgr.GetClient(),
+			logger:      opts.log.Named("nameserver-reconciler"),
+			clock:       tstime.DefaultClock{},
+		})
+	if err != nil {
+		startlog.Fatalf("could not create nameserver reconciler: %v", err)
 	}
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
 		Complete(&ProxyClassReconciler{
 			Client:   mgr.GetClient(),
 			recorder: eventRecorder,
-			logger:   zlog.Named("proxyclass-reconciler"),
+			logger:   opts.log.Named("proxyclass-reconciler"),
 			clock:    tstime.DefaultClock{},
 		})
 	if err != nil {
 		startlog.Fatal("could not create proxyclass reconciler: %v", err)
 	}
+	logger := startlog.Named("dns-records-reconciler-event-handlers")
+	// On EndpointSlice events, if it is an EndpointSlice for an
+	// ingress/egress proxy headless Service, reconcile the headless
+	// Service.
+	dnsRREpsOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerEndpointSliceHandler)
+	// On DNSConfig changes, reconcile all headless Services for
+	// ingress/egress proxies in operator namespace.
+	dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(opts.tailscaleNamespace, mgr.GetClient(), logger))
+	// On Service events, if it is an ingress/egress proxy headless Service, reconcile it.
+	dnsRRServiceOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerServiceHandler)
+	// On Ingress events, if it is a tailscale Ingress or if tailscale is the default ingress controller, reconcile the proxy
+	// headless Service.
+	dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(opts.tailscaleNamespace, opts.proxyActAsDefaultLoadBalancer, mgr.GetClient(), logger))
+	err = builder.ControllerManagedBy(mgr).
+		Named("dns-records-reconciler").
+		Watches(&corev1.Service{}, dnsRRServiceOpts).
+		Watches(&networkingv1.Ingress{}, dnsRRIngressOpts).
+		Watches(&discoveryv1.EndpointSlice{}, dnsRREpsOpts).
+		Watches(&tsapi.DNSConfig{}, dnsRRDNSConfigOpts).
+		Complete(&dnsRecordsReconciler{
+			Client:                mgr.GetClient(),
+			tsNamespace:           opts.tailscaleNamespace,
+			logger:                opts.log.Named("dns-records-reconciler"),
+			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create DNS records reconciler: %v", err)
+	}
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
 	}
 }
 
+type reconcilerOpts struct {
+	log                *zap.SugaredLogger
+	tsServer           *tsnet.Server
+	tsClient           *tailscale.Client
+	tailscaleNamespace string       // namespace in which operator resources will be deployed
+	restConfig         *rest.Config // config for connecting to the kube API server
+	proxyImage         string       // <proxy-image-repo>:<proxy-image-tag>
+	// proxyPriorityClassName isPriorityClass to be set for proxy Pods. This
+	// is a legacy mechanism for cluster resource configuration options -
+	// going forward use ProxyClass.
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass
+	proxyPriorityClassName string
+	// proxyTags are ACL tags to tag proxy auth keys. Multiple tags should
+	// be provided as a string with comma-separated tag values. Proxy tags
+	// default to tag:k8s.
+	// https://tailscale.com/kb/1085/auth-keys
+	proxyTags string
+	// proxyActAsDefaultLoadBalancer determines whether this operator
+	// instance should act as the default ingress controller when looking at
+	// Ingress resources with unset ingress.spec.ingressClassName.
+	// TODO (irbekrm): this setting does not respect the default
+	// IngressClass.
+	// https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class
+	// We should fix that and preferably integrate with that mechanism as
+	// well - perhaps make the operator itself create the default
+	// IngressClass if this is set to true.
+	proxyActAsDefaultLoadBalancer bool
+	// proxyFirewallMode determines whether non-userspace proxies should use
+	// iptables or nftables for firewall configuration. Accepted values are
+	// iptables, nftables and auto. If set to auto, proxy will automatically
+	// determine which mode is supported for a given host (prefer nftables).
+	// Auto is usually the best choice, unless you want to explicitly set
+	// specific mode for debugging purposes.
+	proxyFirewallMode string
+}
+
+// enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
+// ingress/egress proxy headless Service found in the provided namespace.
+func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, _ client.Object) []reconcile.Request {
+		reqs := make([]reconcile.Request, 0)
+
+		// Get all headless Services for proxies configured using Service.
+		svcProxyLabels := map[string]string{
+			LabelManaged:    "true",
+			LabelParentType: "svc",
+		}
+		svcHeadlessSvcList := &corev1.ServiceList{}
+		if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
+			logger.Errorf("error listing headless Services for tailscale ingress/egress Services in operator namespace: %v", err)
+			return nil
+		}
+		for _, svc := range svcHeadlessSvcList.Items {
+			reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}})
+		}
+
+		// Get all headless Services for proxies configured using Ingress.
+		ingProxyLabels := map[string]string{
+			LabelManaged:    "true",
+			LabelParentType: "ingress",
+		}
+		ingHeadlessSvcList := &corev1.ServiceList{}
+		if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
+			logger.Errorf("error listing headless Services for tailscale Ingresses in operator namespace: %v", err)
+			return nil
+		}
+		for _, svc := range ingHeadlessSvcList.Items {
+			reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}})
+		}
+		return reqs
+	}
+}
+
+// dnsRecordsReconciler filters EndpointSlice events for which
+// dns-records-reconciler should reconcile a headless Service. The only events
+// it should reconcile are those for EndpointSlices associated with proxy
+// headless Services.
+func dnsRecordsReconcilerEndpointSliceHandler(ctx context.Context, o client.Object) []reconcile.Request {
+	if !isManagedByType(o, "svc") && !isManagedByType(o, "ingress") {
+		return nil
+	}
+	headlessSvcName, ok := o.GetLabels()[discoveryv1.LabelServiceName] // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
+	if !ok {
+		return nil
+	}
+	return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: headlessSvcName}}}
+}
+
+// dnsRecordsReconcilerServiceHandler filters Service events for which
+// dns-records-reconciler should reconcile. If the event is for a cluster
+// ingress/cluster egress proxy's headless Service, returns the Service for
+// reconcile.
+func dnsRecordsReconcilerServiceHandler(ctx context.Context, o client.Object) []reconcile.Request {
+	if isManagedByType(o, "svc") || isManagedByType(o, "ingress") {
+		return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: o.GetName()}}}
+	}
+	return nil
+}
+
+// dnsRecordsReconcilerIngressHandler filters Ingress events to ensure that
+// dns-records-reconciler only reconciles on tailscale Ingress events. When an
+// event is observed on a tailscale Ingress, reconcile the proxy headless Service.
+func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		ing, ok := o.(*networkingv1.Ingress)
+		if !ok {
+			return nil
+		}
+		if !isDefaultLoadBalancer && (ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != "tailscale") {
+			return nil
+		}
+		proxyResourceLabels := childResourceLabels(ing.Name, ing.Namespace, "ingress")
+		headlessSvc, err := getSingleObject[corev1.Service](ctx, cl, ns, proxyResourceLabels)
+		if err != nil {
+			logger.Errorf("error getting headless Service from parent labels: %v", err)
+			return nil
+		}
+		if headlessSvc == nil {
+			return nil
+		}
+		return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: headlessSvc.Namespace, Name: headlessSvc.Name}}}
+	}
+}
+
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error

cmd/k8s-operator/operator_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
@@ -17,10 +18,15 @@ import (
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/net/dns/resolvconffile"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
 	"tailscale.com/types/ptr"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
 )
 
@@ -31,6 +37,7 @@ func TestLoadBalancerClass(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -40,11 +47,13 @@ func TestLoadBalancerClass(t *testing.T) {
 			operatorNamespace: "operator-ns",
 			proxyImage:        "tailscale/tailscale",
 		},
-		logger: zl.Sugar(),
+		logger:   zl.Sugar(),
+		clock:    clock,
+		recorder: record.NewFakeRecorder(100),
 	}
 
-	// Create a service that we should manage, and check that the initial round
-	// of objects looks right.
+	// Create a service that we should manage, but start with a miconfiguration
+	// in the annotations.
 	mustCreate(t, fc, &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
@@ -53,6 +62,9 @@ func TestLoadBalancerClass(t *testing.T) {
 			// doesn't. So, set it explicitly because other code later depends
 			// on it being set.
 			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: "invalid.example.com",
+			},
 		},
 		Spec: corev1.ServiceSpec{
 			ClusterIP:         "10.20.30.40",
@@ -63,6 +75,46 @@ func TestLoadBalancerClass(t *testing.T) {
 
 	expectReconciled(t, sr, "default", "test")
 
+	// The expected value of .status.conditions[0].LastTransitionTime until the
+	// proxy becomes ready.
+	t0 := conditionTime(clock)
+
+	// Should have an error about invalid config.
+	want := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: "invalid.example.com",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyReady),
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: t0,
+				Reason:             reasonProxyInvalid,
+				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-fqdn: "invalid.example.com" does not appear to be a valid MagicDNS name`,
+			}},
+		},
+	}
+	expectEqual(t, fc, want, nil)
+
+	// Delete the misconfiguration so the proxy starts getting created on the
+	// next reconcile.
+	mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+		s.ObjectMeta.Annotations = nil
+	})
+
+	clock.Advance(time.Second)
+	expectReconciled(t, sr, "default", "test")
+
 	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
 	opts := configOpts{
 		stsName:         shortName,
@@ -73,10 +125,23 @@ func TestLoadBalancerClass(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
+	want.Annotations = nil
+	want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
+	want.Status = corev1.ServiceStatus{
+		Conditions: []metav1.Condition{{
+			Type:               string(tsapi.ProxyReady),
+			Status:             metav1.ConditionFalse,
+			LastTransitionTime: t0, // Status is still false, no update to transition time
+			Reason:             reasonProxyPending,
+			Message:            "no Tailscale hostname known yet, waiting for proxy pod to finish auth",
+		}},
+	}
+	expectEqual(t, fc, want, nil)
+
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, then verify reconcile again and verify
 	// that we get to the end.
@@ -88,33 +153,16 @@ func TestLoadBalancerClass(t *testing.T) {
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
 		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
+	clock.Advance(time.Second)
 	expectReconciled(t, sr, "default", "test")
-	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:       "test",
-			Namespace:  "default",
-			Finalizers: []string{"tailscale.com/finalizer"},
-			UID:        types.UID("1234-UID"),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-		Status: corev1.ServiceStatus{
-			LoadBalancer: corev1.LoadBalancerStatus{
-				Ingress: []corev1.LoadBalancerIngress{
-					{
-						Hostname: "tailscale.device.name",
-					},
-					{
-						IP: "100.99.98.97",
-					},
-				},
+	want.Status.Conditions = proxyCreatedCondition(clock)
+	want.Status.LoadBalancer = corev1.LoadBalancerStatus{
+		Ingress: []corev1.LoadBalancerIngress{
+			{
+				Hostname: "tailscale.device.name",
+			},
+			{
+				IP: "100.99.98.97",
 			},
 		},
 	}
@@ -142,11 +190,9 @@ func TestLoadBalancerClass(t *testing.T) {
 	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
+
+	// Note that the Tailscale-specific condition status should be gone now.
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -168,6 +214,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		t.Fatal(err)
 	}
 	tailnetTargetFQDN := "foo.bar.ts.net."
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -178,6 +225,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -214,14 +262,10 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		hostname:          "default-test",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -236,9 +280,12 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 			Type:         corev1.ServiceTypeExternalName,
 			Selector:     nil,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -278,6 +325,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		t.Fatal(err)
 	}
 	tailnetTargetIP := "100.66.66.66"
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -288,6 +336,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -324,14 +373,10 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		hostname:        "default-test",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -346,9 +391,12 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 			Type:         corev1.ServiceTypeExternalName,
 			Selector:     nil,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -387,6 +435,7 @@ func TestAnnotations(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -397,6 +446,7 @@ func TestAnnotations(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -431,14 +481,10 @@ func TestAnnotations(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -452,6 +498,9 @@ func TestAnnotations(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 
@@ -471,10 +520,6 @@ func TestAnnotations(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -495,6 +540,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -505,6 +551,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -539,7 +586,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -556,10 +603,6 @@ func TestAnnotationIntoLB(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -573,6 +616,9 @@ func TestAnnotationIntoLB(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 
@@ -590,10 +636,6 @@ func TestAnnotationIntoLB(t *testing.T) {
 	// ... but the service should have a LoadBalancer status.
 
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -616,6 +658,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 					},
 				},
 			},
+			Conditions: proxyCreatedCondition(clock),
 		},
 	}
 	expectEqual(t, fc, want, nil)
@@ -628,6 +671,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -638,6 +682,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -670,7 +715,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -687,10 +732,6 @@ func TestLBIntoAnnotation(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -713,6 +754,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 					},
 				},
 			},
+			Conditions: proxyCreatedCondition(clock),
 		},
 	}
 	expectEqual(t, fc, want, nil)
@@ -738,10 +780,6 @@ func TestLBIntoAnnotation(t *testing.T) {
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -755,6 +793,9 @@ func TestLBIntoAnnotation(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 }
@@ -766,6 +807,7 @@ func TestCustomHostname(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -776,6 +818,7 @@ func TestCustomHostname(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -811,14 +854,10 @@ func TestCustomHostname(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -833,6 +872,9 @@ func TestCustomHostname(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 
@@ -852,10 +894,6 @@ func TestCustomHostname(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -879,6 +917,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -890,6 +929,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 			proxyPriorityClassName: "custom-priority-class-name",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -933,10 +973,14 @@ func TestProxyClassForService(t *testing.T) {
 	// Setup
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
-		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
-			Annotations: map[string]string{"bar.io/foo": "some-val"},
-			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
+		Spec: tsapi.ProxyClassSpec{
+			TailscaleConfig: &tsapi.TailscaleConfig{
+				AcceptRoutes: true,
+			},
+			StatefulSet: &tsapi.StatefulSet{
+				Labels:      map[string]string{"foo": "bar"},
+				Annotations: map[string]string{"bar.io/foo": "some-val"},
+				Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
@@ -948,6 +992,7 @@ func TestProxyClassForService(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -958,6 +1003,7 @@ func TestProxyClassForService(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// 1. A new tailscale LoadBalancer Service is created without any
@@ -987,7 +1033,7 @@ func TestProxyClassForService(t *testing.T) {
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
@@ -999,21 +1045,23 @@ func TestProxyClassForService(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 
 	// 3. ProxyClass is set to Ready, the Service gets reconciled by the
 	// services-reconciler and the customization from the ProxyClass is
 	// applied to the proxy resources.
 	mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
 		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []tsapi.ConnectorCondition{{
+			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               tsapi.ProxyClassready,
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
 	opts.proxyClass = pc.Name
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
 
 	// 4. tailscale.com/proxy-class label is removed from the Service, the
 	// configuration from the ProxyClass is removed from the cluster
@@ -1033,6 +1081,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1043,6 +1092,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger:                zl.Sugar(),
+		clock:                 clock,
 		isDefaultLoadBalancer: true,
 	}
 
@@ -1087,6 +1137,7 @@ func TestProxyFirewallMode(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1098,6 +1149,7 @@ func TestProxyFirewallMode(t *testing.T) {
 			tsFirewallMode:    "nftables",
 		},
 		logger:                zl.Sugar(),
+		clock:                 clock,
 		isDefaultLoadBalancer: true,
 	}
 
@@ -1140,6 +1192,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1150,6 +1203,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger:                zl.Sugar(),
+		clock:                 clock,
 		isDefaultLoadBalancer: true,
 	}
 
@@ -1180,7 +1234,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1",
+		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 
@@ -1190,11 +1244,10 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "1a087f887825d2b75d3673c7c2b0131f8ec1f0b1cb761d33e236dd28350dfe23"
+	o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }
-
 func Test_isMagicDNSName(t *testing.T) {
 	tests := []struct {
 		in   string
@@ -1352,3 +1405,165 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 		t.Errorf("unexpected reconcile request for a Service that does not belong to any Ingress: %#+v\n", gotReqs)
 	}
 }
+
+func Test_clusterDomainFromResolverConf(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name      string
+		conf      *resolvconffile.Config
+		namespace string
+		want      string
+	}{
+		{
+			name: "success- custom domain",
+			conf: &resolvconffile.Config{
+				SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")},
+			},
+			namespace: "foo",
+			want:      "department.org.io",
+		},
+		{
+			name: "success- default domain",
+			conf: &resolvconffile.Config{
+				SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.cluster.local."), toFQDN(t, "svc.cluster.local."), toFQDN(t, "cluster.local.")},
+			},
+			namespace: "foo",
+			want:      "cluster.local",
+		},
+		{
+			name: "only two search domains found",
+			conf: &resolvconffile.Config{
+				SearchDomains: []dnsname.FQDN{toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")},
+			},
+			namespace: "foo",
+			want:      "cluster.local",
+		},
+		{
+			name: "first search domain does not match the expected structure",
+			conf: &resolvconffile.Config{
+				SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.bar.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")},
+			},
+			namespace: "foo",
+			want:      "cluster.local",
+		},
+		{
+			name: "second search domain does not match the expected structure",
+			conf: &resolvconffile.Config{
+				SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "foo.department.org.io"), toFQDN(t, "some.other.fqdn")},
+			},
+			namespace: "foo",
+			want:      "cluster.local",
+		},
+		{
+			name: "third search domain does not match the expected structure",
+			conf: &resolvconffile.Config{
+				SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")},
+			},
+			namespace: "foo",
+			want:      "cluster.local",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := clusterDomainFromResolverConf(tt.conf, tt.namespace, zl.Sugar()); got != tt.want {
+				t.Errorf("clusterDomainFromResolverConf() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_externalNameService(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 1. A External name Service that should be exposed via Tailscale gets
+	// created.
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+		clock:  clock,
+	}
+
+	// 1. Create an ExternalName Service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationExpose: "true",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type:         corev1.ServiceTypeExternalName,
+			ExternalName: "foo.com",
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
+	opts := configOpts{
+		stsName:          shortName,
+		secretName:       fullName,
+		namespace:        "default",
+		parentType:       "svc",
+		hostname:         "default-test",
+		clusterTargetDNS: "foo.com",
+	}
+
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+
+	// 2. Change the ExternalName and verify that changes get propagated.
+	mustUpdate(t, sr, "default", "test", func(s *corev1.Service) {
+		s.Spec.ExternalName = "bar.com"
+	})
+	expectReconciled(t, sr, "default", "test")
+	opts.clusterTargetDNS = "bar.com"
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+}
+
+func toFQDN(t *testing.T, s string) dnsname.FQDN {
+	t.Helper()
+	fqdn, err := dnsname.ToFQDN(s)
+	if err != nil {
+		t.Fatalf("error coverting %q to dnsname.FQDN: %v", s, err)
+	}
+	return fqdn
+}
+
+func proxyCreatedCondition(clock tstime.Clock) []metav1.Condition {
+	return []metav1.Condition{{
+		Type:               string(tsapi.ProxyReady),
+		Status:             metav1.ConditionTrue,
+		ObservedGeneration: 0,
+		LastTransitionTime: conditionTime(clock),
+		Reason:             reasonProxyCreated,
+		Message:            reasonProxyCreated,
+	}}
+}
+
+func conditionTime(clock tstime.Clock) metav1.Time {
+	return metav1.NewTime(clock.Now().Truncate(time.Second))
+}

cmd/k8s-operator/proxy.go

@@ -11,15 +11,20 @@ import (
 	"log"
 	"net/http"
 	"net/http/httputil"
+	"net/netip"
 	"net/url"
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	kubesessionrecording "tailscale.com/k8s-operator/sessionrecording"
+	tskube "tailscale.com/kube"
+	"tailscale.com/sessionrecording"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -29,10 +34,26 @@ import (
 
 var whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
 
-var counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
+var (
+	// counterNumRequestsproxies counts the number of API server requests proxied via this proxy.
+	counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
+)
 
 type apiServerProxyMode int
 
+func (a apiServerProxyMode) String() string {
+	switch a {
+	case apiserverProxyModeDisabled:
+		return "disabled"
+	case apiserverProxyModeEnabled:
+		return "auth"
+	case apiserverProxyModeNoAuth:
+		return "noauth"
+	default:
+		return "unknown"
+	}
+}
+
 const (
 	apiserverProxyModeDisabled apiServerProxyMode = iota
 	apiserverProxyModeEnabled
@@ -96,26 +117,7 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config,
 	if err != nil {
 		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
 	}
-	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode)
-}
-
-// apiserverProxy is an http.Handler that authenticates requests using the Tailscale
-// LocalAPI and then proxies them to the Kubernetes API.
-type apiserverProxy struct {
-	log *zap.SugaredLogger
-	lc  *tailscale.LocalClient
-	rp  *httputil.ReverseProxy
-}
-
-func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	who, err := h.lc.WhoIs(r.Context(), r.RemoteAddr)
-	if err != nil {
-		h.log.Errorf("failed to authenticate caller: %v", err)
-		http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
-		return
-	}
-	counterNumRequestsProxied.Add(1)
-	h.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host)
 }
 
 // runAPIServerProxy runs an HTTP server that authenticates requests using the
@@ -132,64 +134,42 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 //     are passed through to the Kubernetes API.
 //
 // It never returns.
-func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode) {
+func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode, host string) {
 	if mode == apiserverProxyModeDisabled {
 		return
 	}
-	ln, err := s.Listen("tcp", ":443")
+	ln, err := ts.Listen("tcp", ":443")
 	if err != nil {
 		log.Fatalf("could not listen on :443: %v", err)
 	}
-	u, err := url.Parse(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	u, err := url.Parse(host)
 	if err != nil {
 		log.Fatalf("runAPIServerProxy: failed to parse URL %v", err)
 	}
 
-	lc, err := s.LocalClient()
+	lc, err := ts.LocalClient()
 	if err != nil {
 		log.Fatalf("could not get local client: %v", err)
 	}
+
 	ap := &apiserverProxy{
-		log: log,
-		lc:  lc,
-		rp: &httputil.ReverseProxy{
-			Rewrite: func(r *httputil.ProxyRequest) {
-				// Replace the URL with the Kubernetes APIServer.
-
-				r.Out.URL.Scheme = u.Scheme
-				r.Out.URL.Host = u.Host
-				if mode == apiserverProxyModeNoAuth {
-					// If we are not providing authentication, then we are just
-					// proxying to the Kubernetes API, so we don't need to do
-					// anything else.
-					return
-				}
-
-				// We want to proxy to the Kubernetes API, but we want to use
-				// the caller's identity to do so. We do this by impersonating
-				// the caller using the Kubernetes User Impersonation feature:
-				// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
-
-				// Out of paranoia, remove all authentication headers that might
-				// have been set by the client.
-				r.Out.Header.Del("Authorization")
-				r.Out.Header.Del("Impersonate-Group")
-				r.Out.Header.Del("Impersonate-User")
-				r.Out.Header.Del("Impersonate-Uid")
-				for k := range r.Out.Header {
-					if strings.HasPrefix(k, "Impersonate-Extra-") {
-						r.Out.Header.Del(k)
-					}
-				}
-
-				// Now add the impersonation headers that we want.
-				if err := addImpersonationHeaders(r.Out, log); err != nil {
-					panic("failed to add impersonation headers: " + err.Error())
-				}
-			},
-			Transport: rt,
-		},
+		log:         log,
+		lc:          lc,
+		mode:        mode,
+		upstreamURL: u,
+		ts:          ts,
 	}
+	ap.rp = &httputil.ReverseProxy{
+		Rewrite: func(pr *httputil.ProxyRequest) {
+			ap.addImpersonationHeadersAsRequired(pr.Out)
+		},
+		Transport: rt,
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", ap.serveDefault)
+	mux.HandleFunc("/api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExec)
+
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
 		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
@@ -198,41 +178,139 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 			NextProtos:     []string{"http/1.1"},
 		},
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
-		Handler:      ap,
+		Handler:      mux,
 	}
-	log.Infof("listening on %s", ln.Addr())
+	log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr())
 	if err := hs.ServeTLS(ln, "", ""); err != nil {
 		log.Fatalf("runAPIServerProxy: failed to serve %v", err)
 	}
 }
 
+// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale
+// LocalAPI and then proxies them to the Kubernetes API.
+type apiserverProxy struct {
+	log *zap.SugaredLogger
+	lc  *tailscale.LocalClient
+	rp  *httputil.ReverseProxy
+
+	mode        apiServerProxyMode
+	ts          *tsnet.Server
+	upstreamURL *url.URL
+}
+
+// serveDefault is the default handler for Kubernetes API server requests.
+func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
+	who, err := ap.whoIs(r)
+	if err != nil {
+		ap.authError(w, err)
+		return
+	}
+	counterNumRequestsProxied.Add(1)
+	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+}
+
+// serveExec serves 'kubectl exec' requests, optionally configuring the kubectl
+// exec sessions to be recorded.
+func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
+	who, err := ap.whoIs(r)
+	if err != nil {
+		ap.authError(w, err)
+		return
+	}
+	counterNumRequestsProxied.Add(1)
+	failOpen, addrs, err := determineRecorderConfig(who)
+	if err != nil {
+		ap.log.Errorf("error trying to determine whether the 'kubectl exec' session needs to be recorded: %v", err)
+		return
+	}
+	if failOpen && len(addrs) == 0 { // will not record
+		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+		return
+	}
+	kubesessionrecording.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
+	if !failOpen && len(addrs) == 0 {
+		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
+		ap.log.Error(msg)
+		http.Error(w, msg, http.StatusForbidden)
+		return
+	}
+	if r.Method != "POST" || r.Header.Get("Upgrade") != "SPDY/3.1" {
+		msg := "'kubectl exec' session recording is configured, but the request is not over SPDY. Session recording is currently only supported for SPDY based clients"
+		if failOpen {
+			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
+			ap.log.Warn(msg)
+			ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+			return
+		}
+		ap.log.Error(msg)
+		msg += "; failure mode is 'fail closed'; closing connection."
+		http.Error(w, msg, http.StatusForbidden)
+		return
+	}
+	spdyH := kubesessionrecording.New(ap.ts, r, who, w, r.PathValue("pod"), r.PathValue("namespace"), kubesessionrecording.SPDYProtocol, addrs, failOpen, sessionrecording.ConnectToRecorder, ap.log)
+
+	ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+}
+
+func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
+	r.URL.Scheme = h.upstreamURL.Scheme
+	r.URL.Host = h.upstreamURL.Host
+	if h.mode == apiserverProxyModeNoAuth {
+		// If we are not providing authentication, then we are just
+		// proxying to the Kubernetes API, so we don't need to do
+		// anything else.
+		return
+	}
+
+	// We want to proxy to the Kubernetes API, but we want to use
+	// the caller's identity to do so. We do this by impersonating
+	// the caller using the Kubernetes User Impersonation feature:
+	// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
+
+	// Out of paranoia, remove all authentication headers that might
+	// have been set by the client.
+	r.Header.Del("Authorization")
+	r.Header.Del("Impersonate-Group")
+	r.Header.Del("Impersonate-User")
+	r.Header.Del("Impersonate-Uid")
+	for k := range r.Header {
+		if strings.HasPrefix(k, "Impersonate-Extra-") {
+			r.Header.Del(k)
+		}
+	}
+
+	// Now add the impersonation headers that we want.
+	if err := addImpersonationHeaders(r, h.log); err != nil {
+		log.Printf("failed to add impersonation headers: " + err.Error())
+	}
+}
+func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
+	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
+}
+func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
+	ap.log.Errorf("failed to authenticate caller: %v", err)
+	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
+}
+
 const (
-	capabilityName    = "tailscale.com/cap/kubernetes"
-	oldCapabilityName = "https://" + capabilityName
+	// oldCapabilityName is a legacy form of
+	// tailfcg.PeerCapabilityKubernetes capability. The only capability rule
+	// that is respected for this form is group impersonation - for
+	// backwards compatibility reasons.
+	// TODO (irbekrm): determine if anyone uses this and remove if possible.
+	oldCapabilityName = "https://" + tailcfg.PeerCapabilityKubernetes
 )
 
-type capRule struct {
-	// Impersonate is a list of rules that specify how to impersonate the caller
-	// when proxying to the Kubernetes API.
-	Impersonate *impersonateRule `json:"impersonate,omitempty"`
-}
-
-// TODO(maisem): move this to some well-known location so that it can be shared
-// with control.
-type impersonateRule struct {
-	Groups []string `json:"groups,omitempty"`
-}
-
 // addImpersonationHeaders adds the appropriate headers to r to impersonate the
 // caller when proxying to the Kubernetes API. It uses the WhoIsResponse stashed
 // in the context by the apiserverProxy.
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[capRule](who.CapMap, capabilityName)
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[capRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)
@@ -273,3 +351,34 @@ func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	}
 	return nil
 }
+
+// determineRecorderConfig determines recorder config from requester's peer
+// capabilities. Determines whether a 'kubectl exec' session from this requester
+// needs to be recorded and what recorders the recording should be sent to.
+func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorderAddresses []netip.AddrPort, _ error) {
+	if who == nil {
+		return false, nil, errors.New("[unexpected] cannot determine caller")
+	}
+	failOpen = true
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	if err != nil {
+		return failOpen, nil, fmt.Errorf("failed to unmarshal Kubernetes capability: %w", err)
+	}
+	if len(rules) == 0 {
+		return failOpen, nil, nil
+	}
+
+	for _, rule := range rules {
+		if len(rule.RecorderAddrs) != 0 {
+			// TODO (irbekrm): here or later determine if the
+			// recorders behind those addrs are online - else we
+			// spend 30s trying to reach a recorder whose tailscale
+			// status is offline.
+			recorderAddresses = append(recorderAddresses, rule.RecorderAddrs...)
+		}
+		if rule.EnforceRecorder {
+			failOpen = false
+		}
+	}
+	return failOpen, recorderAddresses, nil
+}

cmd/k8s-operator/proxy_test.go

@@ -7,6 +7,8 @@ package main
 
 import (
 	"net/http"
+	"net/netip"
+	"reflect"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -49,7 +51,7 @@ func TestImpersonationHeaders(t *testing.T) {
 			name:     "user-with-cap",
 			emailish: "foo@example.com",
 			capMap: tailcfg.PeerCapMap{
-				capabilityName: {
+				tailcfg.PeerCapabilityKubernetes: {
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group1","group2"]}}`),
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group1","group3"]}}`), // One group is duplicated.
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group4"]}}`),
@@ -71,7 +73,7 @@ func TestImpersonationHeaders(t *testing.T) {
 			emailish: "tagged-device",
 			tags:     []string{"tag:foo", "tag:bar"},
 			capMap: tailcfg.PeerCapMap{
-				capabilityName: {
+				tailcfg.PeerCapabilityKubernetes: {
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group1"]}}`),
 				},
 			},
@@ -80,12 +82,26 @@ func TestImpersonationHeaders(t *testing.T) {
 				"Impersonate-User":  {"node.ts.net"},
 			},
 		},
+		{
+			name:     "mix-of-caps",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				tailcfg.PeerCapabilityKubernetes: {
+					tailcfg.RawMessage(`{"impersonate":{"groups":["group1"]},"recorder":["tag:foo"],"enforceRecorder":true}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1"},
+				"Impersonate-User":  {"node.ts.net"},
+			},
+		},
 		{
 			name:     "bad-cap",
 			emailish: "tagged-device",
 			tags:     []string{"tag:foo", "tag:bar"},
 			capMap: tailcfg.PeerCapMap{
-				capabilityName: {
+				tailcfg.PeerCapabilityKubernetes: {
 					tailcfg.RawMessage(`[]`),
 				},
 			},
@@ -112,3 +128,72 @@ func TestImpersonationHeaders(t *testing.T) {
 		}
 	}
 }
+
+func Test_determineRecorderConfig(t *testing.T) {
+	addr1, addr2 := netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80"), netip.MustParseAddrPort("100.99.99.99:80")
+	tests := []struct {
+		name                  string
+		wantFailOpen          bool
+		wantRecorderAddresses []netip.AddrPort
+		who                   *apitype.WhoIsResponse
+	}{
+		{
+			name:                  "two_ips_fail_closed",
+			who:                   whoResp(map[string][]string{string(tailcfg.PeerCapabilityKubernetes): {`{"recorderAddrs":["[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80","100.99.99.99:80"],"enforceRecorder":true}`}}),
+			wantRecorderAddresses: []netip.AddrPort{addr1, addr2},
+		},
+		{
+			name:                  "two_ips_fail_open",
+			who:                   whoResp(map[string][]string{string(tailcfg.PeerCapabilityKubernetes): {`{"recorderAddrs":["[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80","100.99.99.99:80"]}`}}),
+			wantRecorderAddresses: []netip.AddrPort{addr1, addr2},
+			wantFailOpen:          true,
+		},
+		{
+			name:                  "odd_rule_combination_fail_closed",
+			who:                   whoResp(map[string][]string{string(tailcfg.PeerCapabilityKubernetes): {`{"recorderAddrs":["100.99.99.99:80"],"enforceRecorder":false}`, `{"recorderAddrs":["[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80"]}`, `{"enforceRecorder":true,"impersonate":{"groups":["system:masters"]}}`}}),
+			wantRecorderAddresses: []netip.AddrPort{addr2, addr1},
+		},
+		{
+			name:         "no_caps",
+			who:          whoResp(map[string][]string{}),
+			wantFailOpen: true,
+		},
+		{
+			name:         "no_recorder_caps",
+			who:          whoResp(map[string][]string{"foo": {`{"x":"y"}`}, string(tailcfg.PeerCapabilityKubernetes): {`{"impersonate":{"groups":["system:masters"]}}`}}),
+			wantFailOpen: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFailOpen, gotRecorderAddresses, err := determineRecorderConfig(tt.who)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if gotFailOpen != tt.wantFailOpen {
+				t.Errorf("determineRecorderConfig() gotFailOpen = %v, want %v", gotFailOpen, tt.wantFailOpen)
+			}
+			if !reflect.DeepEqual(gotRecorderAddresses, tt.wantRecorderAddresses) {
+				t.Errorf("determineRecorderConfig() gotRecorderAddresses = %v, want %v", gotRecorderAddresses, tt.wantRecorderAddresses)
+			}
+		})
+	}
+}
+
+func whoResp(capMap map[string][]string) *apitype.WhoIsResponse {
+	resp := &apitype.WhoIsResponse{
+		CapMap: tailcfg.PeerCapMap{},
+	}
+	for cap, rules := range capMap {
+		resp.CapMap[tailcfg.PeerCapability(cap)] = raw(rules...)
+	}
+	return resp
+}
+
+func raw(in ...string) []tailcfg.RawMessage {
+	var out []tailcfg.RawMessage
+	for _, i := range in {
+		out = append(out, tailcfg.RawMessage(i))
+	}
+	return out
+}

cmd/k8s-operator/proxyclass.go

@@ -3,14 +3,16 @@
 
 //go:build !plan9
 
-// tailscale-operator provides a way to expose services running in a Kubernetes
-// cluster to your Tailnet.
 package main
 
 import (
 	"context"
 	"fmt"
+	"slices"
+	"strings"
+	"sync"
 
+	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -18,6 +20,7 @@ import (
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -25,12 +28,16 @@ import (
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
 )
 
 const (
 	reasonProxyClassInvalid  = "ProxyClassInvalid"
 	reasonProxyClassValid    = "ProxyClassValid"
+	reasonCustomTSEnvVar     = "CustomTSEnvVar"
 	messageProxyClassInvalid = "ProxyClass is not valid: %v"
+	messageCustomTSEnvVar    = "ProxyClass overrides the default value for %s env var for %s container. Running with custom values for Tailscale env vars is not recommended and might break in the future."
 )
 
 type ProxyClassReconciler struct {
@@ -39,8 +46,20 @@ type ProxyClassReconciler struct {
 	recorder record.EventRecorder
 	logger   *zap.SugaredLogger
 	clock    tstime.Clock
+
+	mu sync.Mutex // protects following
+
+	// managedProxyClasses is a set of all ProxyClass resources that we're currently
+	// managing. This is only used for metrics.
+	managedProxyClasses set.Slice[types.UID]
 }
 
+var (
+	// gaugeProxyClassResources tracks the number of ProxyClass resources
+	// that we're currently managing.
+	gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources")
+)
+
 func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := pcr.logger.With("ProxyClass", req.Name)
 	logger.Debugf("starting reconcile")
@@ -55,9 +74,26 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyClass: %w", err)
 	}
 	if !pc.DeletionTimestamp.IsZero() {
-		logger.Debugf("ProxyClass is being deleted, do nothing")
-		return reconcile.Result{}, nil
+		logger.Debugf("ProxyClass is being deleted")
+		return reconcile.Result{}, pcr.maybeCleanup(ctx, logger, pc)
 	}
+
+	// Add a finalizer so that we can ensure that metrics get updated when
+	// this ProxyClass is deleted.
+	if !slices.Contains(pc.Finalizers, FinalizerName) {
+		logger.Debugf("updating ProxyClass finalizers")
+		pc.Finalizers = append(pc.Finalizers, FinalizerName)
+		if err := pcr.Update(ctx, pc); err != nil {
+			return res, fmt.Errorf("failed to add finalizer: %w", err)
+		}
+	}
+
+	// Ensure this ProxyClass is tracked in metrics.
+	pcr.mu.Lock()
+	pcr.managedProxyClasses.Add(pc.UID)
+	gaugeProxyClassResources.Set(int64(pcr.managedProxyClasses.Len()))
+	pcr.mu.Unlock()
+
 	oldPCStatus := pc.Status.DeepCopy()
 	if errs := pcr.validate(pc); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
@@ -75,7 +111,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	return reconcile.Result{}, nil
 }
 
-func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
+func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
 	if sts := pc.Spec.StatefulSet; sts != nil {
 		if len(sts.Labels) > 0 {
 			if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
@@ -98,6 +134,33 @@ func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.
 					violations = append(violations, errs...)
 				}
 			}
+			if tc := pod.TailscaleContainer; tc != nil {
+				for _, e := range tc.Env {
+					if strings.HasPrefix(string(e.Name), "TS_") {
+						pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
+					}
+					if strings.EqualFold(string(e.Name), "EXPERIMENTAL_TS_CONFIGFILE_PATH") {
+						pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
+					}
+					if strings.EqualFold(string(e.Name), "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS") {
+						pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
+					}
+				}
+				if tc.Image != "" {
+					// Same validation as used by kubelet https://github.com/kubernetes/kubernetes/blob/release-1.30/pkg/kubelet/images/image_manager.go#L212
+					if _, err := dockerref.ParseNormalizedNamed(tc.Image); err != nil {
+						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleContainer", "image"), tc.Image, err.Error()))
+					}
+				}
+			}
+			if tc := pod.TailscaleInitContainer; tc != nil {
+				if tc.Image != "" {
+					// Same validation as used by kubelet https://github.com/kubernetes/kubernetes/blob/release-1.30/pkg/kubelet/images/image_manager.go#L212
+					if _, err := dockerref.ParseNormalizedNamed(tc.Image); err != nil {
+						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
+					}
+				}
+			}
 		}
 	}
 	// We do not validate embedded fields (security context, resource
@@ -106,3 +169,27 @@ func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.
 	// time.
 	return violations
 }
+
+// maybeCleanup removes tailscale.com finalizer and ensures that the ProxyClass
+// is no longer counted towards k8s_proxyclass_resources.
+func (pcr *ProxyClassReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, pc *tsapi.ProxyClass) error {
+	ix := slices.Index(pc.Finalizers, FinalizerName)
+	if ix < 0 {
+		logger.Debugf("no finalizer, nothing to do")
+		pcr.mu.Lock()
+		defer pcr.mu.Unlock()
+		pcr.managedProxyClasses.Remove(pc.UID)
+		gaugeProxyClassResources.Set(int64(pcr.managedProxyClasses.Len()))
+		return nil
+	}
+	pc.Finalizers = append(pc.Finalizers[:ix], pc.Finalizers[ix+1:]...)
+	if err := pcr.Update(ctx, pc); err != nil {
+		return fmt.Errorf("failed to remove finalizer: %w", err)
+	}
+	pcr.mu.Lock()
+	defer pcr.mu.Unlock()
+	pcr.managedProxyClasses.Remove(pc.UID)
+	gaugeProxyClassResources.Set(int64(pcr.managedProxyClasses.Len()))
+	logger.Infof("ProxyClass resources have been cleaned up")
+	return nil
+}

cmd/k8s-operator/proxyclass_test.go

@@ -29,7 +29,8 @@ func TestProxyClass(t *testing.T) {
 			// The apiserver is supposed to set the UID, but the fake client
 			// doesn't. So, set it explicitly because other code later depends
 			// on it being set.
-			UID: types.UID("1234-UID"),
+			UID:        types.UID("1234-UID"),
+			Finalizers: []string{"tailscale.com/finalizer"},
 		},
 		Spec: tsapi.ProxyClassSpec{
 			StatefulSet: &tsapi.StatefulSet{
@@ -38,6 +39,11 @@ func TestProxyClass(t *testing.T) {
 				Pod: &tsapi.Pod{
 					Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
 					Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
+					TailscaleContainer: &tsapi.Container{
+						Env:             []tsapi.Env{{Name: "FOO", Value: "BAR"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.my-repo/tailscale:v0.01testsomething",
+					},
 				},
 			},
 		},
@@ -51,27 +57,28 @@ func TestProxyClass(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	fr := record.NewFakeRecorder(3) // bump this if you expect a test case to throw more events
 	cl := tstest.NewClock(tstest.ClockOpts{})
 	pcr := &ProxyClassReconciler{
 		Client:   fc,
 		logger:   zl.Sugar(),
 		clock:    cl,
-		recorder: record.NewFakeRecorder(1),
+		recorder: fr,
 	}
-	expectReconciled(t, pcr, "", "test")
 
 	// 1. A valid ProxyClass resource gets its status updated to Ready.
-	pc.Status.Conditions = append(pc.Status.Conditions, tsapi.ConnectorCondition{
-		Type:               tsapi.ProxyClassready,
+	expectReconciled(t, pcr, "", "test")
+	pc.Status.Conditions = append(pc.Status.Conditions, metav1.Condition{
+		Type:               string(tsapi.ProxyClassready),
 		Status:             metav1.ConditionTrue,
 		Reason:             reasonProxyClassValid,
 		Message:            reasonProxyClassValid,
-		LastTransitionTime: &metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
 	})
 
 	expectEqual(t, fc, pc, nil)
 
-	// 2. An invalid ProxyClass resource gets its status updated to Invalid.
+	// 2. A ProxyClass resource with invalid labels gets its status updated to Invalid with an error message.
 	pc.Spec.StatefulSet.Labels["foo"] = "?!someVal"
 	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
 		proxyClass.Spec.StatefulSet.Labels = pc.Spec.StatefulSet.Labels
@@ -80,4 +87,51 @@ func TestProxyClass(t *testing.T) {
 	msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
 	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
+	expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 3. A ProxyClass resource with invalid image reference gets it status updated to Invalid with an error message.
+	pc.Spec.StatefulSet.Labels = nil
+	pc.Spec.StatefulSet.Pod.TailscaleContainer.Image = "FOO bar"
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Labels = nil
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleContainer.Image
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 4. A ProxyClass resource with invalid init container image reference gets it status updated to Invalid with an error message.
+	pc.Spec.StatefulSet.Labels = nil
+	pc.Spec.StatefulSet.Pod.TailscaleContainer.Image = ""
+	pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{
+		Image: "FOO bar",
+	}
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleContainer.Image
+		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{
+			Image: pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image,
+		}
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 5. An valid ProxyClass but with a Tailscale env vars set results in warning events.
+	pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = "" // unset previous test
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}}
+	})
+	expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
+		"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_TS_CONFIGFILE_PATH env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
+		"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."}
+	expectReconciled(t, pcr, "", "test")
+	expectEvents(t, fr, expectedEvents)
 }

cmd/k8s-operator/sts.go

@@ -35,7 +35,6 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/ptr"
-	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
 )
 
@@ -87,14 +86,11 @@ const (
 	// ensure that it does not get removed when a ProxyClass configuration
 	// is applied.
 	podAnnotationLastSetClusterIP         = "tailscale.com/operator-last-set-cluster-ip"
+	podAnnotationLastSetClusterDNSName    = "tailscale.com/operator-last-set-cluster-dns-name"
 	podAnnotationLastSetTailnetTargetIP   = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
 	podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
 	// podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents.
 	podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash"
-
-	// tailscaledConfigKey is the name of the key in proxy Secret Data that
-	// holds the tailscaled config contents.
-	tailscaledConfigKey = "tailscaled"
 )
 
 var (
@@ -109,8 +105,9 @@ type tailscaleSTSConfig struct {
 	ParentResourceUID   string
 	ChildResourceLabels map[string]string
 
-	ServeConfig     *ipn.ServeConfig // if serve config is set, this is a proxy for Ingress
-	ClusterTargetIP string           // ingress target
+	ServeConfig          *ipn.ServeConfig // if serve config is set, this is a proxy for Ingress
+	ClusterTargetIP      string           // ingress target IP
+	ClusterTargetDNSName string           // ingress target DNS name
 	// If set to true, operator should configure containerboot to forward
 	// cluster traffic via the proxy set up for Kubernetes Ingress.
 	ForwardClusterTrafficViaL7IngressProxy bool
@@ -126,7 +123,9 @@ type tailscaleSTSConfig struct {
 	// what this StatefulSet should be created for.
 	Connector *connector
 
-	ProxyClass string
+	ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy
+
+	ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
 }
 
 type connector struct {
@@ -172,11 +171,23 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
 		return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
 	}
 
-	secretName, tsConfigHash, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
+	proxyClass := new(tsapi.ProxyClass)
+	if sts.ProxyClassName != "" {
+		if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil {
+			return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
+		}
+		if !tsoperator.ProxyClassIsReady(proxyClass) {
+			logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
+			return nil, nil
+		}
+	}
+	sts.ProxyClass = proxyClass
+
+	secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
 	}
-	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash)
+	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash, configs)
 	if err != nil {
 		return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
 	}
@@ -283,13 +294,14 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 			Selector: map[string]string{
 				"app": sts.ParentResourceUID,
 			},
+			IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack),
 		},
 	}
 	logger.Debugf("reconciling headless service for StatefulSet")
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, string, error) {
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaleConfigs, _ error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			// Hardcode a -0 suffix so that in future, if we support
@@ -305,25 +317,23 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
 		orig = secret.DeepCopy()
 	} else if !apierrors.IsNotFound(err) {
-		return "", "", err
+		return "", "", nil, err
 	}
 
-	var (
-		authKey, hash string
-	)
+	var authKey string
 	if orig == nil {
 		// Initially it contains only tailscaled config, but when the
 		// proxy starts, it will also store there the state, certs and
 		// ACME account key.
 		sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
 		if err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 		if sts != nil {
 			// StatefulSet exists, so we have already created the secret.
 			// If the secret is missing, they should delete the StatefulSet.
 			logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
-			return "", "", nil
+			return "", "", nil, nil
 		}
 		// Create API Key secret which is going to be used by the statefulset
 		// to authenticate with Tailscale.
@@ -334,45 +344,58 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		}
 		authKey, err = a.newAuthKey(ctx, tags)
 		if err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 	}
-	confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig)
+	configs, err := tailscaledConfig(stsC, authKey, orig)
 	if err != nil {
-		return "", "", fmt.Errorf("error creating tailscaled config: %w", err)
+		return "", "", nil, fmt.Errorf("error creating tailscaled config: %w", err)
+	}
+	hash, err = tailscaledConfigHash(configs)
+	if err != nil {
+		return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err)
+	}
+
+	latest := tailcfg.CapabilityVersion(-1)
+	var latestConfig ipn.ConfigVAlpha
+	for key, val := range configs {
+		fn := tsoperator.TailscaledConfigFileNameForCap(key)
+		b, err := json.Marshal(val)
+		if err != nil {
+			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
+		}
+		mak.Set(&secret.StringData, fn, string(b))
+		if key > latest {
+			latest = key
+			latestConfig = val
+		}
 	}
-	hash = h
-	mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes))
 
 	if stsC.ServeConfig != nil {
 		j, err := json.Marshal(stsC.ServeConfig)
 		if err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 		mak.Set(&secret.StringData, "serve-config", string(j))
 	}
 
 	if orig != nil {
-		logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(secret.Data[tailscaledConfigKey]))
+		logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
 		if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 	} else {
-		logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes([]byte(secret.StringData[tailscaledConfigKey])))
+		logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
 		if err := a.Create(ctx, secret); err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 	}
-	return secret.Name, hash, nil
+	return secret.Name, hash, configs, nil
 }
 
 // sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
 // auth key.
-func sanitizeConfigBytes(bs []byte) string {
-	c := &ipn.ConfigVAlpha{}
-	if err := json.Unmarshal(bs, c); err != nil {
-		return "invalid config"
-	}
+func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
 	if c.AuthKey != nil {
 		c.AuthKey = ptr.To("**redacted**")
 	}
@@ -383,8 +406,10 @@ func sanitizeConfigBytes(bs []byte) string {
 	return string(sanitizedBytes)
 }
 
-// DeviceInfo returns the device ID and hostname for the Tailscale device
-// associated with the given labels.
+// DeviceInfo returns the device ID, hostname and IPs for the Tailscale device
+// that acts as an operator proxy. It retrieves info from a Kubernetes Secret
+// labeled with the provided labels.
+// Either of device ID, hostname and IPs can be empty string if not found in the Secret.
 func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
 	if err != nil {
@@ -401,7 +426,12 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	// to remove it.
 	hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
 	if hostname == "" {
-		return "", "", nil, nil
+		// Device ID gets stored and retrieved in a different flow than
+		// FQDN and IPs. A device that acts as Kubernetes operator
+		// proxy, but whose route setup has failed might have an device
+		// ID, but no FQDN/IPs. If so, return the ID, to allow the
+		// operator to clean up such devices.
+		return id, "", nil, nil
 	}
 	if rawDeviceIPs, ok := sec.Data["device_ips"]; ok {
 		if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
@@ -435,7 +465,7 @@ var proxyYaml []byte
 //go:embed deploy/manifests/userspace-proxy.yaml
 var userspaceProxyYaml []byte
 
-func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) {
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -455,16 +485,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	}
 	pod := &ss.Spec.Template
 	container := &pod.Spec.Containers[0]
-	proxyClass := new(tsapi.ProxyClass)
-	if sts.ProxyClass != "" {
-		if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil {
-			return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
-		}
-		if !tsoperator.ProxyClassIsReady(proxyClass) {
-			logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
-			return nil, nil
-		}
-	}
 	container.Image = a.proxyImage
 	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:      headlessSvc.Name,
@@ -491,9 +511,15 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Value: proxySecret,
 		},
 		corev1.EnvVar{
+			// Old tailscaled config key is still used for backwards compatibility.
 			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
 			Value: "/etc/tsconfig/tailscaled",
 		},
+		corev1.EnvVar{
+			// New style is in the form of cap-<capability-version>.hujson.
+			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
+			Value: "/etc/tsconfig",
+		},
 	)
 	if sts.ForwardClusterTrafficViaL7IngressProxy {
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -503,18 +529,16 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	}
 	// Configure containeboot to run tailscaled with a configfile read from the state Secret.
 	mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash)
-	pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+
+	configVolume := corev1.Volume{
 		Name: "tailscaledconfig",
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
 				SecretName: proxySecret,
-				Items: []corev1.KeyToPath{{
-					Key:  tailscaledConfigKey,
-					Path: tailscaledConfigKey,
-				}},
 			},
 		},
-	})
+	}
+	pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
 	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
 		Name:      "tailscaledconfig",
 		ReadOnly:  true,
@@ -536,6 +560,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Value: sts.ClusterTargetIP,
 		})
 		mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetClusterIP, sts.ClusterTargetIP)
+	} else if sts.ClusterTargetDNSName != "" {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_EXPERIMENTAL_DEST_DNS_NAME",
+			Value: sts.ClusterTargetDNSName,
+		})
+		mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetClusterDNSName, sts.ClusterTargetDNSName)
 	} else if sts.TailnetTargetIP != "" {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "TS_TAILNET_TARGET_IP",
@@ -563,18 +593,15 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: proxySecret,
-					Items: []corev1.KeyToPath{{
-						Key:  "serve-config",
-						Path: "serve-config",
-					}},
+					Items:      []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
 				},
 			},
 		})
 	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
-	if sts.ProxyClass != "" {
-		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
-		ss = applyProxyClassToStatefulSet(proxyClass, ss)
+	if sts.ProxyClassName != "" {
+		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
+		ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger)
 	}
 	updateSS := func(s *appsv1.StatefulSet) {
 		s.Spec = ss.Spec
@@ -605,8 +632,28 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [
 	return custom
 }
 
-func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet {
-	if pc == nil || ss == nil || pc.Spec.StatefulSet == nil {
+func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet {
+	if pc == nil || ss == nil {
+		return ss
+	}
+	if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
+		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+			enableMetrics(ss, pc)
+		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+			// TODO (irbekrm): fix this
+			// For Ingress proxies that have been configured with
+			// tailscale.com/experimental-forward-cluster-traffic-via-ingress
+			// annotation, all cluster traffic is forwarded to the
+			// Ingress backend(s).
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		} else {
+			// TODO (irbekrm): fix this
+			// For egress proxies, currently all cluster traffic is forwarded to the tailnet target.
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		}
+	}
+
+	if pc.Spec.StatefulSet == nil {
 		return ss
 	}
 
@@ -633,6 +680,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet)
 	ss.Spec.Template.Spec.ImagePullSecrets = wantsPod.ImagePullSecrets
 	ss.Spec.Template.Spec.NodeName = wantsPod.NodeName
 	ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
+	ss.Spec.Template.Spec.Affinity = wantsPod.Affinity
 	ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
 
 	// Update containers.
@@ -644,6 +692,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet)
 			base.SecurityContext = overlay.SecurityContext
 		}
 		base.Resources = overlay.Resources
+		for _, e := range overlay.Env {
+			// Env vars configured via ProxyClass might override env
+			// vars that have been specified by the operator, i.e
+			// TS_USERSPACE. The intended behaviour is to allow this
+			// and in practice it works without explicitly removing
+			// the operator configured value here as a later value
+			// in the env var list overrides an earlier one.
+			base.Env = append(base.Env, corev1.EnvVar{Name: string(e.Name), Value: e.Value})
+		}
+		if overlay.Image != "" {
+			base.Image = overlay.Image
+		}
+		if overlay.ImagePullPolicy != "" {
+			base.ImagePullPolicy = overlay.ImagePullPolicy
+		}
 		return base
 	}
 	for i, c := range ss.Spec.Template.Spec.Containers {
@@ -663,42 +726,105 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet)
 	return ss
 }
 
+func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) {
+	for i, c := range ss.Spec.Template.Spec.Containers {
+		if c.Name == "tailscale" {
+			// Serve metrics on on <pod-ip>:9001/debug/metrics. If
+			// we didn't specify Pod IP here, the proxy would, in
+			// some cases, also listen to its Tailscale IP- we don't
+			// want folks to start relying on this side-effect as a
+			// feature.
+			ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
+			ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001})
+			break
+		}
+	}
+}
+
+func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
+	origConf := &ipn.ConfigVAlpha{}
+	if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil {
+		return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err)
+	}
+	return origConf.AuthKey, nil
+}
+
 // tailscaledConfig takes a proxy config, a newly generated auth key if
 // generated and a Secret with the previous proxy state and auth key and
-// produces returns tailscaled configuration and a hash of that configuration.
-func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) ([]byte, string, error) {
-	conf := ipn.ConfigVAlpha{
-		Version:      "alpha0",
-		AcceptDNS:    "false",
-		AcceptRoutes: "false", // AcceptRoutes defaults to true
-		Locked:       "false",
-		Hostname:     &stsC.Hostname,
+// returns tailscaled configuration and a hash of that configuration.
+//
+// As of 2024-05-09 it also returns legacy tailscaled config without the
+// later added NoStatefulFilter field to support proxies older than cap95.
+// TODO (irbekrm): remove the legacy config once we no longer need to support
+// versions older than cap94,
+// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaleConfigs, error) {
+	conf := &ipn.ConfigVAlpha{
+		Version:             "alpha0",
+		AcceptDNS:           "false",
+		AcceptRoutes:        "false", // AcceptRoutes defaults to true
+		Locked:              "false",
+		Hostname:            &stsC.Hostname,
+		NoStatefulFiltering: "false",
+	}
+
+	// For egress proxies only, we need to ensure that stateful filtering is
+	// not in place so that traffic from cluster can be forwarded via
+	// Tailscale IPs.
+	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
+		conf.NoStatefulFiltering = "true"
 	}
 	if stsC.Connector != nil {
 		routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
 		if err != nil {
-			return nil, "", fmt.Errorf("error calculating routes: %w", err)
+			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
 		conf.AdvertiseRoutes = routes
 	}
+	if shouldAcceptRoutes(stsC.ProxyClass) {
+		conf.AcceptRoutes = "true"
+	}
+
 	if newAuthkey != "" {
 		conf.AuthKey = &newAuthkey
-	} else if oldSecret != nil && len(oldSecret.Data[tailscaledConfigKey]) > 0 { // write to StringData, read from Data as StringData is write-only
-		origConf := &ipn.ConfigVAlpha{}
-		if err := json.Unmarshal([]byte(oldSecret.Data[tailscaledConfigKey]), origConf); err != nil {
-			return nil, "", fmt.Errorf("error unmarshaling previous tailscaled config: %w", err)
+	} else if oldSecret != nil {
+		var err error
+		latest := tailcfg.CapabilityVersion(-1)
+		latestStr := ""
+		for k, data := range oldSecret.Data {
+			// write to StringData, read from Data as StringData is write-only
+			if len(data) == 0 {
+				continue
+			}
+			v, err := tsoperator.CapVerFromFileName(k)
+			if err != nil {
+				continue
+			}
+			if v > latest {
+				latestStr = k
+				latest = v
+			}
+		}
+		// Allow for configs that don't contain an auth key. Perhaps
+		// users have some mechanisms to delete them. Auth key is
+		// normally not needed after the initial login.
+		if latestStr != "" {
+			conf.AuthKey, err = readAuthKey(oldSecret, latestStr)
+			if err != nil {
+				return nil, err
+			}
 		}
-		conf.AuthKey = origConf.AuthKey
 	}
-	confFileBytes, err := json.Marshal(conf)
-	if err != nil {
-		return nil, "", fmt.Errorf("error marshaling tailscaled config : %w", err)
-	}
-	hash, err := hashBytes(confFileBytes)
-	if err != nil {
-		return nil, "", fmt.Errorf("error calculating config hash: %w", err)
-	}
-	return confFileBytes, hash, nil
+	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[95] = *conf
+	// legacy config should not contain NoStatefulFiltering field.
+	conf.NoStatefulFiltering.Clear()
+	capVerConfigs[94] = *conf
+	return capVerConfigs, nil
+}
+
+func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
+	return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
 }
 
 // ptrObject is a type constraint for pointer types that implement
@@ -708,7 +834,9 @@ type ptrObject[T any] interface {
 	*T
 }
 
-// hashBytes produces a hash for the provided bytes that is the same across
+type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
+
+// hashBytes produces a hash for the provided tailscaled config that is the same across
 // different invocations of this code. We do not use the
 // tailscale.com/deephash.Hash here because that produces a different hash for
 // the same value in different tailscale builds. The hash we are producing here
@@ -717,10 +845,13 @@ type ptrObject[T any] interface {
 // thing that changed is operator version (the hash is also exposed to users via
 // an annotation and might be confusing if it changes without the config having
 // changed).
-func hashBytes(b []byte) (string, error) {
-	h := sha256.New()
-	_, err := h.Write(b)
+func tailscaledConfigHash(c tailscaleConfigs) (string, error) {
+	b, err := json.Marshal(c)
 	if err != nil {
+		return "", fmt.Errorf("error marshalling tailscaled configs: %w", err)
+	}
+	h := sha256.New()
+	if _, err = h.Write(b); err != nil {
 		return "", fmt.Errorf("error calculating hash: %w", err)
 	}
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
@@ -816,14 +947,11 @@ func defaultEnv(envName, defVal string) string {
 	return v
 }
 
-func nameForService(svc *corev1.Service) (string, error) {
+func nameForService(svc *corev1.Service) string {
 	if h, ok := svc.Annotations[AnnotationHostname]; ok {
-		if err := dnsname.ValidLabel(h); err != nil {
-			return "", fmt.Errorf("invalid Tailscale hostname %q: %w", h, err)
-		}
-		return h, nil
+		return h
 	}
-	return svc.Namespace + "-" + svc.Name, nil
+	return svc.Namespace + "-" + svc.Name
 }
 
 func isValidFirewallMode(m string) bool {

cmd/k8s-operator/sts_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -51,6 +52,10 @@ func Test_statefulSetNameBase(t *testing.T) {
 }
 
 func Test_applyProxyClassToStatefulSet(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Setup
 	proxyClassAllOpts := &tsapi.ProxyClass{
 		Spec: tsapi.ProxyClassSpec{
@@ -66,6 +71,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 					ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}},
 					NodeName:         "some-node",
 					NodeSelector:     map[string]string{"beta.kubernetes.io/os": "linux"},
+					Affinity:         &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}},
 					Tolerations:      []corev1.Toleration{{Key: "", Operator: "Exists"}},
 					TailscaleContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
@@ -75,6 +81,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 							Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
 							Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
 						},
+						Env:             []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.io/my-repo/tailscale:v0.01testsomething",
 					},
 					TailscaleInitContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
@@ -85,6 +94,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 							Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
 							Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
 						},
+						Env:             []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.io/my-repo/tailscale:v0.01testsomething",
 					},
 				},
 			},
@@ -102,6 +114,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 			},
 		},
 	}
+	proxyClassMetrics := &tsapi.ProxyClass{
+		Spec: tsapi.ProxyClassSpec{
+			Metrics: &tsapi.Metrics{Enable: true},
+		},
+	}
+
 	var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
 	if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil {
 		t.Fatalf("unmarshaling userspace proxy template: %v", err)
@@ -121,10 +139,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	env := []corev1.EnvVar{{Name: "TS_HOSTNAME", Value: "nginx"}}
 	userspaceProxySS.Labels = labels
 	userspaceProxySS.Annotations = annots
+	userspaceProxySS.Spec.Template.Spec.Containers[0].Image = "tailscale/tailscale:v0.0.1"
 	userspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
 	nonUserspaceProxySS.ObjectMeta.Labels = labels
 	nonUserspaceProxySS.ObjectMeta.Annotations = annots
 	nonUserspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
+	nonUserspaceProxySS.Spec.Template.Spec.InitContainers[0].Image = "tailscale/tailscale:v0.0.1"
 
 	// 1. Test that a ProxyClass with all fields set gets correctly applied
 	// to a Statefulset built from non-userspace proxy template.
@@ -137,13 +157,20 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
 	wantSS.Spec.Template.Spec.NodeName = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeName
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
+	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.InitContainers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.Resources
+	wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent"
+	wantSS.Spec.Template.Spec.InitContainers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	wantSS.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = "IfNotPresent"
 
-	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy())
+	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
@@ -156,7 +183,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
 	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
-	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy())
+	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
@@ -172,12 +199,16 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
 	wantSS.Spec.Template.Spec.NodeName = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeName
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
+	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
-	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy())
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent"
+	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
@@ -187,10 +218,19 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
 	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
-	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy())
+	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
+
+	// 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
 }
 
 func mergeMapKeys(a, b map[string]string) map[string]string {

cmd/k8s-operator/svc.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/netip"
 	"slices"
@@ -15,17 +16,32 @@ import (
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/net/dns/resolvconffile"
+	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/set"
 )
 
+const (
+	resolvConfPath       = "/etc/resolv.conf"
+	defaultClusterDomain = "cluster.local"
+
+	reasonProxyCreated = "ProxyCreated"
+	reasonProxyInvalid = "ProxyInvalid"
+	reasonProxyFailed  = "ProxyFailed"
+	reasonProxyPending = "ProxyPending"
+)
+
 type ServiceReconciler struct {
 	client.Client
 	ssr                   *tailscaleSTSReconciler
@@ -42,6 +58,10 @@ type ServiceReconciler struct {
 	managedEgressProxies set.Slice[types.UID]
 
 	recorder record.EventRecorder
+
+	tsNamespace string
+
+	clock tstime.Clock
 }
 
 var (
@@ -68,6 +88,12 @@ func childResourceLabels(name, ns, typ string) map[string]string {
 	}
 }
 
+func (a *ServiceReconciler) isTailscaleService(svc *corev1.Service) bool {
+	targetIP := tailnetTargetAnnotation(svc)
+	targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN]
+	return a.shouldExpose(svc) || targetIP != "" || targetFQDN != ""
+}
+
 func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
 	logger := a.logger.With("service-ns", req.Namespace, "service-name", req.Name)
 	logger.Debugf("starting reconcile")
@@ -82,9 +108,8 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	} else if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
-	targetIP := a.tailnetTargetAnnotation(svc)
-	targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN]
-	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && targetIP == "" && targetFQDN == "" {
+
+	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
 	}
@@ -96,7 +121,14 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 //
 // This function is responsible for removing the finalizer from the service,
 // once all associated resources are gone.
-func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
+	oldSvcStatus := svc.Status.DeepCopy()
+	defer func() {
+		if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			err = errors.Join(err, a.Client.Status().Update(ctx, svc))
+		}
+	}()
 	ix := slices.Index(svc.Finalizers, FinalizerName)
 	if ix < 0 {
 		logger.Debugf("no finalizer, nothing to do")
@@ -106,6 +138,10 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 		a.managedEgressProxies.Remove(svc.UID)
 		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
 		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+
+		if !a.isTailscaleService(svc) {
+			tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady)
+		}
 		return nil
 	}
 
@@ -125,7 +161,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 	// exactly once at the very end of cleanup, because the final step of
 	// cleanup removes the tailscale finalizer, which will make all future
 	// reconciles exit early.
-	logger.Infof("unexposed service from tailnet")
+	logger.Infof("unexposed Service from tailnet")
 
 	a.mu.Lock()
 	defer a.mu.Unlock()
@@ -133,6 +169,10 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 	a.managedEgressProxies.Remove(svc.UID)
 	gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
 	gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+
+	if !a.isTailscaleService(svc) {
+		tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady)
+	}
 	return nil
 }
 
@@ -141,7 +181,15 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 //
 // This function adds a finalizer to svc, ensuring that we can handle orderly
 // deprovisioning later.
-func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
+	oldSvcStatus := svc.Status.DeepCopy()
+	defer func() {
+		if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			err = errors.Join(err, a.Client.Status().Update(ctx, svc))
+		}
+	}()
+
 	// Run for proxy config related validations here as opposed to running
 	// them earlier. This is to prevent cleanup being blocked on a
 	// misconfigured proxy param.
@@ -149,30 +197,31 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err)
 		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg)
 		a.logger.Error(msg)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
 		return nil
 	}
 	if violations := validateService(svc); len(violations) > 0 {
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", "))
-		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVCICE", msg)
+		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
 		a.logger.Error(msg)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
 		return nil
 	}
 
 	proxyClass := proxyClassForObject(svc)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
-			return fmt.Errorf("error verifying ProxyClass for Service: %w", err)
+			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
+			tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+			return errMsg
 		} else if !ready {
-			logger.Infof("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass)
+			msg := fmt.Sprintf("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass)
+			tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
+			logger.Info(msg)
 			return nil
 		}
 	}
 
-	hostname, err := nameForService(svc)
-	if err != nil {
-		return err
-	}
-
 	if !slices.Contains(svc.Finalizers, FinalizerName) {
 		// This log line is printed exactly once during initial provisioning,
 		// because once the finalizer is in place this block gets skipped. So,
@@ -181,7 +230,9 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		logger.Infof("exposing service over tailscale")
 		svc.Finalizers = append(svc.Finalizers, FinalizerName)
 		if err := a.Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to add finalizer: %w", err)
+			errMsg := fmt.Errorf("failed to add finalizer: %w", err)
+			tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+			return errMsg
 		}
 	}
 	crl := childResourceLabels(svc.Name, svc.Namespace, "svc")
@@ -193,18 +244,22 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	sts := &tailscaleSTSConfig{
 		ParentResourceName:  svc.Name,
 		ParentResourceUID:   string(svc.UID),
-		Hostname:            hostname,
+		Hostname:            nameForService(svc),
 		Tags:                tags,
 		ChildResourceLabels: crl,
-		ProxyClass:          proxyClass,
+		ProxyClassName:      proxyClass,
 	}
 
 	a.mu.Lock()
-	if a.shouldExpose(svc) {
+	if a.shouldExposeClusterIP(svc) {
 		sts.ClusterTargetIP = svc.Spec.ClusterIP
 		a.managedIngressProxies.Add(svc.UID)
 		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
-	} else if ip := a.tailnetTargetAnnotation(svc); ip != "" {
+	} else if a.shouldExposeDNSName(svc) {
+		sts.ClusterTargetDNSName = svc.Spec.ExternalName
+		a.managedIngressProxies.Add(svc.UID)
+		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
+	} else if ip := tailnetTargetAnnotation(svc); ip != "" {
 		sts.TailnetTargetIP = ip
 		a.managedEgressProxies.Add(svc.UID)
 		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
@@ -221,27 +276,31 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 
 	var hsvc *corev1.Service
 	if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil {
-		return fmt.Errorf("failed to provision: %w", err)
+		errMsg := fmt.Errorf("failed to provision: %w", err)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+		return errMsg
 	}
 
-	if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" {
-		// TODO (irbekrm): cluster.local is the default DNS name, but
-		// can be changed by users. Make this configurable or figure out
-		// how to discover the DNS name from within operator
-		headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc.cluster.local"
+	if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" { // if an egress proxy
+		clusterDomain := retrieveClusterDomain(a.tsNamespace, logger)
+		headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc." + clusterDomain
 		if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName {
 			svc.Spec.ExternalName = headlessSvcName
 			svc.Spec.Selector = nil
 			svc.Spec.Type = corev1.ServiceTypeExternalName
 			if err := a.Update(ctx, svc); err != nil {
-				return fmt.Errorf("failed to update service: %w", err)
+				errMsg := fmt.Errorf("failed to update service: %w", err)
+				tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+				return errMsg
 			}
 		}
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
 		return nil
 	}
 
-	if !a.hasLoadBalancerClass(svc) {
+	if !isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) {
 		logger.Debugf("service is not a LoadBalancer, so not updating ingress")
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
 		return nil
 	}
 
@@ -250,22 +309,23 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return fmt.Errorf("failed to get device ID: %w", err)
 	}
 	if tsHost == "" {
-		logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
+		msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
+		logger.Debug(msg)
 		// No hostname yet. Wait for the proxy pod to auth.
 		svc.Status.LoadBalancer.Ingress = nil
-		if err := a.Status().Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to update service status: %w", err)
-		}
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
 		return nil
 	}
 
-	logger.Debugf("setting ingress to %q, %s", tsHost, strings.Join(tsIPs, ", "))
+	logger.Debugf("setting Service LoadBalancer status to %q, %s", tsHost, strings.Join(tsIPs, ", "))
 	ingress := []corev1.LoadBalancerIngress{
 		{Hostname: tsHost},
 	}
 	clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
 	if err != nil {
-		return fmt.Errorf("failed to parse cluster IP: %w", err)
+		msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
+		return fmt.Errorf(msg)
 	}
 	for _, ip := range tsIPs {
 		addr, err := netip.ParseAddr(ip)
@@ -277,53 +337,64 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 	svc.Status.LoadBalancer.Ingress = ingress
-	if err := a.Status().Update(ctx, svc); err != nil {
-		return fmt.Errorf("failed to update service status: %w", err)
-	}
+	tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
 	return nil
 }
 
 func validateService(svc *corev1.Service) []string {
 	violations := make([]string, 0)
 	if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" {
-		violations = append(violations, "only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN)
+		violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN))
 	}
 	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
 		if !isMagicDNSName(fqdn) {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
+	svcName := nameForService(svc)
+	if err := dnsname.ValidLabel(svcName); err != nil {
+		if _, ok := svc.Annotations[AnnotationHostname]; ok {
+			violations = append(violations, fmt.Sprintf("invalid Tailscale hostname specified %q: %s", svcName, err))
+		} else {
+			violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err))
+		}
+	}
 	return violations
 }
 
 func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
-	// Headless services can't be exposed, since there is no ClusterIP to
-	// forward to.
+	return a.shouldExposeClusterIP(svc) || a.shouldExposeDNSName(svc)
+}
+
+func (a *ServiceReconciler) shouldExposeDNSName(svc *corev1.Service) bool {
+	return hasExposeAnnotation(svc) && svc.Spec.Type == corev1.ServiceTypeExternalName && svc.Spec.ExternalName != ""
+}
+
+func (a *ServiceReconciler) shouldExposeClusterIP(svc *corev1.Service) bool {
 	if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
 		return false
 	}
-
-	return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc)
+	return isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) || hasExposeAnnotation(svc)
 }
 
-func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
+func isTailscaleLoadBalancerService(svc *corev1.Service, isDefaultLoadBalancer bool) bool {
 	return svc != nil &&
 		svc.Spec.Type == corev1.ServiceTypeLoadBalancer &&
 		(svc.Spec.LoadBalancerClass != nil && *svc.Spec.LoadBalancerClass == "tailscale" ||
-			svc.Spec.LoadBalancerClass == nil && a.isDefaultLoadBalancer)
+			svc.Spec.LoadBalancerClass == nil && isDefaultLoadBalancer)
 }
 
 // hasExposeAnnotation reports whether Service has the tailscale.com/expose
 // annotation set
-func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
+func hasExposeAnnotation(svc *corev1.Service) bool {
 	return svc != nil && svc.Annotations[AnnotationExpose] == "true"
 }
 
-// hasTailnetTargetAnnotation returns the value of tailscale.com/tailnet-ip
+// tailnetTargetAnnotation returns the value of tailscale.com/tailnet-ip
 // annotation or of the deprecated tailscale.com/ts-tailnet-target-ip
 // annotation. If neither is set, it returns an empty string. If both are set,
 // it returns the value of the new annotation.
-func (a *ServiceReconciler) tailnetTargetAnnotation(svc *corev1.Service) string {
+func tailnetTargetAnnotation(svc *corev1.Service) string {
 	if svc == nil {
 		return ""
 	}
@@ -344,3 +415,51 @@ func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool
 	}
 	return tsoperator.ProxyClassIsReady(proxyClass), nil
 }
+
+// retrieveClusterDomain determines and retrieves cluster domain i.e
+// (cluster.local) in which this Pod is running by parsing search domains in
+// /etc/resolv.conf. If an error is encountered at any point during the process,
+// defaults cluster domain to 'cluster.local'.
+func retrieveClusterDomain(namespace string, logger *zap.SugaredLogger) string {
+	logger.Infof("attempting to retrieve cluster domain..")
+	conf, err := resolvconffile.ParseFile(resolvConfPath)
+	if err != nil {
+		// Vast majority of clusters use the cluster.local domain, so it
+		// is probably better to fall back to that than error out.
+		logger.Infof("[unexpected] error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.")
+		return defaultClusterDomain
+	}
+	return clusterDomainFromResolverConf(conf, namespace, logger)
+}
+
+// clusterDomainFromResolverConf attempts to retrieve cluster domain from the provided resolver config.
+// It expects the first three search domains in the resolver config to be be ['<namespace>.svc.<cluster-domain>, svc.<cluster-domain>, <cluster-domain>, ...]
+// If the first three domains match the expected structure, it returns the third.
+// If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain.
+func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string {
+	if len(conf.SearchDomains) < 3 {
+		logger.Infof("[unexpected] resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.")
+		return defaultClusterDomain
+	}
+	first := conf.SearchDomains[0]
+	if !strings.HasPrefix(string(first), namespace+".svc") {
+		logger.Infof("[unexpected] first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.<cluster-domain>")
+		return defaultClusterDomain
+	}
+	second := conf.SearchDomains[1]
+	if !strings.HasPrefix(string(second), "svc") {
+		logger.Infof("[unexpected] second search domain in resolver config is %s; expected 'svc.<cluster-domain>'.\nDefaulting cluster domain to 'cluster.local'.", second)
+		return defaultClusterDomain
+	}
+	// Trim the trailing dot for backwards compatibility purposes as the
+	// cluster domain was previously hardcoded to 'cluster.local' without a
+	// trailing dot.
+	probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.")
+	third := conf.SearchDomains[2]
+	if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) {
+		logger.Infof("[unexpected] expected resolver config to contain serch domains <namespace>.svc.<cluster-domain>, svc.<cluster-domain>, <cluster-domain>; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third)
+		return defaultClusterDomain
+	}
+	logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain)
+	return probablyClusterDomain
+}

cmd/k8s-operator/testutils_test.go

@@ -15,11 +15,13 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/client/tailscale"
@@ -42,6 +44,7 @@ type configOpts struct {
 	tailnetTargetIP                                string
 	tailnetTargetFQDN                              string
 	clusterTargetIP                                string
+	clusterTargetDNS                               string
 	subnetRoutes                                   string
 	isExitNode                                     bool
 	confFileHash                                   string
@@ -52,6 +55,10 @@ type configOpts struct {
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
 	t.Helper()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
 	tsContainer := corev1.Container{
 		Name:  "tailscale",
 		Image: "tailscale/tailscale",
@@ -60,6 +67,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
+			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
 			Capabilities: &corev1.Capabilities{
@@ -82,12 +90,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: opts.secretName,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "tailscaled",
-							Path: "tailscaled",
-						},
-					},
 				},
 			},
 		},
@@ -125,15 +127,19 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			Value: opts.clusterTargetIP,
 		})
 		annots["tailscale.com/operator-last-set-cluster-ip"] = opts.clusterTargetIP
+	} else if opts.clusterTargetDNS != "" {
+		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
+			Name:  "TS_EXPERIMENTAL_DEST_DNS_NAME",
+			Value: opts.clusterTargetDNS,
+		})
+		annots["tailscale.com/operator-last-set-cluster-dns-name"] = opts.clusterTargetDNS
 	}
 	if opts.serveConfig != nil {
 		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
 			Name:  "TS_SERVE_CONFIG",
 			Value: "/etc/tailscaled/serve-config",
 		})
-		volumes = append(volumes, corev1.Volume{
-			Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Path: "serve-config", Key: "serve-config"}}}},
-		})
+		volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
 		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
 	}
 	ss := &appsv1.StatefulSet{
@@ -176,8 +182,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 						{
 							Name:    "sysctler",
 							Image:   "tailscale/tailscale",
-							Command: []string{"/bin/sh"},
-							Args:    []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
+							Command: []string{"/bin/sh", "-c"},
+							Args:    []string{"sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi"},
 							SecurityContext: &corev1.SecurityContext{
 								Privileged: ptr.To(true),
 							},
@@ -197,20 +203,26 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
 			t.Fatalf("error getting ProxyClass: %v", err)
 		}
-		return applyProxyClassToStatefulSet(proxyClass, ss)
+		return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar())
 	}
 	return ss
 }
 
 func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
 	t.Helper()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
 	tsContainer := corev1.Container{
 		Name:  "tailscale",
 		Image: "tailscale/tailscale",
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
+			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
+			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 		},
 		ImagePullPolicy: "Always",
@@ -225,20 +237,12 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: opts.secretName,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "tailscaled",
-							Path: "tailscaled",
-						},
-					},
 				},
 			},
 		},
 		{Name: "serve-config",
 			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName,
-					Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}},
-		},
+				Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}},
 	}
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
@@ -293,17 +297,13 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 		if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
 			t.Fatalf("error getting ProxyClass: %v", err)
 		}
-		return applyProxyClassToStatefulSet(proxyClass, ss)
+		return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar())
 	}
 	return ss
 }
 
 func expectedHeadlessService(name string, parentType string) *corev1.Service {
 	return &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:         name,
 			GenerateName: "ts-test-",
@@ -319,18 +319,15 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
 			Selector: map[string]string{
 				"app": "1234-UID",
 			},
-			ClusterIP: "None",
+			ClusterIP:      "None",
+			IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack),
 		},
 	}
 }
 
-func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
+func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
 	t.Helper()
 	s := &corev1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      opts.secretName,
 			Namespace: "operator-ns",
@@ -351,6 +348,16 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 		AuthKey:      ptr.To("secret-authkey"),
 		AcceptRoutes: "false",
 	}
+	if opts.proxyClass != "" {
+		t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
+		proxyClass := new(tsapi.ProxyClass)
+		if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
+			t.Fatalf("error getting ProxyClass: %v", err)
+		}
+		if proxyClass.Spec.TailscaleConfig != nil && proxyClass.Spec.TailscaleConfig.AcceptRoutes {
+			conf.AcceptRoutes = "true"
+		}
+	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -370,7 +377,17 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
+	if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
+		conf.NoStatefulFiltering = "true"
+	} else {
+		conf.NoStatefulFiltering = "false"
+	}
+	bn, err := json.Marshal(conf)
+	if err != nil {
+		t.Fatalf("error marshalling tailscaled config")
+	}
 	mak.Set(&s.StringData, "tailscaled", string(b))
+	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
 	labels := map[string]string{
 		"tailscale.com/managed":              "true",
 		"tailscale.com/parent-resource":      "test",
@@ -441,11 +458,11 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client,
 
 // expectEqual accepts a Kubernetes object and a Kubernetes client. It tests
 // whether an object with equivalent contents can be retrieved by the passed
-// client. If you want to NOT test some object fields for equality, ensure that
-// they are not present in the passed object and use the modify func to remove
-// them from the cluster object. If no such modifications are needed, you can
-// pass nil in place of the modify function.
-func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modify func(O)) {
+// client. If you want to NOT test some object fields for equality, use the
+// modify func to ensure that they are removed from the cluster object and the
+// object passed as 'want'. If no such modifications are needed, you can pass
+// nil in place of the modify function.
+func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) {
 	t.Helper()
 	got := O(new(T))
 	if err := client.Get(context.Background(), types.NamespacedName{
@@ -459,8 +476,9 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 	// so just remove it from both got and want.
 	got.SetResourceVersion("")
 	want.SetResourceVersion("")
-	if modify != nil {
-		modify(got)
+	if modifier != nil {
+		modifier(want)
+		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
 		t.Fatalf("unexpected object (-got +want):\n%s", diff)
@@ -515,6 +533,34 @@ func expectRequeue(t *testing.T, sr reconcile.Reconciler, ns, name string) {
 	}
 }
 
+// expectEvents accepts a test recorder and a list of events, tests that expected
+// events are sent down the recorder's channel. Waits for 5s for each event.
+func expectEvents(t *testing.T, rec *record.FakeRecorder, wantsEvents []string) {
+	t.Helper()
+	// Events are not expected to arrive in order.
+	seenEvents := make([]string, 0)
+	for range len(wantsEvents) {
+		timer := time.NewTimer(time.Second * 5)
+		defer timer.Stop()
+		select {
+		case gotEvent := <-rec.Events:
+			found := false
+			for _, wantEvent := range wantsEvents {
+				if wantEvent == gotEvent {
+					found = true
+					seenEvents = append(seenEvents, gotEvent)
+					break
+				}
+			}
+			if !found {
+				t.Errorf("got unexpected event %q, expected events: %+#v", gotEvent, wantsEvents)
+			}
+		case <-timer.C:
+			t.Errorf("timeout waiting for an event, wants events %#+v, got events %+#v", wantsEvents, seenEvents)
+		}
+	}
+}
+
 type fakeTSClient struct {
 	sync.Mutex
 	keyRequests []tailscale.KeyCapabilities
@@ -566,3 +612,33 @@ func (c *fakeTSClient) Deleted() []string {
 func removeHashAnnotation(sts *appsv1.StatefulSet) {
 	delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
 }
+
+func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
+	return func(secret *corev1.Secret) {
+		t.Helper()
+		if len(secret.StringData["tailscaled"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
+				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
+			}
+			mak.Set(&secret.StringData, "tailscaled", string(b))
+		}
+		if len(secret.StringData["cap-95.hujson"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
+				t.Fatalf("error umarshalling 'cap-95.hujson' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling 'cap-95.huson' contents: %v", err)
+			}
+			mak.Set(&secret.StringData, "cap-95.hujson", string(b))
+		}
+	}
+}

cmd/natc/natc.go

@@ -0,0 +1,567 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The natc command is a work-in-progress implementation of a NAT based
+// connector for Tailscale. It is intended to be used to route traffic to a
+// specific domain through a specific node.
+package main
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand/v2"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gaissmai/bart"
+	"github.com/inetaf/tcpproxy"
+	"github.com/peterbourgon/ff/v3"
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/envknob"
+	"tailscale.com/hostinfo"
+	"tailscale.com/ipn"
+	"tailscale.com/net/netutil"
+	"tailscale.com/syncs"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tsnet"
+	"tailscale.com/tsweb"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/mak"
+)
+
+func main() {
+	hostinfo.SetApp("natc")
+	if !envknob.UseWIPCode() {
+		log.Fatal("cmd/natc is a work in progress and has not been security reviewed;\nits use requires TAILSCALE_USE_WIP_CODE=1 be set in the environment for now.")
+	}
+
+	// Parse flags
+	fs := flag.NewFlagSet("natc", flag.ExitOnError)
+	var (
+		debugPort       = fs.Int("debug-port", 8893, "Listening port for debug/metrics endpoint")
+		hostname        = fs.String("hostname", "", "Hostname to register the service under")
+		siteID          = fs.Uint("site-id", 1, "an integer site ID to use for the ULA prefix which allows for multiple proxies to act in a HA configuration")
+		v4PfxStr        = fs.String("v4-pfx", "100.64.1.0/24", "comma-separated list of IPv4 prefixes to advertise")
+		verboseTSNet    = fs.Bool("verbose-tsnet", false, "enable verbose logging in tsnet")
+		printULA        = fs.Bool("print-ula", false, "print the ULA prefix and exit")
+		ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore")
+		wgPort          = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic")
+	)
+	ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC"))
+
+	if *printULA {
+		fmt.Println(ula(uint16(*siteID)))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if *siteID == 0 {
+		log.Fatalf("site-id must be set")
+	} else if *siteID > 0xffff {
+		log.Fatalf("site-id must be in the range [0, 65535]")
+	}
+
+	var ignoreDstTable *bart.Table[bool]
+	for _, s := range strings.Split(*ignoreDstPfxStr, ",") {
+		s := strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		if ignoreDstTable == nil {
+			ignoreDstTable = &bart.Table[bool]{}
+		}
+		pfx, err := netip.ParsePrefix(s)
+		if err != nil {
+			log.Fatalf("unable to parse prefix: %v", err)
+		}
+		if pfx.Masked() != pfx {
+			log.Fatalf("prefix %v is not normalized (bits are set outside the mask)", pfx)
+		}
+		ignoreDstTable.Insert(pfx, true)
+	}
+	var v4Prefixes []netip.Prefix
+	for _, s := range strings.Split(*v4PfxStr, ",") {
+		p := netip.MustParsePrefix(strings.TrimSpace(s))
+		if p.Masked() != p {
+			log.Fatalf("v4 prefix %v is not a masked prefix", p)
+		}
+		v4Prefixes = append(v4Prefixes, p)
+	}
+	if len(v4Prefixes) == 0 {
+		log.Fatalf("no v4 prefixes specified")
+	}
+	dnsAddr := v4Prefixes[0].Addr()
+	ts := &tsnet.Server{
+		Hostname: *hostname,
+	}
+	if *wgPort != 0 {
+		if *wgPort >= 1<<16 {
+			log.Fatalf("wg-port must be in the range [0, 65535]")
+		}
+		ts.Port = uint16(*wgPort)
+	}
+	defer ts.Close()
+	if *verboseTSNet {
+		ts.Logf = log.Printf
+	}
+
+	// Start special-purpose listeners: dns, http promotion, debug server
+	if *debugPort != 0 {
+		mux := http.NewServeMux()
+		tsweb.Debugger(mux)
+		dln, err := ts.Listen("tcp", fmt.Sprintf(":%d", *debugPort))
+		if err != nil {
+			log.Fatalf("failed listening on debug port: %v", err)
+		}
+		defer dln.Close()
+		go func() {
+			log.Fatalf("debug serve: %v", http.Serve(dln, mux))
+		}()
+	}
+	lc, err := ts.LocalClient()
+	if err != nil {
+		log.Fatalf("LocalClient() failed: %v", err)
+	}
+	if _, err := ts.Up(ctx); err != nil {
+		log.Fatalf("ts.Up: %v", err)
+	}
+
+	c := &connector{
+		ts:         ts,
+		lc:         lc,
+		dnsAddr:    dnsAddr,
+		v4Ranges:   v4Prefixes,
+		v6ULA:      ula(uint16(*siteID)),
+		ignoreDsts: ignoreDstTable,
+	}
+	c.run(ctx)
+}
+
+type connector struct {
+	// ts is the tsnet.Server used to host the connector.
+	ts *tsnet.Server
+	// lc is the LocalClient used to interact with the tsnet.Server hosting this
+	// connector.
+	lc *tailscale.LocalClient
+
+	// dnsAddr is the IPv4 address to listen on for DNS requests. It is used to
+	// prevent the app connector from assigning it to a domain.
+	dnsAddr netip.Addr
+
+	// v4Ranges is the list of IPv4 ranges to advertise and assign addresses from.
+	// These are masked prefixes.
+	v4Ranges []netip.Prefix
+	// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
+	v6ULA netip.Prefix
+
+	perPeerMap syncs.Map[tailcfg.NodeID, *perPeerState]
+
+	// ignoreDsts is initialized at start up with the contents of --ignore-destinations (if none it is nil)
+	// It is never mutated, only used for lookups.
+	// Users who want to natc a DNS wildcard but not every address record in that domain can supply the
+	// exceptions in --ignore-destinations. When we receive a dns request we will look up the fqdn
+	// and if any of the ip addresses in response to the lookup match any 'ignore destinations' prefix we will
+	// return a dns response that contains the ip addresses we discovered with the lookup (ie not the
+	// natc behavior, which would return a dummy ip address pointing at natc).
+	ignoreDsts *bart.Table[bool]
+}
+
+// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
+// The 8th and 9th bytes are used to encode the site ID which allows for
+// multiple proxies to act in a HA configuration.
+// mnemonic: a99c = appc
+var v6ULA = netip.MustParsePrefix("fd7a:115c:a1e0:a99c::/64")
+
+func ula(siteID uint16) netip.Prefix {
+	as16 := v6ULA.Addr().As16()
+	as16[8] = byte(siteID >> 8)
+	as16[9] = byte(siteID)
+	return netip.PrefixFrom(netip.AddrFrom16(as16), 64+16)
+}
+
+// run runs the connector.
+//
+// The passed in context is only used for the initial setup. The connector runs
+// forever.
+func (c *connector) run(ctx context.Context) {
+	if _, err := c.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+		AdvertiseRoutesSet: true,
+		Prefs: ipn.Prefs{
+			AdvertiseRoutes: append(c.v4Ranges, c.v6ULA),
+		},
+	}); err != nil {
+		log.Fatalf("failed to advertise routes: %v", err)
+	}
+	c.ts.RegisterFallbackTCPHandler(c.handleTCPFlow)
+	c.serveDNS()
+}
+
+func (c *connector) serveDNS() {
+	pc, err := c.ts.ListenPacket("udp", net.JoinHostPort(c.dnsAddr.String(), "53"))
+	if err != nil {
+		log.Fatalf("failed listening on port 53: %v", err)
+	}
+	defer pc.Close()
+	log.Printf("Listening for DNS on %s", pc.LocalAddr().String())
+	for {
+		buf := make([]byte, 1500)
+		n, addr, err := pc.ReadFrom(buf)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			log.Printf("serveDNS.ReadFrom failed: %v", err)
+			continue
+		}
+		go c.handleDNS(pc, buf[:n], addr.(*net.UDPAddr))
+	}
+}
+
+func lookupDestinationIP(domain string) ([]netip.Addr, error) {
+	netIPs, err := net.LookupIP(domain)
+	if err != nil {
+		var dnsError *net.DNSError
+		if errors.As(err, &dnsError) && dnsError.IsNotFound {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+	var addrs []netip.Addr
+	for _, ip := range netIPs {
+		a, ok := netip.AddrFromSlice(ip)
+		if ok {
+			addrs = append(addrs, a)
+		}
+	}
+	return addrs, nil
+}
+
+// handleDNS handles a DNS request to the app connector.
+// It generates a response based on the request and the node that sent it.
+//
+// Each node is assigned a unique pair of IP addresses for each domain it
+// queries. This assignment is done lazily and is not persisted across restarts.
+// A per-peer assignment allows the connector to reuse a limited number of IP
+// addresses across multiple nodes and domains. It also allows for clear
+// failover behavior when an app connector is restarted.
+//
+// This assignment later allows the connector to determine where to forward
+// traffic based on the destination IP address.
+func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDPAddr) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	who, err := c.lc.WhoIs(ctx, remoteAddr.String())
+	if err != nil {
+		log.Printf("HandleDNS: WhoIs failed: %v\n", err)
+		return
+	}
+
+	var msg dnsmessage.Message
+	err = msg.Unpack(buf)
+	if err != nil {
+		log.Printf("HandleDNS: dnsmessage unpack failed: %v\n ", err)
+		return
+	}
+
+	// If there are destination ips that we don't want to route, we
+	// have to do a dns lookup here to find the destination ip.
+	if c.ignoreDsts != nil {
+		if len(msg.Questions) > 0 {
+			q := msg.Questions[0]
+			switch q.Type {
+			case dnsmessage.TypeAAAA, dnsmessage.TypeA:
+				dstAddrs, err := lookupDestinationIP(q.Name.String())
+				if err != nil {
+					log.Printf("HandleDNS: lookup destination failed: %v\n ", err)
+					return
+				}
+				if c.ignoreDestination(dstAddrs) {
+					bs, err := dnsResponse(&msg, dstAddrs)
+					// TODO (fran): treat as SERVFAIL
+					if err != nil {
+						log.Printf("HandleDNS: generate ignore response failed: %v\n", err)
+						return
+					}
+					_, err = pc.WriteTo(bs, remoteAddr)
+					if err != nil {
+						log.Printf("HandleDNS: write failed: %v\n", err)
+					}
+					return
+				}
+			}
+		}
+	}
+	// None of the destination IP addresses match an ignore destination prefix, do
+	// the natc thing.
+
+	resp, err := c.generateDNSResponse(&msg, who.Node.ID)
+	// TODO (fran): treat as SERVFAIL
+	if err != nil {
+		log.Printf("HandleDNS: connector handling failed: %v\n", err)
+		return
+	}
+	// TODO (fran): treat as NXDOMAIN
+	if len(resp) == 0 {
+		return
+	}
+	// This connector handled the DNS request
+	_, err = pc.WriteTo(resp, remoteAddr)
+	if err != nil {
+		log.Printf("HandleDNS: write failed: %v\n", err)
+	}
+}
+
+// tsMBox is the mailbox used in SOA records.
+// The convention is to replace the @ symbol with a dot.
+// So in this case, the mailbox is support.tailscale.com. with the trailing dot
+// to indicate that it is a fully qualified domain name.
+var tsMBox = dnsmessage.MustNewName("support.tailscale.com.")
+
+// generateDNSResponse generates a DNS response for the given request. The from
+// argument is the NodeID of the node that sent the request.
+func (c *connector) generateDNSResponse(req *dnsmessage.Message, from tailcfg.NodeID) ([]byte, error) {
+	pm, _ := c.perPeerMap.LoadOrStore(from, &perPeerState{c: c})
+	var addrs []netip.Addr
+	if len(req.Questions) > 0 {
+		switch req.Questions[0].Type {
+		case dnsmessage.TypeAAAA, dnsmessage.TypeA:
+			var err error
+			addrs, err = pm.ipForDomain(req.Questions[0].Name.String())
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return dnsResponse(req, addrs)
+}
+
+// dnsResponse makes a DNS response for the natc. If the dnsmessage is requesting TypeAAAA
+// or TypeA the provided addrs of the requested type will be used.
+func dnsResponse(req *dnsmessage.Message, addrs []netip.Addr) ([]byte, error) {
+	b := dnsmessage.NewBuilder(nil,
+		dnsmessage.Header{
+			ID:            req.Header.ID,
+			Response:      true,
+			Authoritative: true,
+		})
+	b.EnableCompression()
+
+	if len(req.Questions) == 0 {
+		return b.Finish()
+	}
+	q := req.Questions[0]
+	if err := b.StartQuestions(); err != nil {
+		return nil, err
+	}
+	if err := b.Question(q); err != nil {
+		return nil, err
+	}
+	if err := b.StartAnswers(); err != nil {
+		return nil, err
+	}
+	switch q.Type {
+	case dnsmessage.TypeAAAA, dnsmessage.TypeA:
+		want6 := q.Type == dnsmessage.TypeAAAA
+		for _, ip := range addrs {
+			if want6 != ip.Is6() {
+				continue
+			}
+			if want6 {
+				if err := b.AAAAResource(
+					dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 5},
+					dnsmessage.AAAAResource{AAAA: ip.As16()},
+				); err != nil {
+					return nil, err
+				}
+			} else {
+				if err := b.AResource(
+					dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 5},
+					dnsmessage.AResource{A: ip.As4()},
+				); err != nil {
+					return nil, err
+				}
+			}
+		}
+	case dnsmessage.TypeSOA:
+		if err := b.SOAResource(
+			dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 120},
+			dnsmessage.SOAResource{NS: q.Name, MBox: tsMBox, Serial: 2023030600,
+				Refresh: 120, Retry: 120, Expire: 120, MinTTL: 60},
+		); err != nil {
+			return nil, err
+		}
+	case dnsmessage.TypeNS:
+		if err := b.NSResource(
+			dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 120},
+			dnsmessage.NSResource{NS: tsMBox},
+		); err != nil {
+			return nil, err
+		}
+	}
+	return b.Finish()
+}
+
+// handleTCPFlow handles a TCP flow from the given source to the given
+// destination. It uses the source address to determine the node that sent the
+// request and the destination address to determine the domain that the request
+// is for based on the IP address assigned to the destination in the DNS
+// response.
+func (c *connector) handleTCPFlow(src, dst netip.AddrPort) (handler func(net.Conn), intercept bool) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	who, err := c.lc.WhoIs(ctx, src.Addr().String())
+	cancel()
+	if err != nil {
+		log.Printf("HandleTCPFlow: WhoIs failed: %v\n", err)
+		return nil, false
+	}
+
+	from := who.Node.ID
+	ps, ok := c.perPeerMap.Load(from)
+	if !ok {
+		log.Printf("handleTCPFlow: no perPeerState for %v", from)
+		return nil, false
+	}
+	domain, ok := ps.domainForIP(dst.Addr())
+	if !ok {
+		log.Printf("handleTCPFlow: no domain for IP %v\n", dst.Addr())
+		return nil, false
+	}
+	return func(conn net.Conn) {
+		proxyTCPConn(conn, domain)
+	}, true
+}
+
+// ignoreDestination reports whether any of the provided dstAddrs match the prefixes configured
+// in --ignore-destinations
+func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool {
+	for _, a := range dstAddrs {
+		if _, ok := c.ignoreDsts.Lookup(a); ok {
+			return true
+		}
+	}
+	return false
+}
+
+func proxyTCPConn(c net.Conn, dest string) {
+	addrPortStr := c.LocalAddr().String()
+	_, port, err := net.SplitHostPort(addrPortStr)
+	if err != nil {
+		log.Printf("tcpRoundRobinHandler.Handle: bogus addrPort %q", addrPortStr)
+		c.Close()
+		return
+	}
+
+	p := &tcpproxy.Proxy{
+		ListenFunc: func(net, laddr string) (net.Listener, error) {
+			return netutil.NewOneConnListener(c, nil), nil
+		},
+	}
+	p.AddRoute(addrPortStr, &tcpproxy.DialProxy{
+		Addr: fmt.Sprintf("%s:%s", dest, port),
+	})
+	p.Start()
+}
+
+// perPeerState holds the state for a single peer.
+type perPeerState struct {
+	c *connector
+
+	mu           sync.Mutex
+	domainToAddr map[string][]netip.Addr
+	addrToDomain *bart.Table[string]
+}
+
+// domainForIP returns the domain name assigned to the given IP address and
+// whether it was found.
+func (ps *perPeerState) domainForIP(ip netip.Addr) (_ string, ok bool) {
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+	return ps.addrToDomain.Lookup(ip)
+}
+
+// ipForDomain assigns a pair of unique IP addresses for the given domain and
+// returns them. The first address is an IPv4 address and the second is an IPv6
+// address. If the domain already has assigned addresses, it returns them.
+func (ps *perPeerState) ipForDomain(domain string) ([]netip.Addr, error) {
+	fqdn, err := dnsname.ToFQDN(domain)
+	if err != nil {
+		return nil, err
+	}
+	domain = fqdn.WithoutTrailingDot()
+
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+	if addrs, ok := ps.domainToAddr[domain]; ok {
+		return addrs, nil
+	}
+	addrs := ps.assignAddrsLocked(domain)
+	return addrs, nil
+}
+
+// isIPUsedLocked reports whether the given IP address is already assigned to a
+// domain.
+// ps.mu must be held.
+func (ps *perPeerState) isIPUsedLocked(ip netip.Addr) bool {
+	_, ok := ps.addrToDomain.Lookup(ip)
+	return ok
+}
+
+// unusedIPv4Locked returns an unused IPv4 address from the available ranges.
+func (ps *perPeerState) unusedIPv4Locked() netip.Addr {
+	// TODO: skip ranges that have been exhausted
+	for _, r := range ps.c.v4Ranges {
+		ip := randV4(r)
+		for r.Contains(ip) {
+			if !ps.isIPUsedLocked(ip) && ip != ps.c.dnsAddr {
+				return ip
+			}
+			ip = ip.Next()
+		}
+	}
+	return netip.Addr{}
+}
+
+// randV4 returns a random IPv4 address within the given prefix.
+func randV4(maskedPfx netip.Prefix) netip.Addr {
+	bits := 32 - maskedPfx.Bits()
+	randBits := rand.Uint32N(1 << uint(bits))
+
+	ip4 := maskedPfx.Addr().As4()
+	pn := binary.BigEndian.Uint32(ip4[:])
+	binary.BigEndian.PutUint32(ip4[:], randBits|pn)
+	return netip.AddrFrom4(ip4)
+}
+
+// assignAddrsLocked assigns a pair of unique IP addresses for the given domain
+// and returns them. The first address is an IPv4 address and the second is an
+// IPv6 address. It does not check if the domain already has assigned addresses.
+// ps.mu must be held.
+func (ps *perPeerState) assignAddrsLocked(domain string) []netip.Addr {
+	if ps.addrToDomain == nil {
+		ps.addrToDomain = &bart.Table[string]{}
+	}
+	v4 := ps.unusedIPv4Locked()
+	as16 := ps.c.v6ULA.Addr().As16()
+	as4 := v4.As4()
+	copy(as16[12:], as4[:])
+	v6 := netip.AddrFrom16(as16)
+	addrs := []netip.Addr{v4, v6}
+	mak.Set(&ps.domainToAddr, domain, addrs)
+	for _, a := range addrs {
+		ps.addrToDomain.Insert(netip.PrefixFrom(a, a.BitLen()), domain)
+	}
+	return addrs
+}

cmd/netlogfmt/main.go

@@ -314,7 +314,7 @@ func mustMakeNamesByAddr() map[netip.Addr]string {
 	seen := make(map[string]bool)
 	namesByAddr := make(map[netip.Addr]string)
 retry:
-	for i := 0; i < 10; i++ {
+	for i := range 10 {
 		clear(seen)
 		clear(namesByAddr)
 		for _, d := range m.Devices {
@@ -354,7 +354,7 @@ func fieldPrefix(s string, n int) string {
 }
 
 func appendRepeatByte(b []byte, c byte, n int) []byte {
-	for i := 0; i < n; i++ {
+	for range n {
 		b = append(b, c)
 	}
 	return b

cmd/pgproxy/pgproxy.go

@@ -28,7 +28,6 @@ import (
 	"tailscale.com/metrics"
 	"tailscale.com/tsnet"
 	"tailscale.com/tsweb"
-	"tailscale.com/types/logger"
 )
 
 var (
@@ -58,8 +57,6 @@ func main() {
 	ts := &tsnet.Server{
 		Dir:      *tailscaleDir,
 		Hostname: *hostname,
-		// Make the stdout logs a clean audit log of connections.
-		Logf: logger.Discard,
 	}
 
 	if os.Getenv("TS_AUTHKEY") == "" {

cmd/proxy-to-grafana/proxy-to-grafana.go

@@ -46,6 +46,7 @@ var (
 	backendAddr  = flag.String("backend-addr", "", "Address of the Grafana server served over HTTP, in host:port format. Typically localhost:nnnn.")
 	tailscaleDir = flag.String("state-dir", "./", "Alternate directory to use for Tailscale state storage. If empty, a default is used.")
 	useHTTPS     = flag.Bool("use-https", false, "Serve over HTTPS via your *.ts.net subdomain if enabled in Tailscale admin.")
+	loginServer  = flag.String("login-server", "", "URL to alternative control server. If empty, the default Tailscale control is used.")
 )
 
 func main() {
@@ -57,8 +58,9 @@ func main() {
 		log.Fatal("missing --backend-addr")
 	}
 	ts := &tsnet.Server{
-		Dir:      *tailscaleDir,
-		Hostname: *hostname,
+		Dir:        *tailscaleDir,
+		Hostname:   *hostname,
+		ControlURL: *loginServer,
 	}
 
 	// TODO(bradfitz,maisem): move this to a method on tsnet.Server probably.
@@ -88,7 +90,7 @@ func main() {
 
 		go func() {
 			// wait for tailscale to start before trying to fetch cert names
-			for i := 0; i < 60; i++ {
+			for range 60 {
 				st, err := localClient.Status(context.Background())
 				if err != nil {
 					log.Printf("error retrieving tailscale status; retrying: %v", err)

cmd/sniproxy/handlers.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"math/rand"
+	"math/rand/v2"
 	"net"
 	"net/netip"
 	"slices"
@@ -47,7 +47,7 @@ func (h *tcpRoundRobinHandler) Handle(c net.Conn) {
 		return netutil.NewOneConnListener(c, nil), nil
 	}
 
-	dest := h.To[rand.Intn(len(h.To))]
+	dest := h.To[rand.IntN(len(h.To))]
 	dial := &tcpproxy.DialProxy{
 		Addr:        fmt.Sprintf("%s:%s", dest, port),
 		DialContext: h.DialContext,

cmd/sniproxy/sniproxy_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"log"
 	"net"
 	"net/http/httptest"
 	"net/netip"
@@ -24,6 +25,7 @@ import (
 	"tailscale.com/tsnet"
 	"tailscale.com/tstest/integration"
 	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/tstest/nettest"
 	"tailscale.com/types/appctype"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
@@ -98,8 +100,8 @@ func startNode(t *testing.T, ctx context.Context, controlURL, hostname string) (
 		Store:      new(mem.Store),
 		Ephemeral:  true,
 	}
-	if !*verboseNodes {
-		s.Logf = logger.Discard
+	if *verboseNodes {
+		s.Logf = log.Printf
 	}
 	t.Cleanup(func() { s.Close() })
 
@@ -111,6 +113,7 @@ func startNode(t *testing.T, ctx context.Context, controlURL, hostname string) (
 }
 
 func TestSNIProxyWithNetmapConfig(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
 	c, controlURL := startControl(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -158,7 +161,7 @@ func TestSNIProxyWithNetmapConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 	gotConfigured := false
-	for i := 0; i < 100; i++ {
+	for range 100 {
 		s, err := l.StatusWithoutPeers(ctx)
 		if err != nil {
 			t.Fatal(err)
@@ -189,6 +192,7 @@ func TestSNIProxyWithNetmapConfig(t *testing.T) {
 }
 
 func TestSNIProxyWithFlagConfig(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
 	_, controlURL := startControl(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()

cmd/stunc/stunc.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"strconv"
 
 	"tailscale.com/net/stun"
 )
@@ -15,12 +16,20 @@ import (
 func main() {
 	log.SetFlags(0)
 
-	if len(os.Args) != 2 {
-		log.Fatalf("usage: %s <hostname>", os.Args[0])
+	if len(os.Args) < 2 || len(os.Args) > 3 {
+		log.Fatalf("usage: %s <hostname> [port]", os.Args[0])
 	}
 	host := os.Args[1]
+	port := "3478"
+	if len(os.Args) == 3 {
+		port = os.Args[2]
+	}
+	_, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		log.Fatalf("invalid port: %v", err)
+	}
 
-	uaddr, err := net.ResolveUDPAddr("udp", host+":3478")
+	uaddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(host, port))
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/stund/depaware.txt

@@ -2,7 +2,13 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
 
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
-        github.com/google/uuid                                       from tailscale.com/tsweb
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
+        github.com/google/uuid                                       from tailscale.com/util/fastuuid
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
@@ -20,6 +26,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         google.golang.org/protobuf/internal/descfmt                  from google.golang.org/protobuf/internal/filedesc
         google.golang.org/protobuf/internal/descopts                 from google.golang.org/protobuf/internal/filedesc+
         google.golang.org/protobuf/internal/detrand                  from google.golang.org/protobuf/internal/descfmt+
+        google.golang.org/protobuf/internal/editiondefaults          from google.golang.org/protobuf/internal/filedesc
         google.golang.org/protobuf/internal/encoding/defval          from google.golang.org/protobuf/internal/encoding/tag+
         google.golang.org/protobuf/internal/encoding/messageset      from google.golang.org/protobuf/encoding/prototext+
         google.golang.org/protobuf/internal/encoding/tag             from google.golang.org/protobuf/internal/impl
@@ -58,13 +65,14 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/types/lazy                                     from tailscale.com/version+
         tailscale.com/types/logger                                   from tailscale.com/tsweb
         tailscale.com/types/opt                                      from tailscale.com/envknob+
-        tailscale.com/types/ptr                                      from tailscale.com/tailcfg
+        tailscale.com/types/ptr                                      from tailscale.com/tailcfg+
         tailscale.com/types/structs                                  from tailscale.com/tailcfg+
         tailscale.com/types/tkatype                                  from tailscale.com/tailcfg+
         tailscale.com/types/views                                    from tailscale.com/net/tsaddr+
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/tailcfg
+        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
         tailscale.com/util/lineread                                  from tailscale.com/version/distro
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
         tailscale.com/util/slicesx                                   from tailscale.com/tailcfg
@@ -126,6 +134,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
+        encoding/base32                                              from github.com/go-json-experiment/json
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
         encoding/hex                                                 from crypto/x509+
@@ -151,6 +160,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from math/big+
+        math/rand/v2                                                 from tailscale.com/util/fastuuid+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/stunstamp/stunstamp.go

@@ -0,0 +1,792 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The stunstamp binary measures STUN round-trip latency with DERPs.
+package main
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"math/rand/v2"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"os"
+	"os/signal"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/prompb"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/net/stun"
+	"tailscale.com/tailcfg"
+)
+
+var (
+	flagDERPMap        = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map")
+	flagInterval       = flag.Duration("interval", time.Minute, "interval to probe at in time.ParseDuration() format")
+	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
+	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
+	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
+	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+)
+
+const (
+	minInterval       = time.Second
+	maxBufferDuration = time.Hour
+)
+
+func getDERPMap(ctx context.Context, url string) (*tailcfg.DERPMap, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("non-200 derp map resp: %d", resp.StatusCode)
+	}
+	dm := tailcfg.DERPMap{}
+	err = json.NewDecoder(resp.Body).Decode(&dm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode derp map resp: %v", err)
+	}
+	return &dm, nil
+}
+
+type timestampSource int
+
+const (
+	timestampSourceUserspace timestampSource = iota
+	timestampSourceKernel
+)
+
+func (t timestampSource) String() string {
+	switch t {
+	case timestampSourceUserspace:
+		return "userspace"
+	case timestampSourceKernel:
+		return "kernel"
+	default:
+		return "unknown"
+	}
+}
+
+// resultKey contains the stable dimensions and their values for a given
+// timeseries, i.e. not time and not rtt/timeout.
+type resultKey struct {
+	meta            nodeMeta
+	timestampSource timestampSource
+	connStability   connStability
+	dstPort         int
+}
+
+type result struct {
+	key resultKey
+	at  time.Time
+	rtt *time.Duration // nil signifies failure, e.g. timeout
+}
+
+func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+	uconn, ok := conn.(*net.UDPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	err = uconn.SetReadDeadline(time.Now().Add(time.Second * 2))
+	if err != nil {
+		return 0, fmt.Errorf("error setting read deadline: %w", err)
+	}
+	txID := stun.NewTxID()
+	req := stun.Request(txID)
+	txAt := time.Now()
+	_, err = uconn.WriteToUDP(req, dst)
+	if err != nil {
+		return 0, fmt.Errorf("error writing to udp socket: %w", err)
+	}
+	b := make([]byte, 1460)
+	for {
+		n, err := uconn.Read(b)
+		rxAt := time.Now()
+		if err != nil {
+			return 0, fmt.Errorf("error reading from udp socket: %w", err)
+		}
+		gotTxID, _, err := stun.ParseResponse(b[:n])
+		if err != nil || gotTxID != txID {
+			continue
+		}
+		return rxAt.Sub(txAt), nil
+	}
+
+}
+
+func isTemporaryOrTimeoutErr(err error) bool {
+	if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	if err, ok := err.(interface{ Temporary() bool }); ok {
+		return err.Temporary()
+	}
+	return false
+}
+
+type nodeMeta struct {
+	regionID   int
+	regionCode string
+	hostname   string
+	addr       netip.Addr
+}
+
+type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
+
+// probe measures STUN round trip time for the node described by meta over
+// conn against dstPort. It may return a nil duration and nil error if the
+// STUN request timed out. A non-nil error indicates an unrecoverable or
+// non-temporary error.
+func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
+	ua := &net.UDPAddr{
+		IP:   net.IP(meta.addr.AsSlice()),
+		Port: dstPort,
+	}
+
+	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
+	rtt, err := fn(conn, ua)
+	if err != nil {
+		if isTemporaryOrTimeoutErr(err) {
+			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
+			return nil, nil
+		}
+		return nil, err
+	}
+	return &rtt, nil
+}
+
+// nodeMetaFromDERPMap parses the provided DERP map in order to update nodeMeta
+// in the provided nodeMetaByAddr. It returns a slice of nodeMeta containing
+// the nodes that are no longer seen in the DERP map, but were previously held
+// in nodeMetaByAddr.
+func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]nodeMeta, ipv6 bool) (stale []nodeMeta, err error) {
+	// Parse the new derp map before making any state changes in nodeMetaByAddr.
+	// If parse fails we just stick with the old state.
+	updated := make(map[netip.Addr]nodeMeta)
+	for regionID, region := range dm.Regions {
+		for _, node := range region.Nodes {
+			v4, err := netip.ParseAddr(node.IPv4)
+			if err != nil || !v4.Is4() {
+				return nil, fmt.Errorf("invalid ipv4 addr for node in derp map: %v", node.Name)
+			}
+			metas := make([]nodeMeta, 0, 2)
+			metas = append(metas, nodeMeta{
+				regionID:   regionID,
+				regionCode: region.RegionCode,
+				hostname:   node.HostName,
+				addr:       v4,
+			})
+			if ipv6 {
+				v6, err := netip.ParseAddr(node.IPv6)
+				if err != nil || !v6.Is6() {
+					return nil, fmt.Errorf("invalid ipv6 addr for node in derp map: %v", node.Name)
+				}
+				metas = append(metas, metas[0])
+				metas[1].addr = v6
+			}
+			for _, meta := range metas {
+				updated[meta.addr] = meta
+			}
+		}
+	}
+
+	// Find nodeMeta that have changed
+	for addr, updatedMeta := range updated {
+		previousMeta, ok := nodeMetaByAddr[addr]
+		if ok {
+			if previousMeta == updatedMeta {
+				continue
+			}
+			stale = append(stale, previousMeta)
+			nodeMetaByAddr[addr] = updatedMeta
+		} else {
+			nodeMetaByAddr[addr] = updatedMeta
+		}
+	}
+
+	// Find nodeMeta that no longer exist
+	for addr, potentialStale := range nodeMetaByAddr {
+		_, ok := updated[addr]
+		if !ok {
+			stale = append(stale, potentialStale)
+		}
+	}
+
+	return stale, nil
+}
+
+func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
+	conns := [2]io.ReadWriteCloser{}
+	byDstPort, ok := stableConns[addr]
+	if ok {
+		conns, ok = byDstPort[dstPort]
+		if ok {
+			return conns, nil
+		}
+	}
+	if supportsKernelTS() {
+		kconn, err := getConnKernelTimestamp()
+		if err != nil {
+			return conns, err
+		}
+		conns[timestampSourceKernel] = kconn
+	}
+	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
+	if err != nil {
+		if supportsKernelTS() {
+			conns[timestampSourceKernel].Close()
+		}
+		return conns, err
+	}
+	conns[timestampSourceUserspace] = uconn
+	if byDstPort == nil {
+		byDstPort = make(map[int][2]io.ReadWriteCloser)
+	}
+	byDstPort[dstPort] = conns
+	stableConns[addr] = byDstPort
+	return conns, nil
+}
+
+// probeNodes measures the round-trip time for STUN binding requests against the
+// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
+// UDP sockets that should be recycled across runs. It returns the results or
+// an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+	wg := sync.WaitGroup{}
+	results := make([]result, 0)
+	resultsCh := make(chan result)
+	errCh := make(chan error)
+	doneCh := make(chan struct{})
+	numProbes := 0
+	at := time.Now()
+	addrsToProbe := make(map[netip.Addr]bool)
+
+	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+		defer wg.Done()
+		r := result{
+			key: resultKey{
+				meta:            meta,
+				timestampSource: source,
+				dstPort:         dstPort,
+			},
+			at: at,
+		}
+		if conn == nil {
+			var err error
+			if source == timestampSourceKernel {
+				conn, err = getConnKernelTimestamp()
+			} else {
+				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
+			}
+			if err != nil {
+				select {
+				case <-doneCh:
+					return
+				case errCh <- err:
+					return
+				}
+			}
+			defer conn.Close()
+		} else {
+			r.key.connStability = stableConn
+		}
+		fn := measureRTT
+		if source == timestampSourceKernel {
+			fn = measureRTTKernel
+		}
+		rtt, err := probe(meta, conn, fn, dstPort)
+		if err != nil {
+			select {
+			case <-doneCh:
+				return
+			case errCh <- err:
+				return
+			}
+		}
+		r.rtt = rtt
+		select {
+		case <-doneCh:
+		case resultsCh <- r:
+		}
+	}
+
+	for _, meta := range nodeMetaByAddr {
+		addrsToProbe[meta.addr] = true
+		for _, port := range dstPorts {
+			stable, err := getStableConns(stableConns, meta.addr, port)
+			if err != nil {
+				close(doneCh)
+				wg.Wait()
+				return nil, err
+			}
+
+			wg.Add(2)
+			numProbes += 2
+			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
+			go doProbe(nil, meta, timestampSourceUserspace, port)
+			if supportsKernelTS() {
+				wg.Add(2)
+				numProbes += 2
+				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
+				go doProbe(nil, meta, timestampSourceKernel, port)
+			}
+		}
+	}
+
+	// cleanup conns we no longer need
+	for k, byDstPort := range stableConns {
+		if !addrsToProbe[k] {
+			for _, conns := range byDstPort {
+				if conns[timestampSourceKernel] != nil {
+					conns[timestampSourceKernel].Close()
+				}
+				conns[timestampSourceUserspace].Close()
+				delete(stableConns, k)
+			}
+		}
+	}
+
+	for {
+		select {
+		case err := <-errCh:
+			close(doneCh)
+			wg.Wait()
+			return nil, err
+		case result := <-resultsCh:
+			results = append(results, result)
+			if len(results) == numProbes {
+				return results, nil
+			}
+		}
+	}
+}
+
+type connStability bool
+
+const (
+	unstableConn connStability = false
+	stableConn   connStability = true
+)
+
+const (
+	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+)
+
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+	addressFamily := "ipv4"
+	if meta.addr.Is6() {
+		addressFamily = "ipv6"
+	}
+	labels := make([]prompb.Label, 0)
+	labels = append(labels, prompb.Label{
+		Name:  "job",
+		Value: "stunstamp-rw",
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "instance",
+		Value: instance,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "region_id",
+		Value: fmt.Sprintf("%d", meta.regionID),
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "region_code",
+		Value: meta.regionCode,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "address_family",
+		Value: addressFamily,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "hostname",
+		Value: meta.hostname,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "dst_port",
+		Value: strconv.Itoa(dstPort),
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "__name__",
+		Value: metricName,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "timestamp_source",
+		Value: source.String(),
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "stable_conn",
+		Value: fmt.Sprintf("%v", stability),
+	})
+	slices.SortFunc(labels, func(a, b prompb.Label) int {
+		// prometheus remote-write spec requires lexicographically sorted label names
+		return cmp.Compare(a.Name, b.Name)
+	})
+	return labels
+}
+
+const (
+	// https://prometheus.io/docs/concepts/remote_write_spec/#stale-markers
+	staleNaN uint64 = 0x7ff0000000000002
+)
+
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+	staleMarkers := make([]prompb.TimeSeries, 0)
+	now := time.Now()
+	for _, s := range stale {
+		for _, dstPort := range dstPorts {
+			samples := []prompb.Sample{
+				{
+					Timestamp: now.UnixMilli(),
+					Value:     math.Float64frombits(staleNaN),
+				},
+			}
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
+				Samples: samples,
+			})
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
+				Samples: samples,
+			})
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
+				Samples: samples,
+			})
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
+				Samples: samples,
+			})
+			if supportsKernelTS() {
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
+					Samples: samples,
+				})
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
+					Samples: samples,
+				})
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
+					Samples: samples,
+				})
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
+					Samples: samples,
+				})
+			}
+		}
+	}
+	return staleMarkers
+}
+
+// resultsToPromTimeSeries returns a slice of prometheus TimeSeries for the
+// provided results and instance. timeouts is updated based on results, i.e.
+// all result.key's are added to timeouts if they do not exist, and removed
+// from timeouts if they are not present in results.
+func resultsToPromTimeSeries(results []result, instance string, timeouts map[resultKey]uint64) []prompb.TimeSeries {
+	all := make([]prompb.TimeSeries, 0, len(results)*2)
+	seenKeys := make(map[resultKey]bool)
+	for _, r := range results {
+		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
+		seenKeys[r.key] = true
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttSamples := make([]prompb.Sample, 1)
+		rttSamples[0].Timestamp = r.at.UnixMilli()
+		if r.rtt != nil {
+			rttSamples[0].Value = float64(*r.rtt)
+		} else {
+			rttSamples[0].Value = math.NaN()
+			timeoutsCount++
+		}
+		rttTS := prompb.TimeSeries{
+			Labels:  rttLabels,
+			Samples: rttSamples,
+		}
+		all = append(all, rttTS)
+		timeouts[r.key] = timeoutsCount
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsSamples := make([]prompb.Sample, 1)
+		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
+		timeoutsSamples[0].Value = float64(timeoutsCount)
+		timeoutsTS := prompb.TimeSeries{
+			Labels:  timeoutsLabels,
+			Samples: timeoutsSamples,
+		}
+		all = append(all, timeoutsTS)
+	}
+	for k := range timeouts {
+		if !seenKeys[k] {
+			delete(timeouts, k)
+		}
+	}
+	return all
+}
+
+type remoteWriteClient struct {
+	c   *http.Client
+	url string
+}
+
+type recoverableErr struct {
+	error
+}
+
+func newRemoteWriteClient(url string) *remoteWriteClient {
+	return &remoteWriteClient{
+		c: &http.Client{
+			Timeout: time.Second * 30,
+		},
+		url: url,
+	}
+}
+
+func (r *remoteWriteClient) write(ctx context.Context, ts []prompb.TimeSeries) error {
+	wr := &prompb.WriteRequest{
+		Timeseries: ts,
+	}
+	b, err := wr.Marshal()
+	if err != nil {
+		return fmt.Errorf("unable to marshal write request: %w", err)
+	}
+	compressed := snappy.Encode(nil, b)
+	req, err := http.NewRequestWithContext(ctx, "POST", r.url, bytes.NewReader(compressed))
+	if err != nil {
+		return fmt.Errorf("unable to create write request: %w", err)
+	}
+	req.Header.Add("Content-Encoding", "snappy")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("User-Agent", "stunstamp")
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	resp, err := r.c.Do(req)
+	if err != nil {
+		return recoverableErr{fmt.Errorf("error performing write request: %w", err)}
+	}
+	if resp.StatusCode/100 != 2 {
+		err = fmt.Errorf("remote server %s returned HTTP status %d", r.url, resp.StatusCode)
+	}
+	if resp.StatusCode/100 == 5 || resp.StatusCode == http.StatusTooManyRequests {
+		return recoverableErr{err}
+	}
+	return err
+}
+
+func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSeries) {
+	bo := backoff.NewBackoff("remote-write", log.Printf, time.Second*30)
+	// writeErr may contribute to bo's backoff schedule across tsCh read ops,
+	// i.e. if an unrecoverable error occurs for client.write(ctx, A), that
+	// should be accounted against bo prior to attempting to
+	// client.write(ctx, B).
+	var writeErr error
+	for ts := range tsCh {
+		for {
+			bo.BackOff(context.Background(), writeErr)
+			reqCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+			writeErr = client.write(reqCtx, ts)
+			cancel()
+			var re recoverableErr
+			recoverable := errors.As(writeErr, &re)
+			if writeErr != nil {
+				log.Printf("remote write error(recoverable=%v): %v", recoverable, writeErr)
+			}
+			if !recoverable {
+				// a nil err is not recoverable
+				break
+			}
+		}
+	}
+}
+
+func main() {
+	flag.Parse()
+	if len(*flagDstPorts) == 0 {
+		log.Fatal("dst-ports flag is unset")
+	}
+	dstPortsSplit := strings.Split(*flagDstPorts, ",")
+	slices.Sort(dstPortsSplit)
+	dstPortsSplit = slices.Compact(dstPortsSplit)
+	dstPorts := make([]int, 0, len(dstPortsSplit))
+	for _, d := range dstPortsSplit {
+		i, err := strconv.ParseUint(d, 10, 16)
+		if err != nil {
+			log.Fatal("invalid dst-ports")
+		}
+		dstPorts = append(dstPorts, int(i))
+	}
+	if len(*flagDERPMap) < 1 {
+		log.Fatal("derp-map flag is unset")
+	}
+	if *flagInterval < minInterval || *flagInterval > maxBufferDuration {
+		log.Fatalf("interval must be >= %s and <= %s", minInterval, maxBufferDuration)
+	}
+	if len(*flagRemoteWriteURL) < 1 {
+		log.Fatal("rw-url flag is unset")
+	}
+	_, err := url.Parse(*flagRemoteWriteURL)
+	if err != nil {
+		log.Fatalf("invalid rw-url flag value: %v", err)
+	}
+	if len(*flagInstance) < 1 {
+		hostname, err := os.Hostname()
+		if err != nil {
+			log.Fatalf("failed to get hostname: %v", err)
+		}
+		*flagInstance = hostname
+	}
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	dmCh := make(chan *tailcfg.DERPMap)
+
+	go func() {
+		bo := backoff.NewBackoff("derp-map", log.Printf, time.Second*30)
+		for {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+			dm, err := getDERPMap(ctx, *flagDERPMap)
+			cancel()
+			bo.BackOff(context.Background(), err)
+			if err != nil {
+				continue
+			}
+			dmCh <- dm
+			return
+		}
+	}()
+
+	nodeMetaByAddr := make(map[netip.Addr]nodeMeta)
+	select {
+	case <-sigCh:
+		return
+	case dm := <-dmCh:
+		_, err := nodeMetaFromDERPMap(dm, nodeMetaByAddr, *flagIPv6)
+		if err != nil {
+			log.Fatalf("error parsing derp map on startup: %v", err)
+		}
+	}
+
+	tsCh := make(chan []prompb.TimeSeries, maxBufferDuration / *flagInterval)
+	remoteWriteDoneCh := make(chan struct{})
+	rwc := newRemoteWriteClient(*flagRemoteWriteURL)
+	go func() {
+		remoteWriteTimeSeries(rwc, tsCh)
+		close(remoteWriteDoneCh)
+	}()
+
+	shutdown := func() {
+		close(tsCh)
+		select {
+		case <-time.After(time.Second * 10): // give goroutine some time to flush
+		case <-remoteWriteDoneCh:
+		}
+
+		// send stale markers on shutdown
+		staleMeta := make([]nodeMeta, 0, len(nodeMetaByAddr))
+		for _, v := range nodeMetaByAddr {
+			staleMeta = append(staleMeta, v)
+		}
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		if len(staleMarkers) > 0 {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			rwc.write(ctx, staleMarkers)
+			cancel()
+		}
+
+		return
+	}
+
+	log.Println("stunstamp started")
+
+	// Re-using sockets means we get the same 5-tuple across runs. This results
+	// in a higher probability of the packets traversing the same underlay path.
+	// Comparison of stable and unstable 5-tuple results can shed light on
+	// differences between paths where hashing (multipathing/load balancing)
+	// comes into play.
+	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+
+	// timeouts holds counts of timeout events. Values are persisted for the
+	// lifetime of the related node in the DERP map.
+	timeouts := make(map[resultKey]uint64)
+
+	derpMapTicker := time.NewTicker(time.Minute * 5)
+	defer derpMapTicker.Stop()
+	probeTicker := time.NewTicker(*flagInterval)
+	defer probeTicker.Stop()
+
+	for {
+		select {
+		case <-probeTicker.C:
+			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			if err != nil {
+				log.Printf("unrecoverable error while probing: %v", err)
+				shutdown()
+				return
+			}
+			ts := resultsToPromTimeSeries(results, *flagInstance, timeouts)
+			select {
+			case tsCh <- ts:
+			default:
+				select {
+				case <-tsCh:
+					log.Println("prometheus remote-write buffer full, dropped measurements")
+				default:
+					tsCh <- ts
+				}
+			}
+		case dm := <-dmCh:
+			staleMeta, err := nodeMetaFromDERPMap(dm, nodeMetaByAddr, *flagIPv6)
+			if err != nil {
+				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
+				continue
+			}
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			if len(staleMarkers) < 1 {
+				continue
+			}
+			select {
+			case tsCh <- staleMarkers:
+			default:
+				select {
+				case <-tsCh:
+					log.Println("prometheus remote-write buffer full, dropped measurements")
+				default:
+					tsCh <- staleMarkers
+				}
+			}
+		case <-derpMapTicker.C:
+			go func() {
+				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+				defer cancel()
+				updatedDM, err := getDERPMap(ctx, *flagDERPMap)
+				if err != nil {
+					dmCh <- updatedDM
+				}
+			}()
+		case <-sigCh:
+			shutdown()
+			return
+		}
+	}
+}

cmd/stunstamp/stunstamp_default.go

@@ -0,0 +1,25 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package main
+
+import (
+	"errors"
+	"io"
+	"net"
+	"time"
+)
+
+func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unimplemented")
+}
+
+func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+	return 0, errors.New("unimplemented")
+}
+
+func supportsKernelTS() bool {
+	return false
+}

cmd/stunstamp/stunstamp_linux.go

@@ -0,0 +1,143 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"github.com/mdlayher/socket"
+	"golang.org/x/sys/unix"
+	"tailscale.com/net/stun"
+)
+
+const (
+	flags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
+		unix.SOF_TIMESTAMPING_RX_SOFTWARE | // rx timestamp generation in the kernel
+		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
+)
+
+func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
+	if err != nil {
+		return nil, err
+	}
+	sa := unix.SockaddrInet6{}
+	err = sconn.Bind(&sa)
+	if err != nil {
+		return nil, err
+	}
+	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, flags)
+	if err != nil {
+		return nil, err
+	}
+	return sconn, nil
+}
+
+func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
+	msgs, err := unix.ParseSocketControlMessage(oob)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("error parsing oob as cmsgs: %w", err)
+	}
+	for _, msg := range msgs {
+		if msg.Header.Level == unix.SOL_SOCKET && msg.Header.Type == unix.SO_TIMESTAMPING_NEW && len(msg.Data) >= 16 {
+			sec := int64(binary.NativeEndian.Uint64(msg.Data[:8]))
+			ns := int64(binary.NativeEndian.Uint64(msg.Data[8:16]))
+			return time.Unix(sec, ns), nil
+		}
+	}
+	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
+}
+
+func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+	sconn, ok := conn.(*socket.Conn)
+	if !ok {
+		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
+	}
+
+	var to unix.Sockaddr
+	to4 := dst.IP.To4()
+	if to4 != nil {
+		to = &unix.SockaddrInet4{
+			Port: dst.Port,
+		}
+		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+	} else {
+		to = &unix.SockaddrInet6{
+			Port: dst.Port,
+		}
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+	}
+
+	txID := stun.NewTxID()
+	req := stun.Request(txID)
+
+	err = sconn.Sendto(context.Background(), req, 0, to)
+	if err != nil {
+		return 0, fmt.Errorf("sendto error: %v", err) // don't wrap
+	}
+
+	txCtx, txCancel := context.WithTimeout(context.Background(), time.Second*2)
+	defer txCancel()
+
+	buf := make([]byte, 1024)
+	oob := make([]byte, 1024)
+	var txAt time.Time
+
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(txCtx, buf, oob, unix.MSG_ERRQUEUE)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg (MSG_ERRQUEUE) error: %v", err) // don't wrap
+		}
+
+		buf = buf[:n]
+		if n < len(req) || !bytes.Equal(req, buf[len(buf)-len(req):]) {
+			// Spin until we find the message we sent. We get the full packet
+			// looped including eth header so match against the tail.
+			continue
+		}
+		txAt, err = parseTimestampFromCmsgs(oob[:oobn])
+		if err != nil {
+			return 0, fmt.Errorf("failed to get tx timestamp: %v", err) // don't wrap
+		}
+		break
+	}
+
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), time.Second*2)
+	defer rxCancel()
+
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(rxCtx, buf, oob, 0)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg error: %w", err) // wrap for timeout-related error unwrapping
+		}
+
+		gotTxID, _, err := stun.ParseResponse(buf[:n])
+		if err != nil || gotTxID != txID {
+			// Spin until we find the txID we sent. We may end up reading
+			// extremely late arriving responses from previous intervals. As
+			// such, we can't be certain if we're parsing the "current"
+			// response, so spin for parse errors too.
+			continue
+		}
+
+		rxAt, err := parseTimestampFromCmsgs(oob[:oobn])
+		if err != nil {
+			return 0, fmt.Errorf("failed to get rx timestamp: %v", err) // don't wrap
+		}
+
+		return rxAt.Sub(txAt), nil
+	}
+
+}
+
+func supportsKernelTS() bool {
+	return true
+}

cmd/tailscale/cli/bugreport.go

@@ -17,7 +17,7 @@ var bugReportCmd = &ffcli.Command{
 	Name:       "bugreport",
 	Exec:       runBugReport,
 	ShortHelp:  "Print a shareable identifier to help diagnose issues",
-	ShortUsage: "bugreport [note]",
+	ShortUsage: "tailscale bugreport [note]",
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("bugreport")
 		fs.BoolVar(&bugReportArgs.diagnose, "diagnose", false, "run additional in-depth checks")

cmd/tailscale/cli/cert.go

@@ -16,6 +16,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"software.sslmate.com/src/go-pkcs12"
@@ -28,20 +29,22 @@ var certCmd = &ffcli.Command{
 	Name:       "cert",
 	Exec:       runCert,
 	ShortHelp:  "Get TLS certs",
-	ShortUsage: "cert [flags] <domain>",
+	ShortUsage: "tailscale cert [flags] <domain>",
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("cert")
 		fs.StringVar(&certArgs.certFile, "cert-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.crt if --cert-file and --key-file are both unset")
 		fs.StringVar(&certArgs.keyFile, "key-file", "", "output key file or \"-\" for stdout; defaults to DOMAIN.key if --cert-file and --key-file are both unset")
 		fs.BoolVar(&certArgs.serve, "serve-demo", false, "if true, serve on port :443 using the cert as a demo, instead of writing out the files to disk")
+		fs.DurationVar(&certArgs.minValidity, "min-validity", 0, "ensure the certificate is valid for at least this duration; the output certificate is never expired if this flag is unset or 0, but the lifetime may vary; the maximum allowed min-validity depends on the CA")
 		return fs
 	})(),
 }
 
 var certArgs struct {
-	certFile string
-	keyFile  string
-	serve    bool
+	certFile    string
+	keyFile     string
+	serve       bool
+	minValidity time.Duration
 }
 
 func runCert(ctx context.Context, args []string) error {
@@ -102,7 +105,7 @@ func runCert(ctx context.Context, args []string) error {
 		certArgs.certFile = domain + ".crt"
 		certArgs.keyFile = domain + ".key"
 	}
-	certPEM, keyPEM, err := localClient.CertPair(ctx, domain)
+	certPEM, keyPEM, err := localClient.CertPairWithValidity(ctx, domain, certArgs.minValidity)
 	if err != nil {
 		return err
 	}

cmd/tailscale/cli/cli.go

@@ -7,6 +7,7 @@ package cli
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -14,13 +15,15 @@ import (
 	"log"
 	"os"
 	"runtime"
-	"slices"
 	"strings"
 	"sync"
 	"text/tabwriter"
 
+	"github.com/mattn/go-colorable"
+	"github.com/mattn/go-isatty"
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/cmd/tailscale/cli/ffcomplete"
 	"tailscale.com/envknob"
 	"tailscale.com/paths"
 	"tailscale.com/version/distro"
@@ -76,7 +79,9 @@ func CleanUpArgs(args []string) []string {
 	return out
 }
 
-var localClient tailscale.LocalClient
+var localClient = tailscale.LocalClient{
+	Socket: paths.DefaultTailscaledSocket(),
+}
 
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) (err error) {
@@ -93,10 +98,72 @@ func Run(args []string) (err error) {
 		})
 	})
 
-	rootfs := newFlagSet("tailscale")
-	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled socket")
+	rootCmd := newRootCmd()
+	if err := rootCmd.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
+		if noexec := (ffcli.NoExecError{}); errors.As(err, &noexec) {
+			// When the user enters an unknown subcommand, ffcli tries to run
+			// the closest valid parent subcommand with everything else as args,
+			// returning NoExecError if it doesn't have an Exec function.
+			cmd := noexec.Command
+			args := cmd.FlagSet.Args()
+			if len(cmd.Subcommands) > 0 {
+				if len(args) > 0 {
+					return fmt.Errorf("%s: unknown subcommand: %s", fullCmd(rootCmd, cmd), args[0])
+				}
+				subs := make([]string, 0, len(cmd.Subcommands))
+				for _, sub := range cmd.Subcommands {
+					subs = append(subs, sub.Name)
+				}
+				return fmt.Errorf("%s: missing subcommand: %s", fullCmd(rootCmd, cmd), strings.Join(subs, ", "))
+			}
+		}
+		return err
+	}
 
-	rootCmd := &ffcli.Command{
+	if envknob.Bool("TS_DUMP_HELP") {
+		walkCommands(rootCmd, func(w cmdWalk) bool {
+			fmt.Println("===")
+			// UsageFuncs are typically called during Command.Run which ensures
+			// FlagSet is not nil.
+			c := w.Command
+			if c.FlagSet == nil {
+				c.FlagSet = flag.NewFlagSet(c.Name, flag.ContinueOnError)
+			}
+			if c.UsageFunc != nil {
+				fmt.Println(c.UsageFunc(c))
+			} else {
+				fmt.Println(ffcli.DefaultUsageFunc(c))
+			}
+			return true
+		})
+		return
+	}
+
+	err = rootCmd.Run(context.Background())
+	if tailscale.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" {
+		return fmt.Errorf("%v\n\nUse 'sudo tailscale %s' or 'tailscale up --operator=$USER' to not require root.", err, strings.Join(args, " "))
+	}
+	if errors.Is(err, flag.ErrHelp) {
+		return nil
+	}
+	return err
+}
+
+func newRootCmd() *ffcli.Command {
+	rootfs := newFlagSet("tailscale")
+	rootfs.Func("socket", "path to tailscaled socket", func(s string) error {
+		localClient.Socket = s
+		localClient.UseSocketOnly = true
+		return nil
+	})
+	rootfs.Lookup("socket").DefValue = localClient.Socket
+	jsonDocs := rootfs.Bool("json-docs", false, hidden+"print JSON-encoded docs for all subcommands and flags")
+
+	var rootCmd *ffcli.Command
+	rootCmd = &ffcli.Command{
 		Name:       "tailscale",
 		ShortUsage: "tailscale [flags] <subcommand> [command flags]",
 		ShortHelp:  "The easiest, most secure way to use WireGuard.",
@@ -129,59 +196,38 @@ change in the future.
 			certCmd,
 			netlockCmd,
 			licensesCmd,
-			exitNodeCmd,
+			exitNodeCmd(),
 			updateCmd,
 			whoisCmd,
-		},
-		FlagSet:   rootfs,
-		Exec:      func(context.Context, []string) error { return flag.ErrHelp },
-		UsageFunc: usageFunc,
-	}
-	if envknob.UseWIPCode() {
-		rootCmd.Subcommands = append(rootCmd.Subcommands,
+			debugCmd,
+			driveCmd,
 			idTokenCmd,
-		)
+		},
+		FlagSet: rootfs,
+		Exec: func(ctx context.Context, args []string) error {
+			if *jsonDocs {
+				return printJSONDocs(rootCmd)
+			}
+			if len(args) > 0 {
+				return fmt.Errorf("tailscale: unknown subcommand: %s", args[0])
+			}
+			return flag.ErrHelp
+		},
 	}
 
-	// Don't advertise these commands, but they're still explicitly available.
-	switch {
-	case slices.Contains(args, "debug"):
-		rootCmd.Subcommands = append(rootCmd.Subcommands, debugCmd)
-	case slices.Contains(args, "share"):
-		rootCmd.Subcommands = append(rootCmd.Subcommands, shareCmd)
-	}
 	if runtime.GOOS == "linux" && distro.Get() == distro.Synology {
 		rootCmd.Subcommands = append(rootCmd.Subcommands, configureHostCmd)
 	}
 
-	for _, c := range rootCmd.Subcommands {
-		if c.UsageFunc == nil {
-			c.UsageFunc = usageFunc
-		}
-	}
-
-	if err := rootCmd.Parse(args); err != nil {
-		if errors.Is(err, flag.ErrHelp) {
-			return nil
-		}
-		return err
-	}
-
-	localClient.Socket = rootArgs.socket
-	rootfs.Visit(func(f *flag.Flag) {
-		if f.Name == "socket" {
-			localClient.UseSocketOnly = true
+	walkCommands(rootCmd, func(w cmdWalk) bool {
+		if w.UsageFunc == nil {
+			w.UsageFunc = usageFunc
 		}
+		return true
 	})
 
-	err = rootCmd.Run(context.Background())
-	if tailscale.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" {
-		return fmt.Errorf("%v\n\nUse 'sudo tailscale %s' or 'tailscale up --operator=$USER' to not require root.", err, strings.Join(args, " "))
-	}
-	if errors.Is(err, flag.ErrHelp) {
-		return nil
-	}
-	return err
+	ffcomplete.Inject(rootCmd, func(c *ffcli.Command) { c.LongHelp = hidden + c.LongHelp }, usageFunc)
+	return rootCmd
 }
 
 func fatalf(format string, a ...any) {
@@ -196,8 +242,57 @@ func fatalf(format string, a ...any) {
 // Fatalf, if non-nil, is used instead of log.Fatalf.
 var Fatalf func(format string, a ...any)
 
-var rootArgs struct {
-	socket string
+type cmdWalk struct {
+	*ffcli.Command
+	parents []*ffcli.Command
+}
+
+func (w cmdWalk) Path() string {
+	if len(w.parents) == 0 {
+		return w.Name
+	}
+
+	var sb strings.Builder
+	for _, p := range w.parents {
+		sb.WriteString(p.Name)
+		sb.WriteString(" ")
+	}
+	sb.WriteString(w.Name)
+	return sb.String()
+}
+
+// walkCommands calls f for root and all of its nested subcommands until f
+// returns false or all have been visited.
+func walkCommands(root *ffcli.Command, f func(w cmdWalk) (more bool)) {
+	var walk func(cmd *ffcli.Command, parents []*ffcli.Command, f func(cmdWalk) bool) bool
+	walk = func(cmd *ffcli.Command, parents []*ffcli.Command, f func(cmdWalk) bool) bool {
+		if !f(cmdWalk{cmd, parents}) {
+			return false
+		}
+		parents = append(parents, cmd)
+		for _, sub := range cmd.Subcommands {
+			if !walk(sub, parents, f) {
+				return false
+			}
+		}
+		return true
+	}
+	walk(root, nil, f)
+}
+
+// fullCmd returns the full "tailscale ... cmd" invocation for a subcommand.
+func fullCmd(root, cmd *ffcli.Command) (full string) {
+	walkCommands(root, func(w cmdWalk) bool {
+		if w.Command == cmd {
+			full = w.Path()
+			return false
+		}
+		return true
+	})
+	if full == "" {
+		return cmd.Name
+	}
+	return full
 }
 
 // usageFuncNoDefaultValues is like usageFunc but doesn't print default values.
@@ -209,25 +304,36 @@ func usageFunc(c *ffcli.Command) string {
 	return usageFuncOpt(c, true)
 }
 
+// hidden is the prefix that hides subcommands and flags from --help output when
+// found at the start of the subcommand's LongHelp or flag's Usage.
+const hidden = "HIDDEN: "
+
 func usageFuncOpt(c *ffcli.Command, withDefaults bool) string {
 	var b strings.Builder
 
+	if c.ShortHelp != "" {
+		fmt.Fprintf(&b, "%s\n\n", c.ShortHelp)
+	}
+
 	fmt.Fprintf(&b, "USAGE\n")
 	if c.ShortUsage != "" {
-		fmt.Fprintf(&b, "  %s\n", c.ShortUsage)
+		fmt.Fprintf(&b, "  %s\n", strings.ReplaceAll(c.ShortUsage, "\n", "\n  "))
 	} else {
 		fmt.Fprintf(&b, "  %s\n", c.Name)
 	}
 	fmt.Fprintf(&b, "\n")
 
-	if c.LongHelp != "" {
-		fmt.Fprintf(&b, "%s\n\n", c.LongHelp)
+	if help := strings.TrimPrefix(c.LongHelp, hidden); help != "" {
+		fmt.Fprintf(&b, "%s\n\n", help)
 	}
 
 	if len(c.Subcommands) > 0 {
 		fmt.Fprintf(&b, "SUBCOMMANDS\n")
 		tw := tabwriter.NewWriter(&b, 0, 2, 2, ' ', 0)
 		for _, subcommand := range c.Subcommands {
+			if strings.HasPrefix(subcommand.LongHelp, hidden) {
+				continue
+			}
 			fmt.Fprintf(tw, "  %s\t%s\n", subcommand.Name, subcommand.ShortHelp)
 		}
 		tw.Flush()
@@ -240,7 +346,7 @@ func usageFuncOpt(c *ffcli.Command, withDefaults bool) string {
 		c.FlagSet.VisitAll(func(f *flag.Flag) {
 			var s string
 			name, usage := flag.UnquoteUsage(f)
-			if strings.HasPrefix(usage, "HIDDEN: ") {
+			if strings.HasPrefix(usage, hidden) {
 				return
 			}
 			if isBoolFlag(f) {
@@ -287,3 +393,68 @@ func countFlags(fs *flag.FlagSet) (n int) {
 	fs.VisitAll(func(*flag.Flag) { n++ })
 	return n
 }
+
+// colorableOutput returns a colorable writer if stdout is a terminal (not, say,
+// redirected to a file or pipe), the Stdout writer is os.Stdout (we're not
+// embedding the CLI in wasm or a mobile app), and NO_COLOR is not set (see
+// https://no-color.org/). If any of those is not the case, ok is false
+// and w is Stdout.
+func colorableOutput() (w io.Writer, ok bool) {
+	if Stdout != os.Stdout ||
+		os.Getenv("NO_COLOR") != "" ||
+		!isatty.IsTerminal(os.Stdout.Fd()) {
+		return Stdout, false
+	}
+	return colorable.NewColorableStdout(), true
+}
+
+type commandDoc struct {
+	Name        string
+	Desc        string
+	Subcommands []commandDoc `json:",omitempty"`
+	Flags       []flagDoc    `json:",omitempty"`
+}
+
+type flagDoc struct {
+	Name string
+	Desc string
+}
+
+func printJSONDocs(root *ffcli.Command) error {
+	docs := jsonDocsWalk(root)
+	return json.NewEncoder(os.Stdout).Encode(docs)
+}
+
+func jsonDocsWalk(cmd *ffcli.Command) *commandDoc {
+	res := &commandDoc{
+		Name: cmd.Name,
+	}
+	if cmd.LongHelp != "" {
+		res.Desc = cmd.LongHelp
+	} else if cmd.ShortHelp != "" {
+		res.Desc = cmd.ShortHelp
+	} else {
+		res.Desc = cmd.ShortUsage
+	}
+	if strings.HasPrefix(res.Desc, hidden) {
+		return nil
+	}
+	if cmd.FlagSet != nil {
+		cmd.FlagSet.VisitAll(func(f *flag.Flag) {
+			if strings.HasPrefix(f.Usage, hidden) {
+				return
+			}
+			res.Flags = append(res.Flags, flagDoc{
+				Name: f.Name,
+				Desc: f.Usage,
+			})
+		})
+	}
+	for _, sub := range cmd.Subcommands {
+		subj := jsonDocsWalk(sub)
+		if subj != nil {
+			res.Subcommands = append(res.Subcommands, *subj)
+		}
+	}
+	return res
+}

cmd/tailscale/cli/cli_test.go

@@ -16,6 +16,7 @@ import (
 
 	qt "github.com/frankban/quicktest"
 	"github.com/google/go-cmp/cmp"
+	"tailscale.com/envknob"
 	"tailscale.com/health/healthmsg"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -23,11 +24,116 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 	"tailscale.com/version/distro"
 )
 
+func TestPanicIfAnyEnvCheckedInInit(t *testing.T) {
+	envknob.PanicIfAnyEnvCheckedInInit()
+}
+
+func TestShortUsage(t *testing.T) {
+	t.Setenv("TAILSCALE_USE_WIP_CODE", "1")
+	if !envknob.UseWIPCode() {
+		t.Fatal("expected envknob.UseWIPCode() to be true")
+	}
+
+	walkCommands(newRootCmd(), func(w cmdWalk) bool {
+		c, parents := w.Command, w.parents
+
+		// Words that we expect to be in the usage.
+		words := make([]string, len(parents)+1)
+		for i, parent := range parents {
+			words[i] = parent.Name
+		}
+		words[len(parents)] = c.Name
+
+		// Check the ShortHelp starts with a capital letter.
+		if prefix, help := trimPrefixes(c.ShortHelp, "HIDDEN: ", "[ALPHA] ", "[BETA] "); help != "" {
+			if 'a' <= help[0] && help[0] <= 'z' {
+				if len(help) > 20 {
+					help = help[:20] + "…"
+				}
+				caphelp := string(help[0]-'a'+'A') + help[1:]
+				t.Errorf("command: %s: ShortHelp %q should start with a capital letter %q", strings.Join(words, " "), prefix+help, prefix+caphelp)
+			}
+		}
+
+		// Check all words appear in the usage.
+		usage := c.ShortUsage
+		for _, word := range words {
+			var ok bool
+			usage, ok = cutWord(usage, word)
+			if !ok {
+				full := strings.Join(words, " ")
+				t.Errorf("command: %s: usage %q should contain the full path %q", full, c.ShortUsage, full)
+				return true
+			}
+		}
+		return true
+	})
+}
+
+func trimPrefixes(full string, prefixes ...string) (trimmed, remaining string) {
+	s := full
+start:
+	for _, p := range prefixes {
+		var ok bool
+		s, ok = strings.CutPrefix(s, p)
+		if ok {
+			goto start
+		}
+	}
+	return full[:len(full)-len(s)], s
+}
+
+// cutWord("tailscale debug scale 123", "scale") returns (" 123", true).
+func cutWord(s, w string) (after string, ok bool) {
+	var p string
+	for {
+		p, s, ok = strings.Cut(s, w)
+		if !ok {
+			return "", false
+		}
+		if p != "" && isWordChar(p[len(p)-1]) {
+			continue
+		}
+		if s != "" && isWordChar(s[0]) {
+			continue
+		}
+		return s, true
+	}
+}
+
+func isWordChar(r byte) bool {
+	return r == '_' ||
+		('0' <= r && r <= '9') ||
+		('A' <= r && r <= 'Z') ||
+		('a' <= r && r <= 'z')
+}
+
+func TestCutWord(t *testing.T) {
+	tests := []struct {
+		in   string
+		word string
+		out  string
+		ok   bool
+	}{
+		{"tailscale debug", "debug", "", true},
+		{"tailscale debug", "bug", "", false},
+		{"tailscale debug", "tail", "", false},
+		{"tailscale debug scaley scale 123", "scale", " 123", true},
+	}
+	for _, test := range tests {
+		out, ok := cutWord(test.in, test.word)
+		if out != test.out || ok != test.ok {
+			t.Errorf("cutWord(%q, %q) = (%q, %t), wanted (%q, %t)", test.in, test.word, out, ok, test.out, test.ok)
+		}
+	}
+}
+
 // geese is a collection of gooses. It need not be complete.
 // But it should include anything handled specially (e.g. linux, windows)
 // and at least one thing that's not (darwin, freebsd).
@@ -71,9 +177,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "bare_up_means_up",
 			flags: []string{},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         false,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -81,12 +188,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "losing_hostname",
 			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      false,
-				Hostname:         "foo",
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         false,
+				Hostname:            "foo",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
 		},
@@ -94,11 +201,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "hostname_changing_explicitly",
 			flags: []string{"--hostname=bar"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
-				Hostname:         "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -106,11 +213,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "hostname_changing_empty_explicitly",
 			flags: []string{"--hostname="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
-				Hostname:         "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -126,11 +233,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "implicit_operator_change",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				OperatorUser:     "alice",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				OperatorUser:        "alice",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
 			want:    accidentalUpPrefix + " --hostname=foo --operator=alice",
@@ -139,11 +246,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "implicit_operator_matches_shell_user",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				OperatorUser:     "alice",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "alice",
 			want:    "",
@@ -152,15 +259,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "error_advertised_routes_exit_node_removed",
 			flags: []string{"--advertise-routes=10.0.42.0/24"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.42.0/24"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
 		},
@@ -168,15 +275,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertised_routes_exit_node_removed_explicit",
 			flags: []string{"--advertise-routes=10.0.42.0/24", "--advertise-exit-node=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.42.0/24"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -184,15 +291,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
 			flags: []string{"--advertise-routes=11.1.43.0/24,0.0.0.0/0,::/0"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.42.0/24"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -200,10 +307,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node", // Issue 1859
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -211,14 +318,14 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node_over_existing_routes",
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -226,15 +333,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node_over_existing_routes_and_exit_node",
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -242,12 +349,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "exit_node_clearing", // Issue 1777
 			flags: []string{"--exit-node="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeID: "fooID",
+				ExitNodeID:          "fooID",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -255,59 +362,59 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "remove_all_implicit",
 			flags: []string{"--force-reauth"},
 			curPrefs: &ipn.Prefs{
-				WantRunning:      true,
-				ControlURL:       ipn.DefaultControlURL,
-				RouteAll:         true,
-				AllowSingleHosts: false,
-				ExitNodeIP:       netip.MustParseAddr("100.64.5.6"),
-				CorpDNS:          false,
-				ShieldsUp:        true,
-				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
-				Hostname:         "myhostname",
-				ForceDaemon:      true,
+				WantRunning:   true,
+				ControlURL:    ipn.DefaultControlURL,
+				RouteAll:      true,
+				ExitNodeIP:    netip.MustParseAddr("100.64.5.6"),
+				CorpDNS:       false,
+				ShieldsUp:     true,
+				AdvertiseTags: []string{"tag:foo", "tag:bar"},
+				Hostname:      "myhostname",
+				ForceDaemon:   true,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.0.0/16"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
-			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
+			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
 		},
 		{
 			name:  "remove_all_implicit_except_hostname",
 			flags: []string{"--hostname=newhostname"},
 			curPrefs: &ipn.Prefs{
-				WantRunning:      true,
-				ControlURL:       ipn.DefaultControlURL,
-				RouteAll:         true,
-				AllowSingleHosts: false,
-				ExitNodeIP:       netip.MustParseAddr("100.64.5.6"),
-				CorpDNS:          false,
-				ShieldsUp:        true,
-				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
-				Hostname:         "myhostname",
-				ForceDaemon:      true,
+				WantRunning:   true,
+				ControlURL:    ipn.DefaultControlURL,
+				RouteAll:      true,
+				ExitNodeIP:    netip.MustParseAddr("100.64.5.6"),
+				CorpDNS:       false,
+				ShieldsUp:     true,
+				AdvertiseTags: []string{"tag:foo", "tag:bar"},
+				Hostname:      "myhostname",
+				ForceDaemon:   true,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.0.0/16"),
 				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
-			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --netfilter-mode=nodivert --operator=alice --shields-up",
+			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --netfilter-mode=nodivert --operator=alice --shields-up",
 		},
 		{
 			name:  "loggedout_is_implicit",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				LoggedOut:        true,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				LoggedOut:           true,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "", // not an error. LoggedOut is implicit.
 		},
@@ -317,10 +424,9 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "make_windows_exit_node",
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				RouteAll:         true,
+				ControlURL: ipn.DefaultControlURL,
+				CorpDNS:    true,
+				RouteAll:   true,
 
 				// And assume this no-op accidental pre-1.8 value:
 				NoSNAT: true,
@@ -332,8 +438,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_netfilter_change_non_linux",
 			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
+				ControlURL: ipn.DefaultControlURL,
 
 				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
 			},
@@ -344,15 +449,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "operator_losing_routes_step1", // https://twitter.com/EXPbits/status/1390418145047887877
 			flags: []string{"--operator=expbits"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --operator=expbits --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -360,15 +465,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "operator_losing_routes_step2", // https://twitter.com/EXPbits/status/1390418145047887877
 			flags: []string{"--operator=expbits", "--advertise-routes=1.2.0.0/16"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-routes=1.2.0.0/16 --operator=expbits --advertise-exit-node",
 		},
@@ -376,13 +481,13 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "errors_preserve_explicit_flags",
 			flags: []string{"--reset", "--force-reauth=false", "--authkey=secretrand"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      false,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
+				ControlURL:    ipn.DefaultControlURL,
+				WantRunning:   false,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				Hostname: "foo",
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --auth-key=secretrand --force-reauth=false --reset --hostname=foo",
 		},
@@ -390,12 +495,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "error_exit_node_omit_with_ip_pref",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeIP: netip.MustParseAddr("100.64.5.4"),
+				ExitNodeIP:          netip.MustParseAddr("100.64.5.4"),
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.4",
 		},
@@ -404,12 +509,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			flags:         []string{"--hostname=foo"},
 			curExitNodeIP: netip.MustParseAddr("100.64.5.7"),
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeID: "some_stable_id",
+				ExitNodeID:          "some_stable_id",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
@@ -418,13 +523,13 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			flags:         []string{"--hostname=foo"},
 			curExitNodeIP: netip.MustParseAddr("100.2.3.4"),
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
 				ExitNodeAllowLANAccess: true,
 				ExitNodeID:             "some_stable_id",
+				NoStatefulFiltering:    opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node-allow-lan-access --exit-node=100.2.3.4",
 		},
@@ -432,10 +537,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_login_server_synonym",
 			flags: []string{"--login-server=https://controlplane.tailscale.com"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "", // not an error
 		},
@@ -443,10 +548,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_login_server_synonym_on_other_change",
 			flags: []string{"--netfilter-mode=off"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          false,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             false,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
 		},
@@ -456,11 +561,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "synology_permit_omit_accept_routes",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				RouteAll:         true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				RouteAll:            true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos:   "linux",
 			distro: distro.Synology,
@@ -472,11 +577,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "not_synology_dont_permit_omit_accept_routes",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				RouteAll:         true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				RouteAll:            true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos:   "linux",
 			distro: "", // not Synology
@@ -486,11 +591,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "profile_name_ignored_in_up",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				NetfilterMode:    preftype.NetfilterOn,
-				ProfileName:      "foo",
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				ProfileName:         "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos: "linux",
 			want: "",
@@ -550,12 +655,12 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			goos: "linux",
 			args: upArgsFromOSArgs("linux"),
 			want: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      true,
-				NoSNAT:           false,
-				NetfilterMode:    preftype.NetfilterOn,
-				CorpDNS:          true,
-				AllowSingleHosts: true,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         true,
+				NoSNAT:              false,
+				NoStatefulFiltering: "true",
+				NetfilterMode:       preftype.NetfilterOn,
+				CorpDNS:             true,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -566,12 +671,13 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			goos: "windows",
 			args: upArgsFromOSArgs("windows"),
 			want: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      true,
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				RouteAll:         true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         true,
+				CorpDNS:             true,
+				RouteAll:            true,
+				NoSNAT:              false,
+				NoStatefulFiltering: "true",
+				NetfilterMode:       preftype.NetfilterOn,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -581,15 +687,15 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			name: "advertise_default_route",
 			args: upArgsFromOSArgs("linux", "--advertise-exit-node"),
 			want: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      true,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
+				ControlURL:  ipn.DefaultControlURL,
+				WantRunning: true,
+				CorpDNS:     true,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
-				NetfilterMode: preftype.NetfilterOn,
+				NoStatefulFiltering: "true",
+				NetfilterMode:       preftype.NetfilterOn,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -676,9 +782,10 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			},
 			wantWarn: "netfilter=nodivert; add iptables calls to ts-* chains manually.",
 			want: &ipn.Prefs{
-				WantRunning:   true,
-				NetfilterMode: preftype.NetfilterNoDivert,
-				NoSNAT:        true,
+				WantRunning:         true,
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -692,9 +799,10 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			},
 			wantWarn: "netfilter=off; configure iptables yourself.",
 			want: &ipn.Prefs{
-				WantRunning:   true,
-				NetfilterMode: preftype.NetfilterOff,
-				NoSNAT:        true,
+				WantRunning:         true,
+				NetfilterMode:       preftype.NetfilterOff,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -708,8 +816,9 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				netfilterMode:   "off",
 			},
 			want: &ipn.Prefs{
-				WantRunning: true,
-				NoSNAT:      true,
+				WantRunning:         true,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("fd7a:115c:a1e0:b1a::bb:10.0.0.0/112"),
 				},
@@ -726,8 +835,9 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				netfilterMode:   "off",
 			},
 			want: &ipn.Prefs{
-				WantRunning: true,
-				NoSNAT:      true,
+				WantRunning:         true,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("fd7a:115c:a1e0:b1a::aabb:10.0.0.0/112"),
 				},
@@ -803,12 +913,15 @@ func TestPrefFlagMapping(t *testing.T) {
 	}
 
 	prefType := reflect.TypeFor[ipn.Prefs]()
-	for i := 0; i < prefType.NumField(); i++ {
+	for i := range prefType.NumField() {
 		prefName := prefType.Field(i).Name
 		if prefHasFlag[prefName] {
 			continue
 		}
 		switch prefName {
+		case "AllowSingleHosts":
+			// Fake pref for downgrade compat. See #12058.
+			continue
 		case "WantRunning", "Persist", "LoggedOut":
 			// All explicitly handled (ignored) by checkForAccidentalSettingReverts.
 			continue
@@ -829,10 +942,14 @@ func TestPrefFlagMapping(t *testing.T) {
 			// Handled by TS_DEBUG_FIREWALL_MODE env var, we don't want to have
 			// a CLI flag for this. The Pref is used by c2n.
 			continue
-		case "TailFSShares":
+		case "DriveShares":
 			// Handled by the tailscale share subcommand, we don't want a CLI
 			// flag for this.
 			continue
+		case "InternalExitNodePrior":
+			// Used internally by LocalBackend as part of exit node usage toggling.
+			// No CLI flag for this.
+			continue
 		}
 		t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName)
 	}
@@ -912,7 +1029,6 @@ func TestUpdatePrefs(t *testing.T) {
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AdvertiseRoutesSet:        true,
 				AdvertiseTagsSet:          true,
-				AllowSingleHostsSet:       true,
 				AppConnectorSet:           true,
 				ControlURLSet:             true,
 				CorpDNSSet:                true,
@@ -922,6 +1038,7 @@ func TestUpdatePrefs(t *testing.T) {
 				HostnameSet:               true,
 				NetfilterModeSet:          true,
 				NoSNATSet:                 true,
+				NoStatefulFilteringSet:    true,
 				OperatorUserSet:           true,
 				RouteAllSet:               true,
 				RunSSHSet:                 true,
@@ -944,11 +1061,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "change_login_server",
 			flags: []string{"--login-server=https://localhost:1000"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env:            upCheckEnv{backendState: "Running"},
 			wantSimpleUp:   true,
@@ -959,11 +1076,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "change_tags",
 			flags: []string{"--advertise-tags=tag:foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env: upCheckEnv{backendState: "Running"},
 		},
@@ -972,11 +1089,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "explicit_empty_operator",
 			flags: []string{"--operator="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				NetfilterMode:    preftype.NetfilterOn,
-				OperatorUser:     "somebody",
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				OperatorUser:        "somebody",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env: upCheckEnv{user: "somebody", backendState: "Running"},
 			wantJustEditMP: &ipn.MaskedPrefs{
@@ -993,11 +1110,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "enable_ssh",
 			flags: []string{"--ssh"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1014,12 +1131,12 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "disable_ssh",
 			flags: []string{"--ssh=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				RunSSH:           true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				RunSSH:              true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1039,12 +1156,12 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=false"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				RunSSH:           true,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				RunSSH:              true,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1063,11 +1180,11 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=true"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1086,11 +1203,11 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=true", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1108,12 +1225,12 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=false", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				RunSSH:           true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				RunSSH:              true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1131,10 +1248,10 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--force-reauth"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env:          upCheckEnv{backendState: "Running"},
 			wantErrSubtr: "aborted, no changes made",
@@ -1144,10 +1261,10 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--force-reauth", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: nil,
 			env:            upCheckEnv{backendState: "Running"},
@@ -1156,10 +1273,10 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "advertise_connector",
 			flags: []string{"--advertise-connector"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AppConnectorSet: true,
@@ -1176,13 +1293,13 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "no_advertise_connector",
 			flags: []string{"--advertise-connector=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AppConnector: ipn.AppConnectorPrefs{
 					Advertise: true,
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AppConnectorSet: true,

cmd/tailscale/cli/configure-kube.go

@@ -27,7 +27,7 @@ func init() {
 var configureKubeconfigCmd = &ffcli.Command{
 	Name:       "kubeconfig",
 	ShortHelp:  "[ALPHA] Connect to a Kubernetes cluster using a Tailscale Auth Proxy",
-	ShortUsage: "kubeconfig <hostname-or-fqdn>",
+	ShortUsage: "tailscale configure kubeconfig <hostname-or-fqdn>",
 	LongHelp: strings.TrimSpace(`
 Run this command to configure kubectl to connect to a Kubernetes cluster over Tailscale.
 
@@ -43,7 +43,20 @@ See: https://tailscale.com/s/k8s-auth-proxy
 }
 
 // kubeconfigPath returns the path to the kubeconfig file for the current user.
-func kubeconfigPath() string {
+func kubeconfigPath() (string, error) {
+	if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" {
+		if version.IsSandboxedMacOS() {
+			return "", errors.New("$KUBECONFIG is incompatible with the App Store version")
+		}
+		var out string
+		for _, out = range filepath.SplitList(kubeconfig) {
+			if info, err := os.Stat(out); !os.IsNotExist(err) && !info.IsDir() {
+				break
+			}
+		}
+		return out, nil
+	}
+
 	var dir string
 	if version.IsSandboxedMacOS() {
 		// The HOME environment variable in macOS sandboxed apps is set to
@@ -55,7 +68,7 @@ func kubeconfigPath() string {
 	} else {
 		dir = homedir.HomeDir()
 	}
-	return filepath.Join(dir, ".kube", "config")
+	return filepath.Join(dir, ".kube", "config"), nil
 }
 
 func runConfigureKubeconfig(ctx context.Context, args []string) error {
@@ -76,7 +89,11 @@ func runConfigureKubeconfig(ctx context.Context, args []string) error {
 		return fmt.Errorf("no peer found with hostname %q", hostOrFQDN)
 	}
 	targetFQDN = strings.TrimSuffix(targetFQDN, ".")
-	if err := setKubeconfigForPeer(targetFQDN, kubeconfigPath()); err != nil {
+	var kubeconfig string
+	if kubeconfig, err = kubeconfigPath(); err != nil {
+		return err
+	}
+	if err = setKubeconfigForPeer(targetFQDN, kubeconfig); err != nil {
 		return err
 	}
 	printf("kubeconfig configured for %q\n", hostOrFQDN)

cmd/tailscale/cli/configure-synology-cert.go

@@ -0,0 +1,220 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/hostinfo"
+	"tailscale.com/ipn"
+	"tailscale.com/version/distro"
+)
+
+var synologyConfigureCertCmd = &ffcli.Command{
+	Name:       "synology-cert",
+	Exec:       runConfigureSynologyCert,
+	ShortHelp:  "Configure Synology with a TLS certificate for your tailnet",
+	ShortUsage: "synology-cert [--domain <domain>]",
+	LongHelp: strings.TrimSpace(`
+This command is intended to run periodically as root on a Synology device to
+create or refresh the TLS certificate for the tailnet domain.
+
+See: https://tailscale.com/kb/1153/enabling-https
+`),
+	FlagSet: (func() *flag.FlagSet {
+		fs := newFlagSet("synology-cert")
+		fs.StringVar(&synologyConfigureCertArgs.domain, "domain", "", "Tailnet domain to create or refresh certificates for. Ignored if only one domain exists.")
+		return fs
+	})(),
+}
+
+var synologyConfigureCertArgs struct {
+	domain string
+}
+
+func runConfigureSynologyCert(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return errors.New("unknown arguments")
+	}
+	if runtime.GOOS != "linux" || distro.Get() != distro.Synology {
+		return errors.New("only implemented on Synology")
+	}
+	if uid := os.Getuid(); uid != 0 {
+		return fmt.Errorf("must be run as root, not %q (%v)", os.Getenv("USER"), uid)
+	}
+	hi := hostinfo.New()
+	isDSM6 := strings.HasPrefix(hi.DistroVersion, "6.")
+	isDSM7 := strings.HasPrefix(hi.DistroVersion, "7.")
+	if !isDSM6 && !isDSM7 {
+		return fmt.Errorf("unsupported DSM version %q", hi.DistroVersion)
+	}
+
+	domain := synologyConfigureCertArgs.domain
+	if st, err := localClient.Status(ctx); err == nil {
+		if st.BackendState != ipn.Running.String() {
+			return fmt.Errorf("Tailscale is not running.")
+		} else if len(st.CertDomains) == 0 {
+			return fmt.Errorf("TLS certificate support is not enabled/configured for your tailnet.")
+		} else if len(st.CertDomains) == 1 {
+			if domain != "" && domain != st.CertDomains[0] {
+				log.Printf("Ignoring supplied domain %q, TLS certificate will be created for %q.\n", domain, st.CertDomains[0])
+			}
+			domain = st.CertDomains[0]
+		} else {
+			var found bool
+			for _, d := range st.CertDomains {
+				if d == domain {
+					found = true
+					break
+				}
+			}
+			if !found {
+				return fmt.Errorf("Domain %q was not one of the valid domain options: %q.", domain, st.CertDomains)
+			}
+		}
+	}
+
+	// Check for an existing certificate, and replace it if it already exists
+	var id string
+	certs, err := listCerts(ctx, synowebapiCommand{})
+	if err != nil {
+		return err
+	}
+	for _, c := range certs {
+		if c.Subject.CommonName == domain {
+			id = c.ID
+			break
+		}
+	}
+
+	certPEM, keyPEM, err := localClient.CertPair(ctx, domain)
+	if err != nil {
+		return err
+	}
+
+	// Certs have to be written to file for the upload command to work.
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		return fmt.Errorf("can't create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+	keyFile := path.Join(tmpDir, "key.pem")
+	os.WriteFile(keyFile, keyPEM, 0600)
+	certFile := path.Join(tmpDir, "cert.pem")
+	os.WriteFile(certFile, certPEM, 0600)
+
+	if err := uploadCert(ctx, synowebapiCommand{}, certFile, keyFile, id); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type subject struct {
+	CommonName string `json:"common_name"`
+}
+
+type certificateInfo struct {
+	ID      string  `json:"id"`
+	Desc    string  `json:"desc"`
+	Subject subject `json:"subject"`
+}
+
+// listCerts fetches a list of the certificates that DSM knows about
+func listCerts(ctx context.Context, c synoAPICaller) ([]certificateInfo, error) {
+	rawData, err := c.Call(ctx, "SYNO.Core.Certificate.CRT", "list", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var payload struct {
+		Certificates []certificateInfo `json:"certificates"`
+	}
+	if err := json.Unmarshal(rawData, &payload); err != nil {
+		return nil, fmt.Errorf("decoding certificate list response payload: %w", err)
+	}
+
+	return payload.Certificates, nil
+}
+
+// uploadCert creates or replaces a certificate. If id is given, it will attempt to replace the certificate with that ID.
+func uploadCert(ctx context.Context, c synoAPICaller, certFile, keyFile string, id string) error {
+	params := map[string]string{
+		"key_tmp":  keyFile,
+		"cert_tmp": certFile,
+		"desc":     "Tailnet Certificate",
+	}
+	if id != "" {
+		params["id"] = id
+	}
+
+	rawData, err := c.Call(ctx, "SYNO.Core.Certificate", "import", params)
+	if err != nil {
+		return err
+	}
+
+	var payload struct {
+		NewID string `json:"id"`
+	}
+	if err := json.Unmarshal(rawData, &payload); err != nil {
+		return fmt.Errorf("decoding certificate upload response payload: %w", err)
+	}
+	log.Printf("Tailnet Certificate uploaded with ID %q.", payload.NewID)
+
+	return nil
+
+}
+
+type synoAPICaller interface {
+	Call(context.Context, string, string, map[string]string) (json.RawMessage, error)
+}
+
+type apiResponse struct {
+	Success bool            `json:"success"`
+	Error   *apiError       `json:"error,omitempty"`
+	Data    json.RawMessage `json:"data"`
+}
+
+type apiError struct {
+	Code   int64  `json:"code"`
+	Errors string `json:"errors"`
+}
+
+// synowebapiCommand implements synoAPICaller using the /usr/syno/bin/synowebapi binary. Must be run as root.
+type synowebapiCommand struct{}
+
+func (s synowebapiCommand) Call(ctx context.Context, api, method string, params map[string]string) (json.RawMessage, error) {
+	args := []string{"--exec", fmt.Sprintf("api=%s", api), fmt.Sprintf("method=%s", method)}
+
+	for k, v := range params {
+		args = append(args, fmt.Sprintf("%s=%q", k, v))
+	}
+
+	out, err := exec.CommandContext(ctx, "/usr/syno/bin/synowebapi", args...).Output()
+	if err != nil {
+		return nil, fmt.Errorf("calling %q method of %q API: %v, %s", method, api, err, out)
+	}
+
+	var payload apiResponse
+	if err := json.Unmarshal(out, &payload); err != nil {
+		return nil, fmt.Errorf("decoding response json from %q method of %q API: %w", method, api, err)
+	}
+
+	if payload.Error != nil {
+		return nil, fmt.Errorf("error response from %q method of %q API: %v", method, api, payload.Error)
+	}
+
+	return payload.Data, nil
+}

cmd/tailscale/cli/configure-synology-cert_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+type fakeAPICaller struct {
+	Data  json.RawMessage
+	Error error
+}
+
+func (c fakeAPICaller) Call(_ context.Context, _, _ string, _ map[string]string) (json.RawMessage, error) {
+	return c.Data, c.Error
+}
+
+func Test_listCerts(t *testing.T) {
+	tests := []struct {
+		name    string
+		caller  synoAPICaller
+		want    []certificateInfo
+		wantErr bool
+	}{
+		{
+			name: "normal response",
+			caller: fakeAPICaller{
+				Data: json.RawMessage(`{
+"certificates" : [
+	{
+		"desc" : "Tailnet Certificate",
+		"id" : "cG2XBt",
+		"is_broken" : false,
+		"is_default" : false,
+		"issuer" : {
+			"common_name" : "R3",
+			"country" : "US",
+			"organization" : "Let's Encrypt"
+		},
+		"key_types" : "ECC",
+		"renewable" : false,
+		"services" : [
+			{
+			"display_name" : "DSM Desktop Service",
+			"display_name_i18n" : "common:web_desktop",
+			"isPkg" : false,
+			"multiple_cert" : true,
+			"owner" : "root",
+			"service" : "default",
+			"subscriber" : "system",
+			"user_setable" : true
+			}
+		],
+		"signature_algorithm" : "sha256WithRSAEncryption",
+		"subject" : {
+			"common_name" : "foo.tailscale.ts.net",
+			"sub_alt_name" : [ "foo.tailscale.ts.net" ]
+		},
+		"user_deletable" : true,
+		"valid_from" : "Sep 26 11:39:43 2023 GMT",
+		"valid_till" : "Dec 25 11:39:42 2023 GMT"
+	},
+	{
+		"desc" : "",
+		"id" : "sgmnpb",
+		"is_broken" : false,
+		"is_default" : false,
+		"issuer" : {
+			"city" : "Taipei",
+			"common_name" : "Synology Inc. CA",
+			"country" : "TW",
+			"organization" : "Synology Inc."
+		},
+		"key_types" : "",
+		"renewable" : false,
+		"self_signed_cacrt_info" : {
+			"issuer" : {
+			"city" : "Taipei",
+			"common_name" : "Synology Inc. CA",
+			"country" : "TW",
+			"organization" : "Synology Inc."
+			},
+			"subject" : {
+			"city" : "Taipei",
+			"common_name" : "Synology Inc. CA",
+			"country" : "TW",
+			"organization" : "Synology Inc."
+			}
+		},
+		"services" : [],
+		"signature_algorithm" : "sha256WithRSAEncryption",
+		"subject" : {
+			"city" : "Taipei",
+			"common_name" : "synology.com",
+			"country" : "TW",
+			"organization" : "Synology Inc.",
+			"sub_alt_name" : []
+		},
+		"user_deletable" : true,
+		"valid_from" : "May 27 00:23:19 2019 GMT",
+		"valid_till" : "Feb 11 00:23:19 2039 GMT"
+	}
+]
+}`),
+				Error: nil,
+			},
+			want: []certificateInfo{
+				{Desc: "Tailnet Certificate", ID: "cG2XBt", Subject: subject{CommonName: "foo.tailscale.ts.net"}},
+				{Desc: "", ID: "sgmnpb", Subject: subject{CommonName: "synology.com"}},
+			},
+		},
+		{
+			name:    "call error",
+			caller:  fakeAPICaller{nil, fmt.Errorf("caller failed")},
+			wantErr: true,
+		},
+		{
+			name:    "payload decode error",
+			caller:  fakeAPICaller{json.RawMessage("This isn't JSON!"), nil},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := listCerts(context.Background(), tt.caller)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("listCerts() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("listCerts() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}