cmd/quictest: WIP

Change-Id: Iac2efd6e4e974fabe54a88374d64d2715aa40769
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/checklocks.yml

@@ -18,11 +18,17 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks
 
       - name: Run checklocks vet
-        # TODO: remove || true once we have applied checklocks annotations everywhere.
-        run: ./tool/go vet -vettool=/tmp/checklocks ./... || true
+        # TODO(#12625): add more packages as we add annotations
+        run: |-
+          ./tool/go vet -vettool=/tmp/checklocks \
+            ./envknob           \
+            ./ipn/store/mem     \
+            ./net/stun/stuntest \
+            ./net/wsconn        \
+            ./proxymap

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@294a9d92911152fe08befb9ec03e240add280cb3 # v3.26.8
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@294a9d92911152fe08befb9ec03e240add280cb3 # v3.26.8
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@294a9d92911152fe08befb9ec03e240add280cb3 # v3.26.8

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: "actions/checkout@v4"
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,18 +23,18 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: go.mod
           cache: false
 
       - name: golangci-lint
-        # Note: this is the 'v3' tag as of 2023-08-14
-        uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299
+        # Note: this is the 'v6.1.0' tag as of 2024-08-21
+        uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86
         with:
-          version: v1.56
+          version: v1.60
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -24,7 +24,7 @@ jobs:
 
       - name: Post to slack
         if: failure() && github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
         with:

.github/workflows/installer.yml

@@ -67,6 +67,11 @@ jobs:
       image: ${{ matrix.image }}
       options: --user root
     steps:
+    - name: install dependencies (pacman)
+      # Refresh the package databases to ensure that the tailscale package is
+      # defined.
+      run: pacman -Sy
+      if: contains(matrix.image, 'archlinux')
     - name: install dependencies (yum)
       # tar and gzip are needed by the actions/checkout below.
       run: yum install -y --allowerasing tar gzip ${{ matrix.deps }}
@@ -93,7 +98,7 @@ jobs:
       # We cannot use v4, as it requires a newer glibc version than some of the
       # tested images provide. See
       # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@v3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -78,9 +78,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -150,16 +150,16 @@ jobs:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -190,7 +190,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +202,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +214,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +258,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -295,7 +295,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -317,9 +317,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -350,7 +350,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -365,9 +365,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -399,7 +399,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -456,18 +456,22 @@ jobs:
         fuzz-seconds: 300
         dry-run: false
         language: go
+    - name: Set artifacts_path in env (workaround for actions/upload-artifact#176)
+      if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
+      run: |
+        echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
-        path: ./out/artifacts
+        path: ${{ env.artifacts_path }}/out/artifacts
 
   depaware:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
@@ -477,7 +481,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -490,7 +494,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -502,7 +506,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -518,7 +522,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck
@@ -559,7 +563,7 @@ jobs:
       # By having the job always run, but skipping its only step as needed, we
       # let the CI output collapse nicely in PRs.
       if: failure() && github.event_name == 'push'
-      uses: ruby/action-slack@v3.2.1
+      uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
       with:
         payload: |
           {
@@ -574,6 +578,7 @@ jobs:
           }
       env:
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   check_mergeability:
     if: always()
@@ -596,6 +601,6 @@ jobs:
     steps:
     - name: Decide if change is okay to merge
       if: github.event_name != 'push'
-      uses: re-actors/alls-green@release/v1
+      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
       with:
         jobs: ${{ toJSON(needs) }}

.github/workflows/update-flake.yml

@@ -21,21 +21,22 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run update-flakes
         run: ./update-flake.sh
 
       - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         id: generate-token
         with:
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_retrieval_mode: "id"
+          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 #v7.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run go get
         run: |
@@ -23,18 +23,19 @@ jobs:
           ./tool/go mod tidy
 
       - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         id: generate-token
         with:
           # TODO(will): this should use the code updater app rather than licensing.
           # It has the same permissions, so not a big deal, but still.
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_retrieval_mode: "id"
+          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 #v7.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

.gitignore

@@ -43,3 +43,9 @@ client/web/build/assets
 
 /gocross
 /dist
+
+# Ignore xcode userstate and workspace data
+*.xcuserstate
+*.xcworkspacedata
+/tstest/tailmac/bin
+/tstest/tailmac/build

Dockerfile

@@ -1,17 +1,13 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
-############################################################################
+# Note that this Dockerfile is currently NOT used to build any of the published
+# Tailscale container images and may have drifted from the image build mechanism
+# we use.
+# Tailscale images are currently built using https://github.com/tailscale/mkctr,
+# and the build script can be found in ./build_docker.sh.
 #
-# WARNING: Tailscale is not yet officially supported in container
-# environments, such as Docker and Kubernetes. Though it should work, we
-# don't regularly test it, and we know there are some feature limitations.
 #
-# See current bugs tagged "containers":
-#    https://github.com/tailscale/tailscale/labels/containers
-#
-############################################################################
-
 # This Dockerfile includes all the tailscale binaries.
 #
 # To build the Dockerfile:
@@ -31,7 +27,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.22-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
@@ -46,7 +42,7 @@ RUN go install \
     gvisor.dev/gvisor/pkg/tcpip/stack \
     golang.org/x/crypto/ssh \
     golang.org/x/crypto/acme \
-    nhooyr.io/websocket \
+    github.com/coder/websocket \
     github.com/mdlayher/netlink
 
 COPY . .

Makefile

@@ -21,6 +21,7 @@ updatedeps: ## Update depaware deps
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \
+		tailscale.com/cmd/k8s-operator \
 		tailscale.com/cmd/stund
 
 depaware: ## Run depaware checks
@@ -30,6 +31,7 @@ depaware: ## Run depaware checks
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \
+		tailscale.com/cmd/k8s-operator \
 		tailscale.com/cmd/stund
 
 buildwindows: ## Build tailscale CLI for windows/amd64
@@ -115,7 +117,8 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
+	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
 
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"

README.md

@@ -37,7 +37,7 @@ not open source.
 
 ## Building
 
-We always require the latest Go release, currently Go 1.22. (While we build
+We always require the latest Go release, currently Go 1.23. (While we build
 releases with our [Go fork](https://github.com/tailscale/go/), its use is not
 required.)
 

VERSION.txt

@@ -1 +1 @@
-1.69.0
+1.75.0

api.md

@@ -1,101 +1,2 @@
-# Tailscale API
-
-The Tailscale API documentation is located in **[tailscale/publicapi](./publicapi/readme.md#tailscale-api)**.
-
-# APIs
-
-**[Overview](./publicapi/readme.md)**
-
-**[Device](./publicapi/device.md#device)**
-
-<a href="device-delete"></a>
-<a href="expire-device-key"></a>
-<a href="device-routes-get">
-<a href="device-routes-post"></a>
-<a href="#device-authorized-post"></a>
-<a href="device-tags-post"></a>
-<a href="device-key-post"></a>
-<a href="tailnet-acl-get"></a>
-
-- Get a device: [`GET /api/v2/device/{deviceid}`](./publicapi/device.md#get-device)
-- Delete a device: [`DELETE /api/v2/device/{deviceID}`](./publicapi/device.md#delete-device)
-- Expire device key: [`POST /api/v2/device/{deviceID}/expire`](./publicapi/device.md#expire-device-key)
-- [**Routes**](./publicapi/device.md#routes)
-  - Get device routes: [`GET /api/v2/device/{deviceID}/routes`](./publicapi/device.md#get-device-routes)
-  - Set device routes: [`POST /api/v2/device/{deviceID}/routes`](./publicapi/device.md#set-device-routes)
-- [**Authorize**](./publicapi/device.md#authorize)
-  - Authorize a device: [`POST /api/v2/device/{deviceID}/authorized`](./publicapi/device.md#authorize-device)
-- [**Tags**](./publicapi/device.md#tags)
-  - Update tags: [`POST /api/v2/device/{deviceID}/tags`](./publicapi/device.md#update-device-tags)
-- [**Keys**](./publicapi/device.md#keys)
-  - Update device key: [`POST /api/v2/device/{deviceID}/key`](./publicapi/device.md#update-device-key)
-- [**IP Addresses**](./publicapi/device.md#ip-addresses)
-  - Set device IPv4 address: [`POST /api/v2/device/{deviceID}/ip`](./publicapi/device.md#set-device-ipv4-address)
-- [**Device posture attributes**](./publicapi/device.md#device-posture-attributes)
-  - Get device posture attributes: [`GET /api/v2/device/{deviceID}/attributes`](./publicapi/device.md#get-device-posture-attributes)
-  - Set custom device posture attributes: [`POST /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#set-device-posture-attributes)
-  - Delete custom device posture attributes: [`DELETE /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#delete-custom-device-posture-attributes)
-- [**Device invites**](./publicapi/device.md#invites-to-a-device)
-  - List device invites: [`GET /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#list-device-invites)
-  - Create device invites: [`POST /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#create-device-invites)
-
-**[Tailnet](./publicapi/tailnet.md#tailnet)**
-
-<a href="tailnet-acl-post"></a>
-<a href="tailnet-acl-preview-post"></a>
-<a href="tailnet-acl-validate-post"></a>
-<a href="tailnet-devices"></a>
-<a href="tailnet-keys-get"></a>
-<a href="tailnet-keys-post"></a>
-<a href="tailnet-keys-key-get"></a>
-<a href="tailnet-keys-key-delete"></a>
-<a href="tailnet-dns"></a>
-<a href="tailnet-dns-nameservers-get"></a>
-<a href="tailnet-dns-nameservers-post"></a>
-<a href="tailnet-dns-preferences-get"></a>
-<a href="tailnet-dns-preferences-post"></a>
-<a href="tailnet-dns-searchpaths-get"></a>
-<a href="tailnet-dns-searchpaths-post"></a>
-
-- [**Policy File**](./publicapi/tailnet.md#policy-file)
-  - Get policy file: [`GET /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#get-policy-file)
-  - Update policy file: [`POST /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#update-policy-file)
-  - Preview rule matches: [`POST /api/v2/tailnet/{tailnet}/acl/preview`](./publicapi/tailnet.md#preview-policy-file-rule-matches)
-  - Validate and test policy file: [`POST /api/v2/tailnet/{tailnet}/acl/validate`](./publicapi/tailnet.md#validate-and-test-policy-file)
-- [**Devices**](./publicapi/tailnet.md#devices)
-  - List tailnet devices: [`GET /api/v2/tailnet/{tailnet}/devices`](./publicapi/tailnet.md#list-tailnet-devices)
-- [**Keys**](./publicapi/tailnet.md#tailnet-keys)
-  - List tailnet keys: [`GET /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#list-tailnet-keys)
-  - Create an auth key: [`POST /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#create-auth-key)
-  - Get a key: [`GET /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#get-key)
-  - Delete a key: [`DELETE /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#delete-key)
-- [**DNS**](./publicapi/tailnet.md#dns)
-  - [**Nameservers**](./publicapi/tailnet.md#nameservers)
-    - Get nameservers: [`GET /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#get-nameservers)
-    - Set nameservers: [`POST /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#set-nameservers)
-  - [**Preferences**](./publicapi/tailnet.md#preferences)
-    - Get DNS preferences: [`GET /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#get-dns-preferences)
-    - Set DNS preferences: [`POST /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#set-dns-preferences)
-  - [**Search Paths**](./publicapi/tailnet.md#search-paths)
-    - Get search paths: [`GET /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#get-search-paths)
-    - Set search paths: [`POST /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#set-search-paths)
-  - [**Split DNS**](./publicapi/tailnet.md#split-dns)
-    - Get split DNS: [`GET /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#get-split-dns)
-    - Update split DNS: [`PATCH /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#update-split-dns)
-    - Set split DNS: [`PUT /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#set-split-dns)
-- [**User invites**](./publicapi/tailnet.md#tailnet-user-invites)
-  - List user invites: [`GET /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#list-user-invites)
-  - Create user invites: [`POST /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#create-user-invites)
-
-**[User invites](./publicapi/userinvites.md#user-invites)**
-
-- Get user invite: [`GET /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#get-user-invite)
-- Delete user invite: [`DELETE /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#delete-user-invite)
-- Resend user invite (by email): [`POST /api/v2/user-invites/{userInviteId}/resend`](#resend-user-invite)
-
-**[Device invites](./publicapi/deviceinvites.md#device-invites)**
-
-- Get device invite: [`GET /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#get-device-invite)
-- Delete device invite: [`DELETE /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#delete-device-invite)
-- Resend device invite (by email): [`POST /api/v2/device-invites/{deviceInviteId}/resend`](./publicapi/deviceinvites.md#resend-device-invite)
-- Accept device invite [`POST /api/v2/device-invites/-/accept`](#accept-device-invite)
+> [!IMPORTANT]
+> The Tailscale API documentation has moved to https://tailscale.com/api

appc/appconnector.go

@@ -11,6 +11,7 @@ package appc
 
 import (
 	"context"
+	"fmt"
 	"net/netip"
 	"slices"
 	"strings"
@@ -21,6 +22,7 @@ import (
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/execqueue"
 	"tailscale.com/util/mak"
@@ -78,6 +80,42 @@ type RouteAdvertiser interface {
 	UnadvertiseRoute(...netip.Prefix) error
 }
 
+var (
+	metricStoreRoutesRateBuckets = []int64{1, 2, 3, 4, 5, 10, 100, 1000}
+	metricStoreRoutesNBuckets    = []int64{1, 2, 3, 4, 5, 10, 100, 1000, 10000}
+	metricStoreRoutesRate        []*clientmetric.Metric
+	metricStoreRoutesN           []*clientmetric.Metric
+)
+
+func initMetricStoreRoutes() {
+	for _, n := range metricStoreRoutesRateBuckets {
+		metricStoreRoutesRate = append(metricStoreRoutesRate, clientmetric.NewCounter(fmt.Sprintf("appc_store_routes_rate_%d", n)))
+	}
+	metricStoreRoutesRate = append(metricStoreRoutesRate, clientmetric.NewCounter("appc_store_routes_rate_over"))
+	for _, n := range metricStoreRoutesNBuckets {
+		metricStoreRoutesN = append(metricStoreRoutesN, clientmetric.NewCounter(fmt.Sprintf("appc_store_routes_n_routes_%d", n)))
+	}
+	metricStoreRoutesN = append(metricStoreRoutesN, clientmetric.NewCounter("appc_store_routes_n_routes_over"))
+}
+
+func recordMetric(val int64, buckets []int64, metrics []*clientmetric.Metric) {
+	if len(buckets) < 1 {
+		return
+	}
+	// finds the first bucket where val <=, or len(buckets) if none match
+	// for bucket values of 1, 10, 100; 0-1 goes to [0], 2-10 goes to [1], 11-100 goes to [2], 101+ goes to [3]
+	bucket, _ := slices.BinarySearch(buckets, val)
+	metrics[bucket].Add(1)
+}
+
+func metricStoreRoutes(rate, nRoutes int64) {
+	if len(metricStoreRoutesRate) == 0 {
+		initMetricStoreRoutes()
+	}
+	recordMetric(rate, metricStoreRoutesRateBuckets, metricStoreRoutesRate)
+	recordMetric(nRoutes, metricStoreRoutesNBuckets, metricStoreRoutesN)
+}
+
 // RouteInfo is a data structure used to persist the in memory state of an AppConnector
 // so that we can know, even after a restart, which routes came from ACLs and which were
 // learned from domains.
@@ -141,6 +179,7 @@ func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInf
 	}
 	ac.writeRateMinute = newRateLogger(time.Now, time.Minute, func(c int64, s time.Time, l int64) {
 		ac.logf("routeInfo write rate: %d in minute starting at %v (%d routes)", c, s, l)
+		metricStoreRoutes(c, l)
 	})
 	ac.writeRateDay = newRateLogger(time.Now, 24*time.Hour, func(c int64, s time.Time, l int64) {
 		ac.logf("routeInfo write rate: %d in 24 hours starting at %v (%d routes)", c, s, l)
@@ -442,8 +481,10 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			}
 		}
 
-		e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
-		e.scheduleAdvertisement(domain, toAdvertise...)
+		if len(toAdvertise) > 0 {
+			e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
+			e.scheduleAdvertisement(domain, toAdvertise...)
+		}
 	}
 }
 

appc/appconnector_test.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
 	"tailscale.com/tstest"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
@@ -569,3 +570,35 @@ func TestRateLogger(t *testing.T) {
 		t.Fatalf("wasCalled: got false, want true")
 	}
 }
+
+func TestRouteStoreMetrics(t *testing.T) {
+	metricStoreRoutes(1, 1)
+	metricStoreRoutes(1, 1)         // the 1 buckets value should be 2
+	metricStoreRoutes(5, 5)         // the 5 buckets value should be 1
+	metricStoreRoutes(6, 6)         // the 10 buckets value should be 1
+	metricStoreRoutes(10001, 10001) // the over buckets value should be 1
+	wanted := map[string]int64{
+		"appc_store_routes_n_routes_1":    2,
+		"appc_store_routes_rate_1":        2,
+		"appc_store_routes_n_routes_5":    1,
+		"appc_store_routes_rate_5":        1,
+		"appc_store_routes_n_routes_10":   1,
+		"appc_store_routes_rate_10":       1,
+		"appc_store_routes_n_routes_over": 1,
+		"appc_store_routes_rate_over":     1,
+	}
+	for _, x := range clientmetric.Metrics() {
+		if x.Value() != wanted[x.Name()] {
+			t.Errorf("%s: want: %d, got: %d", x.Name(), wanted[x.Name()], x.Value())
+		}
+	}
+}
+
+func TestMetricBucketsAreSorted(t *testing.T) {
+	if !slices.IsSorted(metricStoreRoutesRateBuckets) {
+		t.Errorf("metricStoreRoutesRateBuckets must be in order")
+	}
+	if !slices.IsSorted(metricStoreRoutesNBuckets) {
+		t.Errorf("metricStoreRoutesNBuckets must be in order")
+	}
+}

appc/appctest/appctest.go

@@ -1,6 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+// Package appctest contains code to help test App Connectors.
 package appctest
 
 import (

build_docker.sh

@@ -1,21 +1,11 @@
 #!/usr/bin/env sh
-
 #
-# Runs `go build` with flags configured for docker distribution. All
-# it does differently from `go build` is burn git commit and version
-# information into the binaries inside docker, so that we can track down user
-# issues.
-#
-############################################################################
-#
-# WARNING: Tailscale is not yet officially supported in container
-# environments, such as Docker and Kubernetes. Though it should work, we
-# don't regularly test it, and we know there are some feature limitations.
-#
-# See current bugs tagged "containers":
-#    https://github.com/tailscale/tailscale/labels/containers
-#
-############################################################################
+# This script builds Tailscale container images using
+# github.com/tailscale/mkctr.
+# By default the images will be tagged with the current version and git
+# hash of this repository as produced by ./cmd/mkversion.
+# This is the image build mechanim used to build the official Tailscale
+# container images.
 
 set -eu
 
@@ -49,7 +39,7 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
-      --gotags="ts_kube" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \

client/tailscale/acl.go

@@ -19,6 +19,7 @@ import (
 // Only one of Src/Dst or Users/Ports may be specified.
 type ACLRow struct {
 	Action string   `json:"action,omitempty"` // valid values: "accept"
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Users  []string `json:"users,omitempty"`  // old name for src
 	Ports  []string `json:"ports,omitempty"`  // old name for dst
 	Src    []string `json:"src,omitempty"`
@@ -31,12 +32,23 @@ type ACLRow struct {
 type ACLTest struct {
 	Src    string   `json:"src,omitempty"`    // source
 	User   string   `json:"user,omitempty"`   // old name for source
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Accept []string `json:"accept,omitempty"` // expected destination ip:port that user can access
 	Deny   []string `json:"deny,omitempty"`   // expected destination ip:port that user cannot access
 
 	Allow []string `json:"allow,omitempty"` // old name for accept
 }
 
+// NodeAttrGrant defines additional string attributes that apply to specific devices.
+type NodeAttrGrant struct {
+	// Target specifies which nodes the attributes apply to. The nodes can be a
+	// tag (tag:server), user (alice@example.com), group (group:kids), or *.
+	Target []string `json:"target,omitempty"`
+
+	// Attr are the attributes to set on Target(s).
+	Attr []string `json:"attr,omitempty"`
+}
+
 // ACLDetails contains all the details for an ACL.
 type ACLDetails struct {
 	Tests     []ACLTest           `json:"tests,omitempty"`
@@ -44,6 +56,7 @@ type ACLDetails struct {
 	Groups    map[string][]string `json:"groups,omitempty"`
 	TagOwners map[string][]string `json:"tagowners,omitempty"`
 	Hosts     map[string]string   `json:"hosts,omitempty"`
+	NodeAttrs []NodeAttrGrant     `json:"nodeAttrs,omitempty"`
 }
 
 // ACL contains an ACLDetails and metadata.
@@ -150,7 +163,12 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 // ACLTestFailureSummary specifies the JSON format sent to the
 // JavaScript client to be rendered in the HTML.
 type ACLTestFailureSummary struct {
-	User     string   `json:"user,omitempty"`
+	// User is the source ("src") value of the ACL test that failed.
+	// The name "user" is a legacy holdover from the original naming and
+	// is kept for compatibility but it may also contain any value
+	// that's valid in a ACL test "src" field.
+	User string `json:"user,omitempty"`
+
 	Errors   []string `json:"errors,omitempty"`
 	Warnings []string `json:"warnings,omitempty"`
 }
@@ -270,6 +288,9 @@ type UserRuleMatch struct {
 	Users      []string `json:"users"`
 	Ports      []string `json:"ports"`
 	LineNumber int      `json:"lineNumber"`
+	// Via is the list of targets through which Users can access Ports.
+	// See https://tailscale.com/kb/1378/via for more information.
+	Via []string `json:"via,omitempty"`
 
 	// Postures is a list of posture policies that are
 	// associated with this match. The rules can be looked

client/tailscale/apitype/apitype.go

@@ -4,7 +4,10 @@
 // Package apitype contains types for the Tailscale LocalAPI and control plane API.
 package apitype
 
-import "tailscale.com/tailcfg"
+import (
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/dnstype"
+)
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
@@ -57,3 +60,19 @@ type ExitNodeSuggestionResponse struct {
 	Name     string
 	Location tailcfg.LocationView `json:",omitempty"`
 }
+
+// DNSOSConfig mimics dns.OSConfig without forcing us to import the entire dns package
+// into the CLI.
+type DNSOSConfig struct {
+	Nameservers   []string
+	SearchDomains []string
+	MatchDomains  []string
+}
+
+// DNSQueryResponse is the response to a DNS query request sent via LocalAPI.
+type DNSQueryResponse struct {
+	// Bytes is the raw DNS response bytes.
+	Bytes []byte
+	// Resolvers is the list of resolvers that the forwarder deemed able to resolve the query.
+	Resolvers []*dnstype.Resolver
+}

client/tailscale/localclient.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
+	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
 )
@@ -69,6 +70,14 @@ type LocalClient struct {
 	// connecting to the GUI client variants.
 	UseSocketOnly bool
 
+	// OmitAuth, if true, omits sending the local Tailscale daemon any
+	// authentication token that might be required by the platform.
+	//
+	// As of 2024-08-12, only macOS uses an authentication token. OmitAuth is
+	// meant for when Dial is set and the LocalAPI is being proxied to a
+	// different operating system, such as in integration tests.
+	OmitAuth bool
+
 	// tsClient does HTTP requests to the local Tailscale daemon.
 	// It's lazily initialized on first use.
 	tsClient     *http.Client
@@ -103,7 +112,7 @@ func (lc *LocalClient) defaultDialer(ctx context.Context, network, addr string)
 			return d.DialContext(ctx, "tcp", "127.0.0.1:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(lc.socket())
+	return safesocket.ConnectContext(ctx, lc.socket())
 }
 
 // DoLocalRequest makes an HTTP request to the local machine's Tailscale daemon.
@@ -124,8 +133,10 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 			},
 		}
 	})
-	if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
-		req.SetBasicAuth("", token)
+	if !lc.OmitAuth {
+		if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
+			req.SetBasicAuth("", token)
+		}
 	}
 	return lc.tsClient.Do(req)
 }
@@ -343,6 +354,12 @@ func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
+// UserMetrics returns the user metrics in
+// the Prometheus text exposition format.
+func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
+	return lc.get200(ctx, "/localapi/v0/usermetrics")
+}
+
 // IncrementCounter increments the value of a Tailscale daemon's counter
 // metric by the given delta. If the metric has yet to exist, a new counter
 // metric is created and initialized to delta.
@@ -797,6 +814,35 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
+// GetDNSOSConfig returns the system DNS configuration for the current device.
+// That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
+func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
+	if err != nil {
+		return nil, err
+	}
+	var osCfg apitype.DNSOSConfig
+	if err := json.Unmarshal(body, &osCfg); err != nil {
+		return nil, fmt.Errorf("invalid dns.OSConfig: %w", err)
+	}
+	return &osCfg, nil
+}
+
+// QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
+// It returns the raw DNS response bytes and the resolvers that were used to answer the query
+// (often just one, but can be more if we raced multiple resolvers).
+func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
+	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
+	if err != nil {
+		return nil, nil, err
+	}
+	var res apitype.DNSQueryResponse
+	if err := json.Unmarshal(body, &res); err != nil {
+		return nil, nil, fmt.Errorf("invalid query response: %w", err)
+	}
+	return res.Bytes, res.Resolvers, nil
+}
+
 // StartLoginInteractive starts an interactive login.
 func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)
@@ -933,7 +979,20 @@ func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err e
 //
 // API maturity: this is considered a stable API.
 func (lc *LocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
-	res, err := lc.send(ctx, "GET", "/localapi/v0/cert/"+domain+"?type=pair", 200, nil)
+	return lc.CertPairWithValidity(ctx, domain, 0)
+}
+
+// CertPairWithValidity returns a cert and private key for the provided DNS
+// domain.
+//
+// It returns a cached certificate from disk if it's still valid.
+// When minValidity is non-zero, the returned certificate will be valid for at
+// least the given duration, if permitted by the CA. If the certificate is
+// valid, but for less than minValidity, it will be synchronously renewed.
+//
+// API maturity: this is considered a stable API.
+func (lc *LocalClient) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) {
+	res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil)
 	if err != nil {
 		return nil, nil, err
 	}

client/tailscale/required_version.go

@@ -1,10 +1,10 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !go1.21
+//go:build !go1.23
 
 package tailscale
 
 func init() {
-	you_need_Go_1_21_to_compile_Tailscale()
+	you_need_Go_1_23_to_compile_Tailscale()
 }

client/web/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.1",
   "license": "BSD-3-Clause",
   "engines": {
-    "node": "18.16.1",
+    "node": "18.20.4",
     "yarn": "1.22.19"
   },
   "type": "module",

client/web/web.go

@@ -283,6 +283,12 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if r.URL.Path == "/metrics" {
+		r.URL.Path = "/api/local/v0/usermetrics"
+		s.proxyRequestToLocalAPI(w, r)
+		return
+	}
+
 	if strings.HasPrefix(r.URL.Path, "/api/") {
 		switch {
 		case r.URL.Path == "/api/auth" && r.Method == httpm.GET:

client/web/yarn.lock

@@ -5382,9 +5382,9 @@ wrappy@1:
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
 ws@^8.14.2:
-  version "8.14.2"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f"
-  integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
+  integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
 
 xml-name-validator@^5.0.0:
   version "5.0.0"

clientupdate/clientupdate.go

@@ -248,6 +248,11 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 // CanAutoUpdate reports whether auto-updating via the clientupdate package
 // is supported for the current os/distro.
 func CanAutoUpdate() bool {
+	if version.IsMacSysExt() {
+		// Macsys uses Sparkle for auto-updates, which doesn't have an update
+		// function in this package.
+		return true
+	}
 	_, canAutoUpdate := (&Updater{}).getUpdateFunction()
 	return canAutoUpdate
 }

cmd/cloner/cloner.go

@@ -47,7 +47,7 @@ func main() {
 	it := codegen.NewImportTracker(pkg.Types)
 	buf := new(bytes.Buffer)
 	for _, typeName := range typeNames {
-		typ, ok := namedTypes[typeName]
+		typ, ok := namedTypes[typeName].(*types.Named)
 		if !ok {
 			log.Fatalf("could not find type %s", typeName)
 		}
@@ -78,7 +78,11 @@ func main() {
 		w("	return false")
 		w("}")
 	}
-	cloneOutput := pkg.Name + "_clone.go"
+	cloneOutput := pkg.Name + "_clone"
+	if *flagBuildTags == "test" {
+		cloneOutput += "_test"
+	}
+	cloneOutput += ".go"
 	if err := codegen.WritePackageFile("tailscale.com/cmd/cloner", pkg, cloneOutput, it, buf); err != nil {
 		log.Fatal(err)
 	}
@@ -91,16 +95,19 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 	}
 
 	name := typ.Obj().Name()
+	typeParams := typ.Origin().TypeParams()
+	_, typeParamNames := codegen.FormatTypeParams(typeParams, it)
+	nameWithParams := name + typeParamNames
 	fmt.Fprintf(buf, "// Clone makes a deep copy of %s.\n", name)
 	fmt.Fprintf(buf, "// The result aliases no memory with the original.\n")
-	fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", name, name)
+	fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", nameWithParams, nameWithParams)
 	writef := func(format string, args ...any) {
 		fmt.Fprintf(buf, "\t"+format+"\n", args...)
 	}
 	writef("if src == nil {")
 	writef("\treturn nil")
 	writef("}")
-	writef("dst := new(%s)", name)
+	writef("dst := new(%s)", nameWithParams)
 	writef("*dst = *src")
 	for i := range t.NumFields() {
 		fname := t.Field(i).Name()
@@ -108,7 +115,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		if !codegen.ContainsPointers(ft) || codegen.HasNoClone(t.Tag(i)) {
 			continue
 		}
-		if named, _ := ft.(*types.Named); named != nil {
+		if named, _ := codegen.NamedTypeOf(ft); named != nil {
 			if codegen.IsViewType(ft) {
 				writef("dst.%s = src.%s", fname, fname)
 				continue
@@ -126,16 +133,23 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname)
 				writef("for i := range dst.%s {", fname)
 				if ptr, isPtr := ft.Elem().(*types.Pointer); isPtr {
-					if _, isBasic := ptr.Elem().Underlying().(*types.Basic); isBasic {
-						it.Import("tailscale.com/types/ptr")
-						writef("if src.%s[i] == nil { dst.%s[i] = nil } else {", fname, fname)
-						writef("\tdst.%s[i] = ptr.To(*src.%s[i])", fname, fname)
-						writef("}")
+					writef("if src.%s[i] == nil { dst.%s[i] = nil } else {", fname, fname)
+					if codegen.ContainsPointers(ptr.Elem()) {
+						if _, isIface := ptr.Elem().Underlying().(*types.Interface); isIface {
+							it.Import("tailscale.com/types/ptr")
+							writef("\tdst.%s[i] = ptr.To((*src.%s[i]).Clone())", fname, fname)
+						} else {
+							writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+						}
 					} else {
-						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+						it.Import("tailscale.com/types/ptr")
+						writef("\tdst.%s[i] = ptr.To(*src.%s[i])", fname, fname)
 					}
+					writef("}")
 				} else if ft.Elem().String() == "encoding/json.RawMessage" {
 					writef("\tdst.%s[i] = append(src.%s[i][:0:0], src.%s[i]...)", fname, fname, fname)
+				} else if _, isIface := ft.Elem().Underlying().(*types.Interface); isIface {
+					writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
 				} else {
 					writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname)
 				}
@@ -145,14 +159,19 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("dst.%s = append(src.%s[:0:0], src.%s...)", fname, fname, fname)
 			}
 		case *types.Pointer:
-			if named, _ := ft.Elem().(*types.Named); named != nil && codegen.ContainsPointers(ft.Elem()) {
+			base := ft.Elem()
+			hasPtrs := codegen.ContainsPointers(base)
+			if named, _ := codegen.NamedTypeOf(base); named != nil && hasPtrs {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}
 			it.Import("tailscale.com/types/ptr")
 			writef("if dst.%s != nil {", fname)
-			writef("\tdst.%s = ptr.To(*src.%s)", fname, fname)
-			if codegen.ContainsPointers(ft.Elem()) {
+			if _, isIface := base.Underlying().(*types.Interface); isIface && hasPtrs {
+				writef("\tdst.%s = ptr.To((*src.%s).Clone())", fname, fname)
+			} else if !hasPtrs {
+				writef("\tdst.%s = ptr.To(*src.%s)", fname, fname)
+			} else {
 				writef("\t" + `panic("TODO pointers in pointers")`)
 			}
 			writef("}")
@@ -172,18 +191,50 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("if dst.%s != nil {", fname)
 				writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem))
 				writef("\tfor k, v := range src.%s {", fname)
-				switch elem.(type) {
+
+				switch elem := elem.Underlying().(type) {
 				case *types.Pointer:
-					writef("\t\tdst.%s[k] = v.Clone()", fname)
+					writef("\t\tif v == nil { dst.%s[k] = nil } else {", fname)
+					if base := elem.Elem().Underlying(); codegen.ContainsPointers(base) {
+						if _, isIface := base.(*types.Interface); isIface {
+							it.Import("tailscale.com/types/ptr")
+							writef("\t\t\tdst.%s[k] = ptr.To((*v).Clone())", fname)
+						} else {
+							writef("\t\t\tdst.%s[k] = v.Clone()", fname)
+						}
+					} else {
+						it.Import("tailscale.com/types/ptr")
+						writef("\t\t\tdst.%s[k] = ptr.To(*v)", fname)
+					}
+					writef("}")
+				case *types.Interface:
+					if cloneResultType := methodResultType(elem, "Clone"); cloneResultType != nil {
+						if _, isPtr := cloneResultType.(*types.Pointer); isPtr {
+							writef("\t\tdst.%s[k] = *(v.Clone())", fname)
+						} else {
+							writef("\t\tdst.%s[k] = v.Clone()", fname)
+						}
+					} else {
+						writef(`panic("%s (%v) does not have a Clone method")`, fname, elem)
+					}
 				default:
 					writef("\t\tdst.%s[k] = *(v.Clone())", fname)
 				}
+
 				writef("\t}")
 				writef("}")
 			} else {
 				it.Import("maps")
 				writef("\tdst.%s = maps.Clone(src.%s)", fname, fname)
 			}
+		case *types.Interface:
+			// If ft is an interface with a "Clone() ft" method, it can be used to clone the field.
+			// This includes scenarios where ft is a constrained type parameter.
+			if cloneResultType := methodResultType(ft, "Clone"); cloneResultType.Underlying() == ft {
+				writef("dst.%s = src.%s.Clone()", fname, fname)
+				continue
+			}
+			writef(`panic("%s (%v) does not have a compatible Clone method")`, fname, ft)
 		default:
 			writef(`panic("TODO: %s (%T)")`, fname, ft)
 		}
@@ -191,7 +242,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 	writef("return dst")
 	fmt.Fprintf(buf, "}\n\n")
 
-	buf.Write(codegen.AssertStructUnchanged(t, name, "Clone", it))
+	buf.Write(codegen.AssertStructUnchanged(t, name, typeParams, "Clone", it))
 }
 
 // hasBasicUnderlying reports true when typ.Underlying() is a slice or a map.
@@ -203,3 +254,15 @@ func hasBasicUnderlying(typ types.Type) bool {
 		return false
 	}
 }
+
+func methodResultType(typ types.Type, method string) types.Type {
+	viewMethod := codegen.LookupMethod(typ, method)
+	if viewMethod == nil {
+		return nil
+	}
+	sig, ok := viewMethod.Type().(*types.Signature)
+	if !ok || sig.Results().Len() != 1 {
+		return nil
+	}
+	return sig.Results().At(0).Type()
+}

cmd/cloner/clonerex/clonerex.go

@@ -3,6 +3,7 @@
 
 //go:generate go run tailscale.com/cmd/cloner  -clonefunc=true -type SliceContainer
 
+// Package clonerex is an example package for the cloner tool.
 package clonerex
 
 type SliceContainer struct {

cmd/containerboot/forwarding.go

@@ -0,0 +1,262 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"tailscale.com/util/linuxfw"
+)
+
+// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
+	var (
+		v4Forwarding, v6Forwarding bool
+	)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	// Currently we only proxy traffic to the IPv4 address of the tailnet
+	// target.
+	if tailnetTargetFQDN != "" {
+		v4Forwarding = true
+	}
+	if routes != nil && *routes != "" {
+		for _, route := range strings.Split(*routes, ",") {
+			cidr, err := netip.ParsePrefix(route)
+			if err != nil {
+				return fmt.Errorf("invalid subnet route: %v", err)
+			}
+			if cidr.Addr().Is4() {
+				v4Forwarding = true
+			} else {
+				v6Forwarding = true
+			}
+		}
+	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
+
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
+	var paths []string
+	if v4Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
+	}
+	if v6Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
+	}
+
+	// In some common configurations (e.g. default docker,
+	// kubernetes), the container environment denies write access to
+	// most sysctls, including IP forwarding controls. Check the
+	// sysctl values before trying to change them, so that we
+	// gracefully do nothing if the container's already been set up
+	// properly by e.g. a k8s initContainer.
+	for _, path := range paths {
+		bs, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %q: %w", path, err)
+		}
+		if v := strings.TrimSpace(string(bs)); v != "1" {
+			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
+				return fmt.Errorf("enabling %q: %w", path, err)
+			}
+		}
+	}
+	return nil
+}
+
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	return nil
+}
+
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(_ context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	proxyHasIPv4Address := false
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.AddDNATRule(local, dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRuleForDNSTarget(_ context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}

cmd/containerboot/healthz.go

@@ -0,0 +1,51 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"log"
+	"net"
+	"net/http"
+	"sync"
+)
+
+// healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type healthz struct {
+	sync.Mutex
+	hasAddrs bool
+}
+
+func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+	if h.hasAddrs {
+		w.Write([]byte("ok"))
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
+	}
+}
+
+// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
+// provided address. A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func runHealthz(addr string, h *healthz) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", h)
+	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
+	hs := &http.Server{Handler: mux}
+
+	go func() {
+		if err := hs.Serve(lis); err != nil {
+			log.Fatalf("failed running health endpoint: %v", err)
+		}
+	}()
+}

cmd/containerboot/kube.go

@@ -8,21 +8,21 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 	"tailscale.com/tailcfg"
 )
 
 // storeDeviceID writes deviceID to 'device_id' data field of the named
 // Kubernetes Secret.
 func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
 			"device_id": []byte(deviceID),
 		},
@@ -42,7 +42,7 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 		return err
 	}
 
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
 			"device_fqdn": []byte(fqdn),
 			"device_ips":  deviceIPs,
@@ -55,14 +55,14 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 // secret. No-op if there is no authkey in the secret.
 func deleteAuthKey(ctx context.Context, secretName string) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
-	m := []kube.JSONPatch{
+	m := []kubeclient.JSONPatch{
 		{
 			Op:   "remove",
 			Path: "/data/authkey",
 		},
 	}
 	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
-		if s, ok := err.(*kube.Status); ok && s.Code == http.StatusUnprocessableEntity {
+		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
 			return nil
@@ -72,66 +72,16 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kube.Client
-
-// setupKube is responsible for doing any necessary configuration and checks to
-// ensure that tailscale state storage and authentication mechanism will work on
-// Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
-	if cfg.KubeSecret == "" {
-		return nil
-	}
-	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
-	if err != nil {
-		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-	}
-	cfg.KubernetesCanPatch = canPatch
-
-	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
-	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
-		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
-			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
-			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
-			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
-	} else if err != nil && !kube.IsNotFoundErr(err) {
-		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
-	}
-
-	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-		if s == nil {
-			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
-			return nil
-		}
-		keyBytes, _ := s.Data["authkey"]
-		key := string(keyBytes)
-
-		if key != "" {
-			// This behavior of pulling authkeys from kube secrets was added
-			// at the same time as the patch permission, so we can enforce
-			// that we must be able to patch out the authkey after
-			// authenticating if you want to use this feature. This avoids
-			// us having to deal with the case where we might leave behind
-			// an unnecessary reusable authkey in a secret, like a rake in
-			// the grass.
-			if !cfg.KubernetesCanPatch {
-				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-			}
-			cfg.AuthKey = key
-		} else {
-			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-		}
-	}
-	return nil
-}
+var kc kubeclient.Client
 
 func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.
-		kube.SetRootPathForTesting(root)
+		kubeclient.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kube.New()
+	kc, err = kubeclient.New()
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/kube_test.go

@@ -11,7 +11,8 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 )
 
 func TestSetupKube(t *testing.T) {
@@ -20,7 +21,7 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kube.Client
+		kc      kubeclient.Client
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
@@ -28,11 +29,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, nil
 				},
 			},
@@ -47,12 +48,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -66,12 +67,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -86,12 +87,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 403}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 403}
 				},
 			},
 			wantCfg: &settings{
@@ -110,7 +111,7 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
@@ -126,12 +127,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 		},
@@ -144,12 +145,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{}, nil
 				},
 			},
 		},
@@ -158,12 +159,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{
@@ -176,12 +177,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{

cmd/containerboot/main.go

@@ -52,6 +52,12 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
+//   - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
+//     served at /healthz at the provided address, which should be in form [<address>]:<port>.
+//     If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
+//     The health endpoint will return 200 OK if this node has at least one tailnet IP address,
+//     otherwise returns 503.
+//     NB: the health criteria might change in the future.
 //   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
 //     directory that containers tailscaled config in file. The config file needs to be
 //     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
@@ -86,9 +92,7 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -97,24 +101,19 @@ import (
 	"net"
 	"net/netip"
 	"os"
-	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
-	"reflect"
 	"slices"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"golang.org/x/sys/unix"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
-	"tailscale.com/ipn/conffile"
 	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -158,6 +157,7 @@ func main() {
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
 		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -349,6 +349,9 @@ authLoop:
 
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
+
+		h             = &healthz{} // http server for the healthz endpoint
+		healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
 	)
 	if cfg.ServeConfigPath != "" {
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
@@ -476,18 +479,20 @@ runLoop:
 					newCurentEgressIPs = deephash.Hash(&egressAddrs)
 					egressIPsHaveChanged = newCurentEgressIPs != currentEgressIPs
 					if egressIPsHaveChanged && len(egressAddrs) != 0 {
+						var rulesInstalled bool
 						for _, egressAddr := range egressAddrs {
 							ea := egressAddr.Addr()
-							// TODO (irbekrm): make it work for IPv6 too.
-							if ea.Is6() {
-								log.Println("Not installing egress forwarding rules for IPv6 as this is currently not supported")
-								continue
-							}
-							log.Printf("Installing forwarding rules for destination %v", ea.String())
-							if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil {
-								log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err)
+							if ea.Is4() || (ea.Is6() && nfr.HasIPV6NAT()) {
+								rulesInstalled = true
+								log.Printf("Installing forwarding rules for destination %v", ea.String())
+								if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil {
+									log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err)
+								}
 							}
 						}
+						if !rulesInstalled {
+							log.Fatalf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT())
+						}
 					}
 					currentEgressIPs = newCurentEgressIPs
 				}
@@ -563,6 +568,13 @@ runLoop:
 						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
+
+				if cfg.HealthCheckAddrPort != "" {
+					h.Lock()
+					h.hasAddrs = len(addrs) != 0
+					h.Unlock()
+					healthzRunner()
+				}
 			}
 			if !startupTasksDone {
 				// For containerboot instances that act as TCP
@@ -630,221 +642,6 @@ runLoop:
 	wg.Wait()
 }
 
-// watchServeConfigChanges watches path for changes, and when it sees one, reads
-// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
-// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
-// is written to when the certDomain changes, causing the serve config to be
-// re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
-	if certDomainAtomic == nil {
-		panic("cd must not be nil")
-	}
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
-			log.Fatalf("failed to add fsnotify watch: %v", err)
-		}
-		eventChan = w.Events
-	}
-
-	var certDomain string
-	var prevServeConfig *ipn.ServeConfig
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-cdChanged:
-			certDomain = *certDomainAtomic.Load()
-		case <-tickChan:
-		case <-eventChan:
-			// We can't do any reasonable filtering on the event because of how
-			// k8s handles these mounts. So just re-read the file and apply it
-			// if it's changed.
-		}
-		if certDomain == "" {
-			continue
-		}
-		sc, err := readServeConfig(path, certDomain)
-		if err != nil {
-			log.Fatalf("failed to read serve config: %v", err)
-		}
-		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
-			continue
-		}
-		log.Printf("Applying serve config")
-		if err := lc.SetServeConfig(ctx, sc); err != nil {
-			log.Fatalf("failed to set serve config: %v", err)
-		}
-		prevServeConfig = sc
-	}
-}
-
-// readServeConfig reads the ipn.ServeConfig from path, replacing
-// ${TS_CERT_DOMAIN} with certDomain.
-func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
-	if path == "" {
-		return nil, nil
-	}
-	j, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
-	var sc ipn.ServeConfig
-	if err := json.Unmarshal(j, &sc); err != nil {
-		return nil, err
-	}
-	return &sc, nil
-}
-
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
-	args := tailscaledArgs(cfg)
-	// tailscaled runs without context, since it needs to persist
-	// beyond the startup timeout in ctx.
-	cmd := exec.Command("tailscaled", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-	log.Printf("Starting tailscaled")
-	if err := cmd.Start(); err != nil {
-		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
-	}
-
-	// Wait for the socket file to appear, otherwise API ops will racily fail.
-	log.Printf("Waiting for tailscaled socket")
-	for {
-		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
-		}
-		_, err := os.Stat(cfg.Socket)
-		if errors.Is(err, fs.ErrNotExist) {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
-		}
-		break
-	}
-
-	tsClient := &tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
-	return tsClient, cmd.Process, nil
-}
-
-// tailscaledArgs uses cfg to construct the argv for tailscaled.
-func tailscaledArgs(cfg *settings) []string {
-	args := []string{"--socket=" + cfg.Socket}
-	switch {
-	case cfg.InKubernetes && cfg.KubeSecret != "":
-		args = append(args, "--state=kube:"+cfg.KubeSecret)
-		if cfg.StateDir == "" {
-			cfg.StateDir = "/tmp"
-		}
-		fallthrough
-	case cfg.StateDir != "":
-		args = append(args, "--statedir="+cfg.StateDir)
-	default:
-		args = append(args, "--state=mem:", "--statedir=/tmp")
-	}
-
-	if cfg.UserspaceMode {
-		args = append(args, "--tun=userspace-networking")
-	} else if err := ensureTunFile(cfg.Root); err != nil {
-		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
-	}
-
-	if cfg.SOCKSProxyAddr != "" {
-		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
-	}
-	if cfg.HTTPProxyAddr != "" {
-		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
-	}
-	if cfg.TailscaledConfigFilePath != "" {
-		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
-	}
-	if cfg.DaemonExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
-	}
-	return args
-}
-
-// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
-// if TS_AUTH_ONCE is set, only the first time containerboot starts.
-func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	if cfg.AuthKey != "" {
-		args = append(args, "--authkey="+cfg.AuthKey)
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
-	}
-	return nil
-}
-
-// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
-// options that are passed in via environment variables. This is run after the
-// node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}
-
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
 func ensureTunFile(root string) error {
@@ -864,344 +661,6 @@ func ensureTunFile(root string) error {
 	return nil
 }
 
-// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
-	var (
-		v4Forwarding, v6Forwarding bool
-	)
-	if clusterProxyTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid cluster destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	if tailnetTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid tailnet destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	// Currently we only proxy traffic to the IPv4 address of the tailnet
-	// target.
-	if tailnetTargetFQDN != "" {
-		v4Forwarding = true
-	}
-	if routes != nil && *routes != "" {
-		for _, route := range strings.Split(*routes, ",") {
-			cidr, err := netip.ParsePrefix(route)
-			if err != nil {
-				return fmt.Errorf("invalid subnet route: %v", err)
-			}
-			if cidr.Addr().Is4() {
-				v4Forwarding = true
-			} else {
-				v6Forwarding = true
-			}
-		}
-	}
-	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
-}
-
-func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
-	var paths []string
-	if v4Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
-	}
-	if v6Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
-	}
-
-	// In some common configurations (e.g. default docker,
-	// kubernetes), the container environment denies write access to
-	// most sysctls, including IP forwarding controls. Check the
-	// sysctl values before trying to change them, so that we
-	// gracefully do nothing if the container's already been set up
-	// properly by e.g. a k8s initContainer.
-	for _, path := range paths {
-		bs, err := os.ReadFile(path)
-		if err != nil {
-			return fmt.Errorf("reading %q: %w", path, err)
-		}
-		if v := strings.TrimSpace(string(bs)); v != "1" {
-			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
-				return fmt.Errorf("enabling %q: %w", path, err)
-			}
-		}
-	}
-	return nil
-}
-
-func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	return nil
-}
-
-// installTSForwardingRuleForDestination accepts a destination address and a
-// list of node's tailnet addresses, sets up rules to forward traffic for
-// destination to the tailnet IP matching the destination IP family.
-// Destination can be Pod IP of this node.
-func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstFilter)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
-	}
-	if err := nfr.AddDNATRule(dst, local); err != nil {
-		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	proxyHasIPv4Address := false
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() {
-			proxyHasIPv4Address = true
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if proxyHasIPv4Address && dst.Is6() {
-		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.AddDNATRule(local, dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRuleForDNSTarget(ctx context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	var (
-		tsv4       netip.Addr
-		tsv6       netip.Addr
-		v4Backends []netip.Addr
-		v6Backends []netip.Addr
-	)
-	for _, pfx := range tsIPs {
-		if pfx.IsSingleIP() && pfx.Addr().Is4() {
-			tsv4 = pfx.Addr()
-			continue
-		}
-		if pfx.IsSingleIP() && pfx.Addr().Is6() {
-			tsv6 = pfx.Addr()
-			continue
-		}
-	}
-	// TODO: log if more than one backend address is found and firewall is
-	// in nftables mode that only the first IP will be used.
-	for _, ip := range backendAddrs {
-		if ip.To4() != nil {
-			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
-		}
-		if ip.To16() != nil {
-			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
-		}
-	}
-
-	// Enable IP forwarding here as opposed to at the start of containerboot
-	// as the IPv4/IPv6 requirements might have changed.
-	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
-	// enabled by an init container, so in practice enabling forwarding here
-	// is only needed if this proxy has been configured by manually setting
-	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
-	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
-		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
-	}
-
-	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
-		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
-			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
-		}
-		// The backend might advertize MSS higher than that of the
-		// tailscale interfaces. Clamp MSS of packets going out via
-		// tailscale0 interface to its MTU to prevent broken connections
-		// in environments where path MTU discovery is not working.
-		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
-		}
-		return nil
-	}
-
-	if len(v4Backends) != 0 {
-		if !tsv4.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
-		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
-			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
-		}
-	}
-	if len(v6Backends) != 0 && !tsv6.IsValid() {
-		if !tsv6.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
-		} else if !nfr.HasIPV6NAT() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
-		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
-			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
-		}
-	}
-	return nil
-}
-
-// settings is all the configuration for containerboot.
-type settings struct {
-	AuthKey  string
-	Hostname string
-	Routes   *string
-	// ProxyTargetIP is the destination IP to which all incoming
-	// Tailscale traffic should be proxied. If empty, no proxying
-	// is done. This is typically a locally reachable IP.
-	ProxyTargetIP string
-	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
-	// incoming Tailscale traffic should be proxied.
-	ProxyTargetDNSName string
-	// TailnetTargetIP is the destination IP to which all incoming
-	// non-Tailscale traffic should be proxied. This is typically a
-	// Tailscale IP.
-	TailnetTargetIP string
-	// TailnetTargetFQDN is an MagicDNS name to which all incoming
-	// non-Tailscale traffic should be proxied. This must be a full Tailnet
-	// node FQDN.
-	TailnetTargetFQDN             string
-	ServeConfigPath               string
-	DaemonExtraArgs               string
-	ExtraArgs                     string
-	InKubernetes                  bool
-	UserspaceMode                 bool
-	StateDir                      string
-	AcceptDNS                     *bool
-	KubeSecret                    string
-	SOCKSProxyAddr                string
-	HTTPProxyAddr                 string
-	Socket                        string
-	AuthOnce                      bool
-	Root                          string
-	KubernetesCanPatch            bool
-	TailscaledConfigFilePath      string
-	EnableForwardingOptimizations bool
-	// If set to true and, if this containerboot instance is a Kubernetes
-	// ingress proxy, set up rules to forward incoming cluster traffic to be
-	// forwarded to the ingress target in cluster.
-	AllowProxyingClusterTrafficViaIngress bool
-	// PodIP is the IP of the Pod if running in Kubernetes. This is used
-	// when setting up rules to proxy cluster traffic to cluster ingress
-	// target.
-	PodIP string
-}
-
-func (s *settings) validate() error {
-	if s.TailscaledConfigFilePath != "" {
-		dir, file := path.Split(s.TailscaledConfigFilePath)
-		if _, err := os.Stat(dir); err != nil {
-			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
-		}
-		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
-		}
-		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
-		}
-	}
-	if s.ProxyTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
-	}
-	if s.TailnetTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
-		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
-	}
-	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
-	}
-	if s.EnableForwardingOptimizations && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
-	}
-	return nil
-}
-
 func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	// TODO (irbekrm): look at using recursive.Resolver instead to resolve
 	// the DNS names as well as retrieve TTLs. It looks though that this
@@ -1224,57 +683,6 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	return append(ip4s, ip6s...), nil
 }
 
-// defaultEnv returns the value of the given envvar name, or defVal if
-// unset.
-func defaultEnv(name, defVal string) string {
-	if v, ok := os.LookupEnv(name); ok {
-		return v
-	}
-	return defVal
-}
-
-// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being set to empty string vs unset.
-func defaultEnvStringPointer(name string) *string {
-	if v, ok := os.LookupEnv(name); ok {
-		return &v
-	}
-	return nil
-}
-
-// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being explicitly set to false vs unset.
-func defaultEnvBoolPointer(name string) *bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return nil
-	}
-	return &ret
-}
-
-func defaultEnvs(names []string, defVal string) string {
-	for _, name := range names {
-		if v, ok := os.LookupEnv(name); ok {
-			return v
-		}
-	}
-	return defVal
-}
-
-// defaultBool returns the boolean value of the given envvar name, or
-// defVal if unset or not a bool.
-func defaultBool(name string, defVal bool) bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return defVal
-	}
-	return ret
-}
-
 // contextWithExitSignalWatch watches for SIGTERM/SIGINT signals. It returns a
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
@@ -1297,43 +705,6 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 	return ctx, f
 }
 
-// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
-// in two steps and login should only happen once.
-// Step 1: run 'tailscaled'
-// Step 2):
-// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
-// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
-func isTwoStepConfigAuthOnce(cfg *settings) bool {
-	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
-// in two steps and we should log in every time it starts.
-// Step 1: run 'tailscaled'
-// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
-func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
-	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isOneStepConfig returns true if the Tailscale node should always be ran and
-// configured in a single step by running 'tailscaled <config opts>'
-func isOneStepConfig(cfg *settings) bool {
-	return cfg.TailscaledConfigFilePath != ""
-}
-
-// isL3Proxy returns true if the Tailscale node needs to be configured to act
-// as an L3 proxy, proxying to an endpoint provided via one of the config env
-// vars.
-func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
-}
-
-// hasKubeStateStore returns true if the state must be stored in a Kubernetes
-// Secret.
-func hasKubeStateStore(cfg *settings) bool {
-	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
-}
-
 // tailscaledConfigFilePath returns the path to the tailscaled config file that
 // should be used for the current capability version. It is determined by the
 // TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a

cmd/containerboot/main_test.go

@@ -52,7 +52,7 @@ func TestContainerBoot(t *testing.T) {
 	}
 	defer kube.Close()
 
-	tailscaledConf := &ipn.ConfigVAlpha{AuthKey: func(s string) *string { return &s }("foo"), Version: "alpha0"}
+	tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"}
 	tailscaledConfBytes, err := json.Marshal(tailscaledConf)
 	if err != nil {
 		t.Fatalf("error unmarshaling tailscaled config: %v", err)
@@ -116,6 +116,9 @@ func TestContainerBoot(t *testing.T) {
 		// WantFiles files that should exist in the container and their
 		// contents.
 		WantFiles map[string]string
+		// WantFatalLog is the fatal log message we expect from containerboot.
+		// If set for a phase, the test will finish on that phase.
+		WantFatalLog string
 	}
 	runningNotify := &ipn.Notify{
 		State: ptr.To(ipn.Running),
@@ -349,12 +352,57 @@ func TestContainerBoot(t *testing.T) {
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
 				},
 				{
 					Notify: runningNotify,
 				},
 			},
 		},
+		{
+			Name: "egress_proxy_fqdn_ipv6_target_on_ipv4_host",
+			Env: map[string]string{
+				"TS_AUTHKEY":               "tskey-key",
+				"TS_TAILNET_TARGET_FQDN":   "ipv6-node.test.ts.net", // resolves to IPv6 address
+				"TS_USERSPACE":             "false",
+				"TS_TEST_FAKE_NETFILTER_6": "false",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
+				},
+				{
+					Notify: &ipn.Notify{
+						State: ptr.To(ipn.Running),
+						NetMap: &netmap.NetworkMap{
+							SelfNode: (&tailcfg.Node{
+								StableID:  tailcfg.StableNodeID("myID"),
+								Name:      "test-node.test.ts.net",
+								Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
+							}).View(),
+							Peers: []tailcfg.NodeView{
+								(&tailcfg.Node{
+									StableID:  tailcfg.StableNodeID("ipv6ID"),
+									Name:      "ipv6-node.test.ts.net",
+									Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")},
+								}).View(),
+							},
+						},
+					},
+					WantFatalLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false",
+				},
+			},
+		},
 		{
 			Name: "authkey_once",
 			Env: map[string]string{
@@ -697,6 +745,25 @@ func TestContainerBoot(t *testing.T) {
 			var wantCmds []string
 			for i, p := range test.Phases {
 				lapi.Notify(p.Notify)
+				if p.WantFatalLog != "" {
+					err := tstest.WaitFor(2*time.Second, func() error {
+						state, err := cmd.Process.Wait()
+						if err != nil {
+							return err
+						}
+						if state.ExitCode() != 1 {
+							return fmt.Errorf("process exited with code %d but wanted %d", state.ExitCode(), 1)
+						}
+						waitLogLine(t, time.Second, cbOut, p.WantFatalLog)
+						return nil
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					// Early test return, we don't expect the successful startup log message.
+					return
+				}
 				wantCmds = append(wantCmds, p.WantCmds...)
 				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
 				err := tstest.WaitFor(2*time.Second, func() error {

cmd/containerboot/serve.go

@@ -0,0 +1,96 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+)
+
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+	if certDomainAtomic == nil {
+		panic("cd must not be nil")
+	}
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(path)); err != nil {
+			log.Fatalf("failed to add fsnotify watch: %v", err)
+		}
+		eventChan = w.Events
+	}
+
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-eventChan:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		if certDomain == "" {
+			continue
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("failed to read serve config: %v", err)
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		log.Printf("Applying serve config")
+		if err := lc.SetServeConfig(ctx, sc); err != nil {
+			log.Fatalf("failed to set serve config: %v", err)
+		}
+		prevServeConfig = sc
+	}
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}

cmd/containerboot/settings.go

@@ -0,0 +1,259 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"path"
+	"strconv"
+
+	"tailscale.com/ipn/conffile"
+	"tailscale.com/kube/kubeclient"
+)
+
+// settings is all the configuration for containerboot.
+type settings struct {
+	AuthKey  string
+	Hostname string
+	Routes   *string
+	// ProxyTargetIP is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. This is typically a
+	// Tailscale IP.
+	TailnetTargetIP string
+	// TailnetTargetFQDN is an MagicDNS name to which all incoming
+	// non-Tailscale traffic should be proxied. This must be a full Tailnet
+	// node FQDN.
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	PodIP               string
+	HealthCheckAddrPort string
+}
+
+func (s *settings) validate() error {
+	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
+		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
+		}
+	}
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
+	if s.TailnetTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
+		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
+	}
+	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
+	if s.HealthCheckAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+		}
+	}
+	return nil
+}
+
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kubeclient.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kubeclient.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
+
+// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
+// in two steps and login should only happen once.
+// Step 1: run 'tailscaled'
+// Step 2):
+// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
+// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
+func isTwoStepConfigAuthOnce(cfg *settings) bool {
+	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
+// in two steps and we should log in every time it starts.
+// Step 1: run 'tailscaled'
+// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
+func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
+	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isOneStepConfig returns true if the Tailscale node should always be ran and
+// configured in a single step by running 'tailscaled <config opts>'
+func isOneStepConfig(cfg *settings) bool {
+	return cfg.TailscaledConfigFilePath != ""
+}
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
+// defaultEnv returns the value of the given envvar name, or defVal if
+// unset.
+func defaultEnv(name, defVal string) string {
+	if v, ok := os.LookupEnv(name); ok {
+		return v
+	}
+	return defVal
+}
+
+// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being set to empty string vs unset.
+func defaultEnvStringPointer(name string) *string {
+	if v, ok := os.LookupEnv(name); ok {
+		return &v
+	}
+	return nil
+}
+
+// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being explicitly set to false vs unset.
+func defaultEnvBoolPointer(name string) *bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return nil
+	}
+	return &ret
+}
+
+func defaultEnvs(names []string, defVal string) string {
+	for _, name := range names {
+		if v, ok := os.LookupEnv(name); ok {
+			return v
+		}
+	}
+	return defVal
+}
+
+// defaultBool returns the boolean value of the given envvar name, or
+// defVal if unset or not a bool.
+func defaultBool(name string, defVal bool) bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return defVal
+	}
+	return ret
+}

cmd/containerboot/tailscaled.go

@@ -0,0 +1,162 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+
+	"tailscale.com/client/tailscale"
+)
+
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+	args := tailscaledArgs(cfg)
+	// tailscaled runs without context, since it needs to persist
+	// beyond the startup timeout in ctx.
+	cmd := exec.Command("tailscaled", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	log.Printf("Starting tailscaled")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
+	}
+
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
+	log.Printf("Waiting for tailscaled socket")
+	for {
+		if ctx.Err() != nil {
+			log.Fatalf("Timed out waiting for tailscaled socket")
+		}
+		_, err := os.Stat(cfg.Socket)
+		if errors.Is(err, fs.ErrNotExist) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		} else if err != nil {
+			log.Fatalf("Waiting for tailscaled socket: %v", err)
+		}
+		break
+	}
+
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process, nil
+}
+
+// tailscaledArgs uses cfg to construct the argv for tailscaled.
+func tailscaledArgs(cfg *settings) []string {
+	args := []string{"--socket=" + cfg.Socket}
+	switch {
+	case cfg.InKubernetes && cfg.KubeSecret != "":
+		args = append(args, "--state=kube:"+cfg.KubeSecret)
+		if cfg.StateDir == "" {
+			cfg.StateDir = "/tmp"
+		}
+		fallthrough
+	case cfg.StateDir != "":
+		args = append(args, "--statedir="+cfg.StateDir)
+	default:
+		args = append(args, "--state=mem:", "--statedir=/tmp")
+	}
+
+	if cfg.UserspaceMode {
+		args = append(args, "--tun=userspace-networking")
+	} else if err := ensureTunFile(cfg.Root); err != nil {
+		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
+	}
+
+	if cfg.SOCKSProxyAddr != "" {
+		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
+	}
+	if cfg.HTTPProxyAddr != "" {
+		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
+	}
+	if cfg.TailscaledConfigFilePath != "" {
+		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
+	}
+	if cfg.DaemonExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
+	}
+	return args
+}
+
+// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
+// if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleUp(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "up"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale up'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale up failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state and only if TS_AUTH_ONCE is set.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	log.Printf("Running 'tailscale set'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale set failed: %v", err)
+	}
+	return nil
+}

cmd/derper/README.md

@@ -2,7 +2,8 @@
 
 This is the code for the [Tailscale DERP server](https://tailscale.com/kb/1232/derp-servers).
 
-In general, you should not need to nor want to run this code. The overwhelming majority of Tailscale users (both individuals and companies) do not.
+In general, you should not need to or want to run this code. The overwhelming
+majority of Tailscale users (both individuals and companies) do not.
 
 In the happy path, Tailscale establishes direct connections between peers and
 data plane traffic flows directly between them, without using DERP for more than
@@ -11,7 +12,7 @@ find yourself wanting DERP for more bandwidth, the real problem is usually the
 network configuration of your Tailscale node(s), making sure that Tailscale can
 get direction connections via some mechanism.
 
-But if you've decided or been advised to run your own `derper`, then read on.
+If you've decided or been advised to run your own `derper`, then read on.
 
 ## Caveats
 
@@ -28,7 +29,10 @@ But if you've decided or been advised to run your own `derper`, then read on.
 
 * You must build and update the `cmd/derper` binary yourself. There are no
   packages. Use `go install tailscale.com/cmd/derper@latest` with the latest
-  version of Go.
+  version of Go. You should update this binary approximately as regularly as
+  you update Tailscale nodes. If using `--verify-clients`, the `derper` binary
+  and `tailscaled` binary on the machine must be built from the same git revision.
+  (It might work otherwise, but they're developed and only tested together.)
 
 * The DERP protocol does a protocol switch inside TLS from HTTP to a custom
   bidirectional binary protocol. It is thus incompatible with many HTTP proxies.
@@ -55,7 +59,7 @@ rely on its DNS which might be broken and dependent on DERP to get back up.
 * Monitor your DERP servers with [`cmd/derpprobe`](../derpprobe/).
 
 * If using `--verify-clients`, a `tailscaled` must be running alongside the
-  `derper`.
+  `derper`, and all clients must be visible to the derper tailscaled in the ACL.
 
 * If using `--verify-clients`, a `tailscaled` must also be running alongside
   your `derpprobe`, and `derpprobe` needs to use `--derp-map=local`.
@@ -72,3 +76,34 @@ rely on its DNS which might be broken and dependent on DERP to get back up.
 * Don't rate-limit UDP STUN packets.
 
 * Don't rate-limit outbound TCP traffic (only inbound).
+
+## Diagnostics
+
+This is not a complete guide on DERP diagnostics.
+
+Running your own DERP services requires exeprtise in multi-layer network and
+application diagnostics. As the DERP runs multiple protocols at multiple layers
+and is not a regular HTTP(s) server you will need expertise in correlative
+analysis to diagnose the most tricky problems. There is no "plain text" or
+"open" mode of operation for DERP.
+
+* The debug handler is accessible at URL path `/debug/`. It is only accessible
+  over localhost or from a Tailscale IP address.
+
+* Go pprof can be accessed via the debug handler at `/debug/pprof/`
+
+* Prometheus compatible metrics can be gathered from the debug handler at
+  `/debug/varz`.
+
+* `cmd/stunc` in the Tailscale repository provides a basic tool for diagnosing
+  issues with STUN.
+
+* `cmd/derpprobe` provides a service for monitoring DERP cluster health.
+
+* `tailscale debug derp` and `tailscale netcheck` provide additional client
+  driven diagnostic information for DERP communications.
+
+* Tailscale logs may provide insight for certain problems, such as if DERPs are
+  unreachable or peers are regularly not reachable in their DERP home regions.
+  There are many possible misconfiguration causes for these problems, but
+  regular log entries are a good first indicator that there is a problem.

cmd/derper/depaware.txt

@@ -7,9 +7,19 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/cmd/derper+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
+        github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
    L    github.com/google/nftables                                   from tailscale.com/util/linuxfw
    L 💣 github.com/google/nftables/alignedbuff                       from github.com/google/nftables/xt
@@ -42,7 +52,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
@@ -76,10 +86,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoiface                from google.golang.org/protobuf/internal/impl+
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
-        nhooyr.io/websocket                                          from tailscale.com/cmd/derper+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -93,13 +99,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
      💣 tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
-        tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
+     💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale
         tailscale.com/net/sockstats                                  from tailscale.com/derp/derphttp
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
@@ -114,14 +121,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/derp
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -140,9 +147,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -153,6 +162,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
@@ -165,15 +177,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from tailscale.com/tka
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -245,6 +259,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/ipn+
@@ -260,7 +275,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -289,3 +304,4 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/derper/derper.go

@@ -2,6 +2,12 @@
 // SPDX-License-Identifier: BSD-3-Clause
 
 // The derper binary is a simple DERP server.
+//
+// For more information, see:
+//
+//   - About: https://tailscale.com/kb/1232/derp-servers
+//   - Protocol & Go docs: https://pkg.go.dev/tailscale.com/derp
+//   - Running a DERP server: https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp
 package main // import "tailscale.com/cmd/derper"
 
 import (
@@ -22,6 +28,9 @@ import (
 	"os/signal"
 	"path/filepath"
 	"regexp"
+	"runtime"
+	runtimemetrics "runtime/metrics"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -206,11 +215,16 @@ func main() {
 		io.WriteString(w, `<html><body>
 <h1>DERP</h1>
 <p>
-  This is a
-  <a href="https://tailscale.com/">Tailscale</a>
-  <a href="https://pkg.go.dev/tailscale.com/derp">DERP</a>
-  server.
+  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
 </p>
+<p>
+  Documentation:
+</p>
+<ul>
+  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
+  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
+  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
+</ul>
 `)
 		if !*runDERP {
 			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
@@ -223,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -236,6 +250,20 @@ func main() {
 		}
 	}))
 	debug.Handle("traffic", "Traffic check", http.HandlerFunc(s.ServeDebugTraffic))
+	debug.Handle("set-mutex-profile-fraction", "SetMutexProfileFraction", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		s := r.FormValue("rate")
+		if s == "" || r.Header.Get("Sec-Debug") != "derp" {
+			http.Error(w, "To set, use: curl -HSec-Debug:derp 'http://derp/debug/set-mutex-profile-fraction?rate=100'", http.StatusBadRequest)
+			return
+		}
+		v, err := strconv.Atoi(s)
+		if err != nil {
+			http.Error(w, "bad rate value", http.StatusBadRequest)
+			return
+		}
+		old := runtime.SetMutexProfileFraction(v)
+		fmt.Fprintf(w, "mutex changed from %v to %v\n", old, v)
+	}))
 
 	// Longer lived DERP connections send an application layer keepalive. Note
 	// if the keepalive is hit, the user timeout will take precedence over the
@@ -309,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -350,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {
@@ -452,3 +455,16 @@ func (l *rateLimitedListener) Accept() (net.Conn, error) {
 	l.numAccepts.Add(1)
 	return cn, nil
 }
+
+func init() {
+	expvar.Publish("go_sync_mutex_wait_seconds", expvar.Func(func() any {
+		const name = "/sync/mutex/wait/total:seconds" // Go 1.20+
+		var s [1]runtimemetrics.Sample
+		s[0].Name = name
+		runtimemetrics.Read(s[:])
+		if v := s[0].Value; v.Kind() == runtimemetrics.KindFloat64 {
+			return v.Float64()
+		}
+		return 0
+	}))
+}

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/derper/mesh.go

@@ -9,14 +9,12 @@ import (
 	"fmt"
 	"log"
 	"net"
-	"net/netip"
 	"strings"
 	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/net/netmon"
-	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
 
@@ -71,8 +69,8 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return d.DialContext(ctx, network, addr)
 	})
 
-	add := func(k key.NodePublic, _ netip.AddrPort) { s.AddPacketForwarder(k, c) }
-	remove := func(k key.NodePublic) { s.RemovePacketForwarder(k, c) }
+	add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) }
+	remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)
 	return nil
 }

cmd/derper/websocket.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"strings"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/derp"
 	"tailscale.com/net/wsconn"
 )

cmd/derpprobe/derpprobe.go

@@ -7,8 +7,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"html"
-	"io"
 	"log"
 	"net/http"
 	"sort"
@@ -70,8 +68,13 @@ func main() {
 	}
 
 	mux := http.NewServeMux()
-	tsweb.Debugger(mux)
-	mux.HandleFunc("/", http.HandlerFunc(serveFunc(p)))
+	d := tsweb.Debugger(mux)
+	d.Handle("probe-run", "Run a probe", tsweb.StdHandler(tsweb.ReturnHandlerFunc(p.RunHandler), tsweb.HandlerOptions{Logf: log.Printf}))
+	mux.Handle("/", tsweb.StdHandler(p.StatusHandler(
+		prober.WithTitle("DERP Prober"),
+		prober.WithPageLink("Prober metrics", "/debug/varz"),
+		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
+	), tsweb.HandlerOptions{Logf: log.Printf}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }
@@ -105,26 +108,3 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 	sort.Strings(o.good)
 	return
 }
-
-func serveFunc(p *prober.Prober) func(w http.ResponseWriter, r *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		st := getOverallStatus(p)
-		summary := "All good"
-		if (float64(len(st.bad)) / float64(len(st.bad)+len(st.good))) > 0.25 {
-			// Returning a 500 allows monitoring this server externally and configuring
-			// an alert on HTTP response code.
-			w.WriteHeader(500)
-			summary = fmt.Sprintf("%d problems", len(st.bad))
-		}
-
-		io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
-		fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
-		for _, s := range st.bad {
-			fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
-		}
-		for _, s := range st.good {
-			fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
-		}
-		io.WriteString(w, "</ul></body></html>\n")
-	}
-}

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/k8s-operator/connector.go

@@ -26,6 +26,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -61,11 +62,11 @@ type ConnectorReconciler struct {
 
 var (
 	// gaugeConnectorResources tracks the overall number of Connectors currently managed by this operator instance.
-	gaugeConnectorResources = clientmetric.NewGauge("k8s_connector_resources")
+	gaugeConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorResourceCount)
 	// gaugeConnectorSubnetRouterResources tracks the number of Connectors managed by this operator instance that are subnet routers.
-	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
+	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
-	gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
+	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {

cmd/k8s-operator/connector_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/util/mak"
 )
@@ -74,6 +75,7 @@ func TestConnector(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -169,6 +171,7 @@ func TestConnector(t *testing.T) {
 		parentType:   "connector",
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -254,6 +257,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -77,6 +77,13 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            {{- if .Values.proxyConfig.defaultProxyClass }}
+            - name: PROXY_DEFAULT_CLASS
+              value: {{ .Values.proxyConfig.defaultProxyClass }}
+            {{- end }}
+            {{- with .Values.operatorConfig.extraEnv }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
           volumeMounts:
           - name: oauth
             mountPath: /oauth

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -14,10 +14,10 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["events", "services", "services/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingresses", "ingresses/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
@@ -27,6 +27,9 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["dnsconfigs", "dnsconfigs/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["recorders", "recorders/status"]
+  verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -49,13 +52,16 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
   verbs: ["get", "list", "watch"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings"]
+  verbs: ["get", "create", "patch", "update", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -15,7 +15,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -48,14 +48,21 @@ operatorConfig:
 
   securityContext: {}
 
+  extraEnv: []
+  # - name: EXTRA_VAR1
+  #   value: "value1"
+  # - name: EXTRA_VAR2
+  #   value: "value2"
+
+
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.
-# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
-# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
+# https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
+# https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
 # Note that this section contains only a few global configuration options and
 # will not be updated with more configuration options in the future.
 # If you need more configuration options, take a look at ProxyClass:
-# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
+# https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
@@ -71,10 +78,13 @@ proxyConfig:
   # Note that if you pass multiple tags to this field via `--set` flag to helm upgrade/install commands you must escape the comma (for example, "tag:k8s-proxies\,tag:prod"). See https://github.com/helm/helm/issues/1556
   defaultTags: "tag:k8s"
   firewallMode: auto
+  # If defined, this proxy class will be used as the default proxy class for
+  # service and ingress resources that do not have a proxy class defined.
+  defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.
-# https://tailscale.com/kb/1236/kubernetes-operator/#accessing-the-kubernetes-control-plane-using-an-api-server-proxy
+# https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
 apiServerProxyConfig:
   mode: "false" # "true", "false", "noauth"
 

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -37,7 +37,7 @@ spec:
             exit node.
             Connector is a cluster-scoped resource.
             More info:
-            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
+            https://tailscale.com/kb/1441/kubernetes-operator-connector
           type: object
           required:
             - spec
@@ -115,7 +115,7 @@ spec:
                     To autoapprove the subnet routes or exit node defined by a Connector,
                     you can configure Tailscale ACLs to give these tags the necessary
                     permissions.
-                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
+                    See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
                     If you specify custom tags here, you must also make the operator an owner of these tags.
                     See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
                     Tags cannot be changed once a Connector node has been created.

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -89,14 +89,14 @@ spec:
                   type: object
                   properties:
                     image:
-                      description: Nameserver image.
+                      description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
                       type: object
                       properties:
                         repo:
                           description: Repo defaults to tailscale/k8s-nameserver.
                           type: string
                         tag:
-                          description: Tag defaults to operator's own tag.
+                          description: Tag defaults to unstable.
                           type: string
             status:
               description: |-

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -30,7 +30,7 @@ spec:
             connector.spec.proxyClass field.
             ProxyClass is a cluster scoped resource.
             More info:
-            https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource.
+            https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
           type: object
           required:
             - spec
@@ -1908,7 +1908,7 @@ spec:
                         routes advertized by other nodes on the tailnet, such as subnet
                         routes.
                         This is equivalent of passing --accept-routes flag to a tailscale Linux client.
-                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
+                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
             status:

cmd/k8s-operator/deploy/examples/recorder.yaml

@@ -0,0 +1,6 @@
+apiVersion: tailscale.com/v1alpha1
+kind: Recorder
+metadata:
+  name: recorder
+spec:
+  enableUI: true

cmd/k8s-operator/dnsrecords.go

@@ -3,9 +3,6 @@
 
 //go:build !plan9
 
-// tailscale-operator provides a way to expose services running in a Kubernetes
-// cluster to your Tailnet and to make Tailscale nodes available to cluster
-// workloads
 package main
 
 import (
@@ -27,6 +24,7 @@ import (
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
 )
 
 const (
@@ -170,36 +168,49 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 		}
 	}
 
-	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
-	// headless Service.
+	// Get the Pod IP addresses for the proxy from the EndpointSlices for
+	// the headless Service. The Service can have multiple EndpointSlices
+	// associated with it, for example in dual-stack clusters.
 	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
-	if err != nil {
-		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
+	var eps = new(discoveryv1.EndpointSliceList)
+	if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error listing EndpointSlices for the proxy's headless Service: %w", err)
 	}
-	if eps == nil {
+	if len(eps.Items) == 0 {
 		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
 		return nil
 	}
-	// An EndpointSlice for a Service can have a list of endpoints that each
+	// Each EndpointSlice for a Service can have a list of endpoints that each
 	// can have multiple addresses - these are the IP addresses of any Pods
 	// selected by that Service. Pick all the IPv4 addresses.
-	ips := make([]string, 0)
-	for _, ep := range eps.Endpoints {
-		for _, ip := range ep.Addresses {
-			if !net.IsIPv4String(ip) {
-				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
-			} else {
-				ips = append(ips, ip)
+	// It is also possible that multiple EndpointSlices have overlapping addresses.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints
+	ips := make(set.Set[string], 0)
+	for _, slice := range eps.Items {
+		if slice.AddressType != discoveryv1.AddressTypeIPv4 {
+			logger.Infof("EndpointSlice is for AddressType %s, currently only IPv4 address type is supported", slice.AddressType)
+			continue
+		}
+		for _, ep := range slice.Endpoints {
+			if !epIsReady(&ep) {
+				logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String())
+				continue
+			}
+			for _, ip := range ep.Addresses {
+				if !net.IsIPv4String(ip) {
+					logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+				} else {
+					ips.Add(ip)
+				}
 			}
 		}
 	}
-	if len(ips) == 0 {
+	if ips.Len() == 0 {
 		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
 		return nil
 	}
 	updateFunc := func(rec *operatorutils.Records) {
-		mak.Set(&rec.IP4, fqdn, ips)
+		mak.Set(&rec.IP4, fqdn, ips.Slice())
 	}
 	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
 		return fmt.Errorf("error updating DNS records: %w", err)
@@ -207,6 +218,17 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 	return nil
 }
 
+// epIsReady reports whether the endpoint is currently in a state to receive new
+// traffic. As per kube docs, only explicitly set 'false' for 'Ready' or
+// 'Serving' conditions or explicitly set 'true' for 'Terminating' condition
+// means that the Endpoint is NOT ready.
+// https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/apis/discovery/types.go#L109-L131
+func epIsReady(ep *discoveryv1.Endpoint) bool {
+	return (ep.Conditions.Ready == nil || *ep.Conditions.Ready) &&
+		(ep.Conditions.Serving == nil || *ep.Conditions.Serving) &&
+		(ep.Conditions.Terminating == nil || !*ep.Conditions.Terminating)
+}
+
 // maybeCleanup ensures that the DNS record for the proxy has been removed from
 // dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
 // has been removed from the Service. If the record is not found in the

cmd/k8s-operator/dnsrecords_test.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -87,13 +88,16 @@ func TestDNSRecordsReconciler(t *testing.T) {
 		},
 	}
 	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
-	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4)
+	epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6)
+
 	mustCreate(t, fc, egressSvcFQDN)
 	mustCreate(t, fc, headlessForEgressSvcFQDN)
 	mustCreate(t, fc, ep)
+	mustCreate(t, fc, epv6)
 	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
 	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
-	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} // IPv6 endpoint is currently ignored
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
@@ -106,7 +110,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 3. DNS record is updated if the IP address of the proxy Pod changes.
-	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4)
 	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
 		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
 	})
@@ -116,7 +120,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 
 	// 4. DNS record is created for an ingress proxy configured via Ingress
 	headlessForIngress := headlessSvcForParent(ing, "ingress")
-	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4)
 	mustCreate(t, fc, headlessForIngress)
 	mustCreate(t, fc, ep)
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
@@ -140,6 +144,17 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
 	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
 	expectHostsRecords(t, fc, wantHosts)
+
+	// 7. A not-ready Endpoint is removed from DNS config.
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Conditions.Ready = ptr.To(false)
+		ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{
+			Addresses: []string{"1.2.3.4"},
+		})
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
+	wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"}
+	expectHostsRecords(t, fc, wantHosts)
 }
 
 func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
@@ -162,15 +177,21 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 	}
 }
 
-func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
+func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.AddressType) *discoveryv1.EndpointSlice {
 	return &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      svc.Name,
+			Name:      fmt.Sprintf("%s-%s", svc.Name, string(fam)),
 			Namespace: svc.Namespace,
 			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
 		},
+		AddressType: fam,
 		Endpoints: []discoveryv1.Endpoint{{
 			Addresses: []string{ip},
+			Conditions: discoveryv1.EndpointConditions{
+				Ready:       ptr.To(true),
+				Serving:     ptr.To(true),
+				Terminating: ptr.To(false),
+			},
 		}},
 	}
 }

cmd/k8s-operator/generate/main.go

@@ -3,6 +3,7 @@
 
 //go:build !plan9
 
+// The generate command creates tailscale.com CRDs.
 package main
 
 import (
@@ -23,10 +24,12 @@ const (
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
+	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
+	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -110,7 +113,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass, DNSConfig, Recorder) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -136,28 +139,32 @@ func generate(baseDir string) error {
 		}
 		return nil
 	}
-	if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
+	for _, crd := range []struct {
+		crdPath, templatePath string
+	}{
+		{connectorCRDPath, connectorCRDHelmTemplatePath},
+		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
+		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
+		{recorderCRDPath, recorderCRDHelmTemplatePath},
+	} {
+		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
+			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
+		}
 	}
 	return nil
 }
 
 func cleanup(baseDir string) error {
 	log.Print("Cleaning up CRD from Helm templates")
-	if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up Connector CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
+	for _, path := range []string{
+		connectorCRDHelmTemplatePath,
+		proxyClassCRDHelmTemplatePath,
+		dnsConfigCRDHelmTemplatePath,
+		recorderCRDHelmTemplatePath,
+	} {
+		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("error cleaning up %s: %w", path, err)
+		}
 	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -59,6 +59,9 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -77,4 +80,7 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/ingress.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -46,12 +47,14 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
+
+	proxyDefaultClass string
 }
 
 var (
 	// gaugeIngressResources tracks the number of ingress resources that we're
 	// currently managing.
-	gaugeIngressResources = clientmetric.NewGauge("k8s_ingress_resources")
+	gaugeIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressResourceCount)
 )
 
 func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
@@ -133,7 +136,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing)
+	proxyClass := proxyClassForObject(ing, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/ingress_test.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
@@ -93,6 +94,7 @@ func TestTailscaleIngress(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -224,6 +226,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},

cmd/k8s-operator/nameserver.go

@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/yaml"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -62,9 +63,7 @@ type NameserverReconciler struct {
 	managedNameservers set.Slice[types.UID] // one or none
 }
 
-var (
-	gaugeNameserverResources = clientmetric.NewGauge("k8s_nameserver_resources")
-)
+var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
 
 func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := a.logger.With("dnsConfig", req.Name)

cmd/k8s-operator/nameserver_test.go

@@ -33,7 +33,7 @@ func TestNameserverReconciler(t *testing.T) {
 		},
 		Spec: tsapi.DNSConfigSpec{
 			Nameserver: &tsapi.Nameserver{
-				Image: &tsapi.Image{
+				Image: &tsapi.NameserverImage{
 					Repo: "test",
 					Tag:  "v0.0.1",
 				},

cmd/k8s-operator/operator.go

@@ -22,6 +22,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -39,6 +40,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/kubestore"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
 	"tailscale.com/types/logger"
@@ -51,8 +53,8 @@ import (
 // Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
 //go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
 
-// Generate CRD docs from the yamls
-//go:generate go run fybrik.io/crdoc --resources=./deploy/crds --output=../../k8s-operator/api.md
+// Generate CRD API docs.
+//go:generate go run github.com/elastic/crd-ref-docs --renderer=markdown --source-path=../../k8s-operator/apis/ --config=../../k8s-operator/api-docs-config.yaml --output-path=../../k8s-operator/api.md
 
 func main() {
 	// Required to use our client API. We're fine with the instability since the
@@ -66,6 +68,7 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
@@ -86,9 +89,9 @@ func main() {
 	// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
 	mode := parseAPIProxyMode()
 	if mode == apiserverProxyModeDisabled {
-		hostinfo.SetApp("k8s-operator")
+		hostinfo.SetApp(kubetypes.AppOperator)
 	} else {
-		hostinfo.SetApp("k8s-operator-proxy")
+		hostinfo.SetApp(kubetypes.AppAPIServerProxy)
 	}
 
 	s, tsClient := initTSNet(zlog)
@@ -106,6 +109,7 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
+		proxyDefaultClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -238,6 +242,8 @@ func runReconcilers(opts reconcilerOpts) {
 				&appsv1.StatefulSet{}:        nsFilter,
 				&appsv1.Deployment{}:         nsFilter,
 				&discoveryv1.EndpointSlice{}: nsFilter,
+				&rbacv1.Role{}:               nsFilter,
+				&rbacv1.RoleBinding{}:        nsFilter,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
@@ -279,6 +285,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
+			proxyDefaultClass:     opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -297,10 +304,11 @@ func runReconcilers(opts reconcilerOpts) {
 		Watches(&corev1.Service{}, svcHandlerForIngress).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress).
 		Complete(&IngressReconciler{
-			ssr:      ssr,
-			recorder: eventRecorder,
-			Client:   mgr.GetClient(),
-			logger:   opts.log.Named("ingress-reconciler"),
+			ssr:               ssr,
+			recorder:          eventRecorder,
+			Client:            mgr.GetClient(),
+			logger:            opts.log.Named("ingress-reconciler"),
+			proxyDefaultClass: opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -384,6 +392,28 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create DNS records reconciler: %v", err)
 	}
+
+	// Recorder reconciler.
+	recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
+	err = builder.ControllerManagedBy(mgr).
+		For(&tsapi.Recorder{}).
+		Watches(&appsv1.StatefulSet{}, recorderFilter).
+		Watches(&corev1.ServiceAccount{}, recorderFilter).
+		Watches(&corev1.Secret{}, recorderFilter).
+		Watches(&rbacv1.Role{}, recorderFilter).
+		Watches(&rbacv1.RoleBinding{}, recorderFilter).
+		Complete(&RecorderReconciler{
+			recorder:    eventRecorder,
+			tsNamespace: opts.tailscaleNamespace,
+			Client:      mgr.GetClient(),
+			l:           opts.log.Named("recorder-reconciler"),
+			clock:       tstime.DefaultClock{},
+			tsClient:    opts.tsClient,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create Recorder reconciler: %v", err)
+	}
+
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
@@ -424,6 +454,10 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
+	// proxyDefaultClass is the name of the ProxyClass to use as the default
+	// class for proxies that do not have a ProxyClass set.
+	// this is defined by an operator env variable.
+	proxyDefaultClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
@@ -516,6 +550,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
+	Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error
 }
 

cmd/k8s-operator/operator_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime"
@@ -123,6 +124,7 @@ func TestLoadBalancerClass(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
@@ -260,6 +262,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		parentType:        "svc",
 		tailnetTargetFQDN: tailnetTargetFQDN,
 		hostname:          "default-test",
+		app:               kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -371,6 +374,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		parentType:      "svc",
 		tailnetTargetIP: tailnetTargetIP,
 		hostname:        "default-test",
+		app:             kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -479,6 +483,7 @@ func TestAnnotations(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -584,6 +589,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -713,6 +719,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -852,6 +859,7 @@ func TestCustomHostname(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "reindeer-flotilla",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -964,6 +972,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 		hostname:          "tailscale-critical",
 		priorityClassName: "custom-priority-class-name",
 		clusterTargetIP:   "10.20.30.40",
+		app:               kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@@ -1032,6 +1041,7 @@ func TestProxyClassForService(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
@@ -1125,6 +1135,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -1181,6 +1192,7 @@ func TestProxyFirewallMode(t *testing.T) {
 		hostname:        "default-test",
 		firewallMode:    "nftables",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 }
@@ -1235,6 +1247,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
 		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 
@@ -1529,6 +1542,7 @@ func Test_externalNameService(t *testing.T) {
 		parentType:       "svc",
 		hostname:         "default-test",
 		clusterTargetDNS: "foo.com",
+		app:              kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)

cmd/k8s-operator/proxy.go

@@ -11,16 +11,19 @@ import (
 	"log"
 	"net/http"
 	"net/http/httputil"
+	"net/netip"
 	"net/url"
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
-	tskube "tailscale.com/kube"
+	ksr "tailscale.com/k8s-operator/sessionrecording"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -28,12 +31,27 @@ import (
 	"tailscale.com/util/set"
 )
 
-var whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
-
-var counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
+var (
+	// counterNumRequestsproxies counts the number of API server requests proxied via this proxy.
+	counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
+	whoIsKey                  = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
+)
 
 type apiServerProxyMode int
 
+func (a apiServerProxyMode) String() string {
+	switch a {
+	case apiserverProxyModeDisabled:
+		return "disabled"
+	case apiserverProxyModeEnabled:
+		return "auth"
+	case apiserverProxyModeNoAuth:
+		return "noauth"
+	default:
+		return "unknown"
+	}
+}
+
 const (
 	apiserverProxyModeDisabled apiServerProxyMode = iota
 	apiserverProxyModeEnabled
@@ -97,26 +115,7 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config,
 	if err != nil {
 		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
 	}
-	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode)
-}
-
-// apiserverProxy is an http.Handler that authenticates requests using the Tailscale
-// LocalAPI and then proxies them to the Kubernetes API.
-type apiserverProxy struct {
-	log *zap.SugaredLogger
-	lc  *tailscale.LocalClient
-	rp  *httputil.ReverseProxy
-}
-
-func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	who, err := h.lc.WhoIs(r.Context(), r.RemoteAddr)
-	if err != nil {
-		h.log.Errorf("failed to authenticate caller: %v", err)
-		http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
-		return
-	}
-	counterNumRequestsProxied.Add(1)
-	h.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host)
 }
 
 // runAPIServerProxy runs an HTTP server that authenticates requests using the
@@ -133,64 +132,43 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 //     are passed through to the Kubernetes API.
 //
 // It never returns.
-func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode) {
+func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode, host string) {
 	if mode == apiserverProxyModeDisabled {
 		return
 	}
-	ln, err := s.Listen("tcp", ":443")
+	ln, err := ts.Listen("tcp", ":443")
 	if err != nil {
 		log.Fatalf("could not listen on :443: %v", err)
 	}
-	u, err := url.Parse(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	u, err := url.Parse(host)
 	if err != nil {
 		log.Fatalf("runAPIServerProxy: failed to parse URL %v", err)
 	}
 
-	lc, err := s.LocalClient()
+	lc, err := ts.LocalClient()
 	if err != nil {
 		log.Fatalf("could not get local client: %v", err)
 	}
+
 	ap := &apiserverProxy{
-		log: log,
-		lc:  lc,
-		rp: &httputil.ReverseProxy{
-			Rewrite: func(r *httputil.ProxyRequest) {
-				// Replace the URL with the Kubernetes APIServer.
-
-				r.Out.URL.Scheme = u.Scheme
-				r.Out.URL.Host = u.Host
-				if mode == apiserverProxyModeNoAuth {
-					// If we are not providing authentication, then we are just
-					// proxying to the Kubernetes API, so we don't need to do
-					// anything else.
-					return
-				}
-
-				// We want to proxy to the Kubernetes API, but we want to use
-				// the caller's identity to do so. We do this by impersonating
-				// the caller using the Kubernetes User Impersonation feature:
-				// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
-
-				// Out of paranoia, remove all authentication headers that might
-				// have been set by the client.
-				r.Out.Header.Del("Authorization")
-				r.Out.Header.Del("Impersonate-Group")
-				r.Out.Header.Del("Impersonate-User")
-				r.Out.Header.Del("Impersonate-Uid")
-				for k := range r.Out.Header {
-					if strings.HasPrefix(k, "Impersonate-Extra-") {
-						r.Out.Header.Del(k)
-					}
-				}
-
-				// Now add the impersonation headers that we want.
-				if err := addImpersonationHeaders(r.Out, log); err != nil {
-					panic("failed to add impersonation headers: " + err.Error())
-				}
-			},
-			Transport: rt,
-		},
+		log:         log,
+		lc:          lc,
+		mode:        mode,
+		upstreamURL: u,
+		ts:          ts,
 	}
+	ap.rp = &httputil.ReverseProxy{
+		Rewrite: func(pr *httputil.ProxyRequest) {
+			ap.addImpersonationHeadersAsRequired(pr.Out)
+		},
+		Transport: rt,
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", ap.serveDefault)
+	mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
+	mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
+
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
 		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
@@ -199,14 +177,153 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 			NextProtos:     []string{"http/1.1"},
 		},
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
-		Handler:      ap,
+		Handler:      mux,
 	}
-	log.Infof("listening on %s", ln.Addr())
+	log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr())
 	if err := hs.ServeTLS(ln, "", ""); err != nil {
 		log.Fatalf("runAPIServerProxy: failed to serve %v", err)
 	}
 }
 
+// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale
+// LocalAPI and then proxies them to the Kubernetes API.
+type apiserverProxy struct {
+	log *zap.SugaredLogger
+	lc  *tailscale.LocalClient
+	rp  *httputil.ReverseProxy
+
+	mode        apiServerProxyMode
+	ts          *tsnet.Server
+	upstreamURL *url.URL
+}
+
+// serveDefault is the default handler for Kubernetes API server requests.
+func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
+	who, err := ap.whoIs(r)
+	if err != nil {
+		ap.authError(w, err)
+		return
+	}
+	counterNumRequestsProxied.Add(1)
+	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+}
+
+// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.SPDYProtocol)
+}
+
+// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.WSProtocol)
+}
+
+func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
+	const (
+		podNameKey       = "pod"
+		namespaceNameKey = "namespace"
+		upgradeHeaderKey = "Upgrade"
+	)
+
+	who, err := ap.whoIs(r)
+	if err != nil {
+		ap.authError(w, err)
+		return
+	}
+	counterNumRequestsProxied.Add(1)
+	failOpen, addrs, err := determineRecorderConfig(who)
+	if err != nil {
+		ap.log.Errorf("error trying to determine whether the 'kubectl exec' session needs to be recorded: %v", err)
+		return
+	}
+	if failOpen && len(addrs) == 0 { // will not record
+		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+		return
+	}
+	ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
+	if !failOpen && len(addrs) == 0 {
+		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
+		ap.log.Error(msg)
+		http.Error(w, msg, http.StatusForbidden)
+		return
+	}
+
+	wantsHeader := upgradeHeaderForProto[proto]
+	if h := r.Header.Get(upgradeHeaderKey); h != wantsHeader {
+		msg := fmt.Sprintf("[unexpected] unable to verify that streaming protocol is %s, wants Upgrade header %q, got: %q", proto, wantsHeader, h)
+		if failOpen {
+			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
+			ap.log.Warn(msg)
+			ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+			return
+		}
+		ap.log.Error(msg)
+		msg += "; failure mode is 'fail closed'; closing connection."
+		http.Error(w, msg, http.StatusForbidden)
+		return
+	}
+
+	opts := ksr.HijackerOpts{
+		Req:       r,
+		W:         w,
+		Proto:     proto,
+		TS:        ap.ts,
+		Who:       who,
+		Addrs:     addrs,
+		FailOpen:  failOpen,
+		Pod:       r.PathValue(podNameKey),
+		Namespace: r.PathValue(namespaceNameKey),
+		Log:       ap.log,
+	}
+	h := ksr.New(opts)
+
+	ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+}
+
+func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
+	r.URL.Scheme = h.upstreamURL.Scheme
+	r.URL.Host = h.upstreamURL.Host
+	if h.mode == apiserverProxyModeNoAuth {
+		// If we are not providing authentication, then we are just
+		// proxying to the Kubernetes API, so we don't need to do
+		// anything else.
+		return
+	}
+
+	// We want to proxy to the Kubernetes API, but we want to use
+	// the caller's identity to do so. We do this by impersonating
+	// the caller using the Kubernetes User Impersonation feature:
+	// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
+
+	// Out of paranoia, remove all authentication headers that might
+	// have been set by the client.
+	r.Header.Del("Authorization")
+	r.Header.Del("Impersonate-Group")
+	r.Header.Del("Impersonate-User")
+	r.Header.Del("Impersonate-Uid")
+	for k := range r.Header {
+		if strings.HasPrefix(k, "Impersonate-Extra-") {
+			r.Header.Del(k)
+		}
+	}
+
+	// Now add the impersonation headers that we want.
+	if err := addImpersonationHeaders(r, h.log); err != nil {
+		log.Printf("failed to add impersonation headers: " + err.Error())
+	}
+}
+
+func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
+	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
+}
+
+func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
+	ap.log.Errorf("failed to authenticate caller: %v", err)
+	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
+}
+
 const (
 	// oldCapabilityName is a legacy form of
 	// tailfcg.PeerCapabilityKubernetes capability. The only capability rule
@@ -222,10 +339,10 @@ const (
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)
@@ -266,3 +383,39 @@ func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	}
 	return nil
 }
+
+// determineRecorderConfig determines recorder config from requester's peer
+// capabilities. Determines whether a 'kubectl exec' session from this requester
+// needs to be recorded and what recorders the recording should be sent to.
+func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorderAddresses []netip.AddrPort, _ error) {
+	if who == nil {
+		return false, nil, errors.New("[unexpected] cannot determine caller")
+	}
+	failOpen = true
+	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	if err != nil {
+		return failOpen, nil, fmt.Errorf("failed to unmarshal Kubernetes capability: %w", err)
+	}
+	if len(rules) == 0 {
+		return failOpen, nil, nil
+	}
+
+	for _, rule := range rules {
+		if len(rule.RecorderAddrs) != 0 {
+			// TODO (irbekrm): here or later determine if the
+			// recorders behind those addrs are online - else we
+			// spend 30s trying to reach a recorder whose tailscale
+			// status is offline.
+			recorderAddresses = append(recorderAddresses, rule.RecorderAddrs...)
+		}
+		if rule.EnforceRecorder {
+			failOpen = false
+		}
+	}
+	return failOpen, recorderAddresses, nil
+}
+
+var upgradeHeaderForProto = map[ksr.Protocol]string{
+	ksr.SPDYProtocol: "SPDY/3.1",
+	ksr.WSProtocol:   "websocket",
+}

cmd/k8s-operator/proxy_test.go

@@ -7,6 +7,8 @@ package main
 
 import (
 	"net/http"
+	"net/netip"
+	"reflect"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -126,3 +128,72 @@ func TestImpersonationHeaders(t *testing.T) {
 		}
 	}
 }
+
+func Test_determineRecorderConfig(t *testing.T) {
+	addr1, addr2 := netip.MustParseAddrPort("[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80"), netip.MustParseAddrPort("100.99.99.99:80")
+	tests := []struct {
+		name                  string
+		wantFailOpen          bool
+		wantRecorderAddresses []netip.AddrPort
+		who                   *apitype.WhoIsResponse
+	}{
+		{
+			name:                  "two_ips_fail_closed",
+			who:                   whoResp(map[string][]string{string(tailcfg.PeerCapabilityKubernetes): {`{"recorderAddrs":["[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80","100.99.99.99:80"],"enforceRecorder":true}`}}),
+			wantRecorderAddresses: []netip.AddrPort{addr1, addr2},
+		},
+		{
+			name:                  "two_ips_fail_open",
+			who:                   whoResp(map[string][]string{string(tailcfg.PeerCapabilityKubernetes): {`{"recorderAddrs":["[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80","100.99.99.99:80"]}`}}),
+			wantRecorderAddresses: []netip.AddrPort{addr1, addr2},
+			wantFailOpen:          true,
+		},
+		{
+			name:                  "odd_rule_combination_fail_closed",
+			who:                   whoResp(map[string][]string{string(tailcfg.PeerCapabilityKubernetes): {`{"recorderAddrs":["100.99.99.99:80"],"enforceRecorder":false}`, `{"recorderAddrs":["[fd7a:115c:a1e0:ab12:4843:cd96:626b:628b]:80"]}`, `{"enforceRecorder":true,"impersonate":{"groups":["system:masters"]}}`}}),
+			wantRecorderAddresses: []netip.AddrPort{addr2, addr1},
+		},
+		{
+			name:         "no_caps",
+			who:          whoResp(map[string][]string{}),
+			wantFailOpen: true,
+		},
+		{
+			name:         "no_recorder_caps",
+			who:          whoResp(map[string][]string{"foo": {`{"x":"y"}`}, string(tailcfg.PeerCapabilityKubernetes): {`{"impersonate":{"groups":["system:masters"]}}`}}),
+			wantFailOpen: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFailOpen, gotRecorderAddresses, err := determineRecorderConfig(tt.who)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if gotFailOpen != tt.wantFailOpen {
+				t.Errorf("determineRecorderConfig() gotFailOpen = %v, want %v", gotFailOpen, tt.wantFailOpen)
+			}
+			if !reflect.DeepEqual(gotRecorderAddresses, tt.wantRecorderAddresses) {
+				t.Errorf("determineRecorderConfig() gotRecorderAddresses = %v, want %v", gotRecorderAddresses, tt.wantRecorderAddresses)
+			}
+		})
+	}
+}
+
+func whoResp(capMap map[string][]string) *apitype.WhoIsResponse {
+	resp := &apitype.WhoIsResponse{
+		CapMap: tailcfg.PeerCapMap{},
+	}
+	for cap, rules := range capMap {
+		resp.CapMap[tailcfg.PeerCapability(cap)] = raw(rules...)
+	}
+	return resp
+}
+
+func raw(in ...string) []tailcfg.RawMessage {
+	var out []tailcfg.RawMessage
+	for _, i := range in {
+		out = append(out, tailcfg.RawMessage(i))
+	}
+	return out
+}

cmd/k8s-operator/proxyclass.go

@@ -8,7 +8,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
+	"sync"
 
 	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
@@ -18,6 +20,7 @@ import (
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -25,6 +28,8 @@ import (
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
 )
 
 const (
@@ -41,8 +46,20 @@ type ProxyClassReconciler struct {
 	recorder record.EventRecorder
 	logger   *zap.SugaredLogger
 	clock    tstime.Clock
+
+	mu sync.Mutex // protects following
+
+	// managedProxyClasses is a set of all ProxyClass resources that we're currently
+	// managing. This is only used for metrics.
+	managedProxyClasses set.Slice[types.UID]
 }
 
+var (
+	// gaugeProxyClassResources tracks the number of ProxyClass resources
+	// that we're currently managing.
+	gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources")
+)
+
 func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := pcr.logger.With("ProxyClass", req.Name)
 	logger.Debugf("starting reconcile")
@@ -57,9 +74,26 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyClass: %w", err)
 	}
 	if !pc.DeletionTimestamp.IsZero() {
-		logger.Debugf("ProxyClass is being deleted, do nothing")
-		return reconcile.Result{}, nil
+		logger.Debugf("ProxyClass is being deleted")
+		return reconcile.Result{}, pcr.maybeCleanup(ctx, logger, pc)
 	}
+
+	// Add a finalizer so that we can ensure that metrics get updated when
+	// this ProxyClass is deleted.
+	if !slices.Contains(pc.Finalizers, FinalizerName) {
+		logger.Debugf("updating ProxyClass finalizers")
+		pc.Finalizers = append(pc.Finalizers, FinalizerName)
+		if err := pcr.Update(ctx, pc); err != nil {
+			return res, fmt.Errorf("failed to add finalizer: %w", err)
+		}
+	}
+
+	// Ensure this ProxyClass is tracked in metrics.
+	pcr.mu.Lock()
+	pcr.managedProxyClasses.Add(pc.UID)
+	gaugeProxyClassResources.Set(int64(pcr.managedProxyClasses.Len()))
+	pcr.mu.Unlock()
+
 	oldPCStatus := pc.Status.DeepCopy()
 	if errs := pcr.validate(pc); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
@@ -77,7 +111,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	return reconcile.Result{}, nil
 }
 
-func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
+func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
 	if sts := pc.Spec.StatefulSet; sts != nil {
 		if len(sts.Labels) > 0 {
 			if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
@@ -103,13 +137,13 @@ func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.
 			if tc := pod.TailscaleContainer; tc != nil {
 				for _, e := range tc.Env {
 					if strings.HasPrefix(string(e.Name), "TS_") {
-						a.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
+						pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
 					}
 					if strings.EqualFold(string(e.Name), "EXPERIMENTAL_TS_CONFIGFILE_PATH") {
-						a.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
+						pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
 					}
 					if strings.EqualFold(string(e.Name), "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS") {
-						a.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
+						pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
 					}
 				}
 				if tc.Image != "" {
@@ -135,3 +169,27 @@ func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.
 	// time.
 	return violations
 }
+
+// maybeCleanup removes tailscale.com finalizer and ensures that the ProxyClass
+// is no longer counted towards k8s_proxyclass_resources.
+func (pcr *ProxyClassReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, pc *tsapi.ProxyClass) error {
+	ix := slices.Index(pc.Finalizers, FinalizerName)
+	if ix < 0 {
+		logger.Debugf("no finalizer, nothing to do")
+		pcr.mu.Lock()
+		defer pcr.mu.Unlock()
+		pcr.managedProxyClasses.Remove(pc.UID)
+		gaugeProxyClassResources.Set(int64(pcr.managedProxyClasses.Len()))
+		return nil
+	}
+	pc.Finalizers = append(pc.Finalizers[:ix], pc.Finalizers[ix+1:]...)
+	if err := pcr.Update(ctx, pc); err != nil {
+		return fmt.Errorf("failed to remove finalizer: %w", err)
+	}
+	pcr.mu.Lock()
+	defer pcr.mu.Unlock()
+	pcr.managedProxyClasses.Remove(pc.UID)
+	gaugeProxyClassResources.Set(int64(pcr.managedProxyClasses.Len()))
+	logger.Infof("ProxyClass resources have been cleaned up")
+	return nil
+}

cmd/k8s-operator/proxyclass_test.go

@@ -29,7 +29,8 @@ func TestProxyClass(t *testing.T) {
 			// The apiserver is supposed to set the UID, but the fake client
 			// doesn't. So, set it explicitly because other code later depends
 			// on it being set.
-			UID: types.UID("1234-UID"),
+			UID:        types.UID("1234-UID"),
+			Finalizers: []string{"tailscale.com/finalizer"},
 		},
 		Spec: tsapi.ProxyClassSpec{
 			StatefulSet: &tsapi.StatefulSet{

cmd/k8s-operator/sts.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
@@ -294,6 +295,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 			Selector: map[string]string{
 				"app": sts.ParentResourceUID,
 			},
+			IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack),
 		},
 	}
 	logger.Debugf("reconciling headless service for StatefulSet")
@@ -341,7 +343,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		if len(tags) == 0 {
 			tags = a.defaultTags
 		}
-		authKey, err = a.newAuthKey(ctx, tags)
+		authKey, err = newAuthKey(ctx, a.tsClient, tags)
 		if err != nil {
 			return "", "", nil, err
 		}
@@ -417,6 +419,11 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	if sec == nil {
 		return "", "", nil, nil
 	}
+
+	return deviceInfo(sec)
+}
+
+func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	id = tailcfg.StableNodeID(sec.Data["device_id"])
 	if id == "" {
 		return "", "", nil, nil
@@ -440,7 +447,7 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	return id, hostname, ips, nil
 }
 
-func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {
+func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) {
 	caps := tailscale.KeyCapabilities{
 		Devices: tailscale.KeyDeviceCapabilities{
 			Create: tailscale.KeyDeviceCreateCapabilities{
@@ -451,7 +458,7 @@ func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string)
 		},
 	}
 
-	key, _, err := a.tsClient.CreateKey(ctx, caps)
+	key, _, err := tsClient.CreateKey(ctx, caps)
 	if err != nil {
 		return "", err
 	}
@@ -597,6 +604,18 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	}
+	app, err := appInfoForProxy(sts)
+	if err != nil {
+		// No need to error out if now or in future we end up in a
+		// situation where app info cannot be determined for one of the
+		// many proxy configurations that the operator can produce.
+		logger.Error("[unexpected] unable to determine proxy type")
+	} else {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_INTERNAL_APP",
+			Value: app,
+		})
+	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	if sts.ProxyClassName != "" {
 		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
@@ -610,6 +629,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS)
 }
 
+func appInfoForProxy(cfg *tailscaleSTSConfig) (string, error) {
+	if cfg.ClusterTargetDNSName != "" || cfg.ClusterTargetIP != "" {
+		return kubetypes.AppIngressProxy, nil
+	}
+	if cfg.TailnetTargetFQDN != "" || cfg.TailnetTargetIP != "" {
+		return kubetypes.AppEgressProxy, nil
+	}
+	if cfg.ServeConfig != nil {
+		return kubetypes.AppIngressResource, nil
+	}
+	if cfg.Connector != nil {
+		return kubetypes.AppConnector, nil
+	}
+	return "", errors.New("unable to determine proxy type")
+}
+
 // mergeStatefulSetLabelsOrAnnots returns a map that contains all keys/values
 // present in 'custom' map as well as those keys/values from the current map
 // whose keys are present in the 'managed' map. The reason why this merge is

cmd/k8s-operator/svc.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
@@ -62,15 +63,17 @@ type ServiceReconciler struct {
 	tsNamespace string
 
 	clock tstime.Clock
+
+	proxyDefaultClass string
 }
 
 var (
 	// gaugeEgressProxies tracks the number of egress proxies that we're
 	// currently managing.
-	gaugeEgressProxies = clientmetric.NewGauge("k8s_egress_proxies")
+	gaugeEgressProxies = clientmetric.NewGauge(kubetypes.MetricEgressProxyCount)
 	// gaugeIngressProxies tracks the number of ingress proxies that we're
 	// currently managing.
-	gaugeIngressProxies = clientmetric.NewGauge("k8s_ingress_proxies")
+	gaugeIngressProxies = clientmetric.NewGauge(kubetypes.MetricIngressProxyCount)
 )
 
 func childResourceLabels(name, ns, typ string) map[string]string {
@@ -208,7 +211,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc)
+	proxyClass := proxyClassForObject(svc, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -325,7 +328,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	if err != nil {
 		msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
 		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
-		return fmt.Errorf(msg)
+		return errors.New(msg)
 	}
 	for _, ip := range tsIPs {
 		addr, err := netip.ParseAddr(ip)
@@ -404,8 +407,14 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-func proxyClassForObject(o client.Object) string {
-	return o.GetLabels()[LabelProxyClass]
+// proxyClassForObject returns the proxy class for the given object. If the
+// object does not have a proxy class label, it returns the default proxy class
+func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+	proxyClass, exists := o.GetLabels()[LabelProxyClass]
+	if !exists {
+		proxyClass = proxyDefaultClass
+	}
+	return proxyClass
 }
 
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {

cmd/k8s-operator/testutils_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/netip"
+	"reflect"
 	"strings"
 	"sync"
 	"testing"
@@ -51,6 +52,7 @@ type configOpts struct {
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
 	proxyClass                                     string // configuration from the named ProxyClass should be applied to proxy resources
+	app                                            string
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -142,6 +144,10 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
 		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
 	}
+	tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
+		Name:  "TS_INTERNAL_APP",
+		Value: opts.app,
+	})
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -224,6 +230,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
+			{Name: "TS_INTERNAL_APP", Value: opts.app},
 		},
 		ImagePullPolicy: "Always",
 		VolumeMounts: []corev1.VolumeMount{
@@ -319,7 +326,8 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
 			Selector: map[string]string{
 				"app": "1234-UID",
 			},
-			ClusterIP: "None",
+			ClusterIP:      "None",
+			IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack),
 		},
 	}
 }
@@ -480,7 +488,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
-		t.Fatalf("unexpected object (-got +want):\n%s", diff)
+		t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
 	}
 }
 
@@ -491,7 +499,7 @@ func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns
 		Name:      name,
 		Namespace: ns,
 	}, obj); !apierrors.IsNotFound(err) {
-		t.Fatalf("object %s/%s unexpectedly present, wanted missing", ns, name)
+		t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
 	}
 }
 
@@ -585,6 +593,17 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili
 	return "secret-authkey", k, nil
 }
 
+func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) {
+	return &tailscale.Device{
+		DeviceID: deviceID,
+		Hostname: "test-device",
+		Addresses: []string{
+			"1.2.3.4",
+			"::1",
+		},
+	}, nil
+}
+
 func (c *fakeTSClient) DeleteDevice(ctx context.Context, deviceID string) error {
 	c.Lock()
 	defer c.Unlock()

cmd/k8s-operator/tsrecorder.go

@@ -0,0 +1,375 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"sync"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/tailscale"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonRecorderCreationFailed = "RecorderCreationFailed"
+	reasonRecorderCreated        = "RecorderCreated"
+	reasonRecorderInvalid        = "RecorderInvalid"
+
+	currentProfileKey = "_current-profile"
+)
+
+var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount)
+
+// RecorderReconciler syncs Recorder statefulsets with their definition in
+// Recorder CRs.
+type RecorderReconciler struct {
+	client.Client
+	l           *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+	tsClient    tsClient
+
+	mu        sync.Mutex           // protects following
+	recorders set.Slice[types.UID] // for recorders gauge
+}
+
+func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger {
+	return r.l.With("Recorder", name)
+}
+
+func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
+	logger := r.logger(req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	tsr := new(tsapi.Recorder)
+	err = r.Get(ctx, req.NamespacedName, tsr)
+	if apierrors.IsNotFound(err) {
+		logger.Debugf("Recorder not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err)
+	}
+	if markedForDeletion(tsr) {
+		logger.Debugf("Recorder is being deleted, cleaning up resources")
+		ix := xslices.Index(tsr.Finalizers, FinalizerName)
+		if ix < 0 {
+			logger.Debugf("no finalizer, nothing to do")
+			return reconcile.Result{}, nil
+		}
+
+		if done, err := r.maybeCleanup(ctx, tsr); err != nil {
+			return reconcile.Result{}, err
+		} else if !done {
+			logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
+			return reconcile.Result{RequeueAfter: shortRequeue}, nil
+		}
+
+		tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
+		if err := r.Update(ctx, tsr); err != nil {
+			return reconcile.Result{}, err
+		}
+		return reconcile.Result{}, nil
+	}
+
+	oldTSRStatus := tsr.Status.DeepCopy()
+	setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
+		if !apiequality.Semantic.DeepEqual(oldTSRStatus, tsr.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
+		}
+		return reconcile.Result{}, err
+	}
+
+	if !slices.Contains(tsr.Finalizers, FinalizerName) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to log that the high level, multi-reconcile
+		// operation is underway.
+		logger.Infof("ensuring Recorder is set up")
+		tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
+		if err := r.Update(ctx, tsr); err != nil {
+			logger.Errorf("error adding finalizer: %w", err)
+			return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
+		}
+	}
+
+	if err := r.validate(tsr); err != nil {
+		logger.Errorf("error validating Recorder spec: %w", err)
+		message := fmt.Sprintf("Recorder is invalid: %s", err)
+		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message)
+		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
+	}
+
+	if err = r.maybeProvision(ctx, tsr); err != nil {
+		logger.Errorf("error creating Recorder resources: %w", err)
+		message := fmt.Sprintf("failed creating Recorder: %s", err)
+		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
+		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, message)
+	}
+
+	logger.Info("Recorder resources synced")
+	return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated)
+}
+
+func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error {
+	logger := r.logger(tsr.Name)
+
+	r.mu.Lock()
+	r.recorders.Add(tsr.UID)
+	gaugeRecorderResources.Set(int64(r.recorders.Len()))
+	r.mu.Unlock()
+
+	if err := r.ensureAuthSecretCreated(ctx, tsr); err != nil {
+		return fmt.Errorf("error creating secrets: %w", err)
+	}
+	// State secret is precreated so we can use the Recorder CR as its owner ref.
+	sec := tsrStateSecret(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
+		s.ObjectMeta.Labels = sec.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error creating state Secret: %w", err)
+	}
+	sa := tsrServiceAccount(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
+		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error creating ServiceAccount: %w", err)
+	}
+	role := tsrRole(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
+		r.ObjectMeta.Labels = role.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
+		r.Rules = role.Rules
+	}); err != nil {
+		return fmt.Errorf("error creating Role: %w", err)
+	}
+	roleBinding := tsrRoleBinding(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
+		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
+		r.RoleRef = roleBinding.RoleRef
+		r.Subjects = roleBinding.Subjects
+	}); err != nil {
+		return fmt.Errorf("error creating RoleBinding: %w", err)
+	}
+	ss := tsrStatefulSet(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
+		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
+		s.Spec = ss.Spec
+	}); err != nil {
+		return fmt.Errorf("error creating StatefulSet: %w", err)
+	}
+
+	var devices []tsapi.TailnetDevice
+
+	device, ok, err := r.getDeviceInfo(ctx, tsr.Name)
+	if err != nil {
+		return fmt.Errorf("failed to get device info: %w", err)
+	}
+	if !ok {
+		logger.Debugf("no Tailscale hostname known yet, waiting for Recorder pod to finish auth")
+		return nil
+	}
+
+	devices = append(devices, device)
+
+	tsr.Status.Devices = devices
+
+	return nil
+}
+
+// maybeCleanup just deletes the device from the tailnet. All the kubernetes
+// resources linked to a Recorder will get cleaned up via owner references
+// (which we can use because they are all in the same namespace).
+func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
+	logger := r.logger(tsr.Name)
+
+	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
+	if err != nil {
+		return false, err
+	}
+	if !ok {
+		logger.Debugf("state Secret %s-0 not found or does not contain node ID, continuing cleanup", tsr.Name)
+		r.mu.Lock()
+		r.recorders.Remove(tsr.UID)
+		gaugeRecorderResources.Set(int64(r.recorders.Len()))
+		r.mu.Unlock()
+		return true, nil
+	}
+
+	logger.Debugf("deleting device %s from control", string(id))
+	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
+			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
+		} else {
+			return false, fmt.Errorf("error deleting device: %w", err)
+		}
+	} else {
+		logger.Debugf("device %s deleted from control", string(id))
+	}
+
+	// Unlike most log entries in the reconcile loop, this will get printed
+	// exactly once at the very end of cleanup, because the final step of
+	// cleanup removes the tailscale finalizer, which will make all future
+	// reconciles exit early.
+	logger.Infof("cleaned up Recorder resources")
+	r.mu.Lock()
+	r.recorders.Remove(tsr.UID)
+	gaugeRecorderResources.Set(int64(r.recorders.Len()))
+	r.mu.Unlock()
+	return true, nil
+}
+
+func (r *RecorderReconciler) ensureAuthSecretCreated(ctx context.Context, tsr *tsapi.Recorder) error {
+	logger := r.logger(tsr.Name)
+	key := types.NamespacedName{
+		Namespace: r.tsNamespace,
+		Name:      tsr.Name,
+	}
+	if err := r.Get(ctx, key, &corev1.Secret{}); err == nil {
+		// No updates, already created the auth key.
+		logger.Debugf("auth Secret %s already exists", key.Name)
+		return nil
+	} else if !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// Create the auth key Secret which is going to be used by the StatefulSet
+	// to authenticate with Tailscale.
+	logger.Debugf("creating authkey for new Recorder")
+	tags := tsr.Spec.Tags
+	if len(tags) == 0 {
+		tags = tsapi.Tags{"tag:k8s"}
+	}
+	authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify())
+	if err != nil {
+		return err
+	}
+
+	logger.Debug("creating a new Secret for the Recorder")
+	if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *RecorderReconciler) validate(tsr *tsapi.Recorder) error {
+	if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil {
+		return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible")
+	}
+
+	return nil
+}
+
+// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// is expected to always be non-empty if the node ID is, but not required.
+func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.tsNamespace,
+			Name:      fmt.Sprintf("%s-0", tsrName),
+		},
+	}
+	if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return "", "", false, nil
+		}
+
+		return "", "", false, err
+	}
+
+	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
+	currentProfile, ok := secret.Data[currentProfileKey]
+	if !ok {
+		return "", "", false, nil
+	}
+	profileBytes, ok := secret.Data[string(currentProfile)]
+	if !ok {
+		return "", "", false, nil
+	}
+	var profile profile
+	if err := json.Unmarshal(profileBytes, &profile); err != nil {
+		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
+	}
+
+	ok = profile.Config.NodeID != ""
+	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
+}
+
+func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.TailnetDevice, ok bool, err error) {
+	nodeID, dnsName, ok, err := r.getNodeMetadata(ctx, tsrName)
+	if !ok || err != nil {
+		return tsapi.TailnetDevice{}, false, err
+	}
+
+	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
+	// need the API. Should we instead update the profile to include addresses?
+	device, err := r.tsClient.Device(ctx, string(nodeID), nil)
+	if err != nil {
+		return tsapi.TailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
+	}
+
+	d = tsapi.TailnetDevice{
+		Hostname:   device.Hostname,
+		TailnetIPs: device.Addresses,
+	}
+	if dnsName != "" {
+		d.URL = fmt.Sprintf("https://%s", dnsName)
+	}
+
+	return d, true, nil
+}
+
+type profile struct {
+	Config struct {
+		NodeID      string `json:"NodeID"`
+		UserProfile struct {
+			LoginName string `json:"LoginName"`
+		} `json:"UserProfile"`
+	} `json:"Config"`
+}
+
+func markedForDeletion(tsr *tsapi.Recorder) bool {
+	return !tsr.DeletionTimestamp.IsZero()
+}

cmd/k8s-operator/tsrecorder_specs.go

@@ -0,0 +1,278 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+	"tailscale.com/version"
+)
+
+func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet {
+	return &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Labels),
+			OwnerReferences: tsrOwnerReference(tsr),
+			Annotations:     tsr.Spec.StatefulSet.Annotations,
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: ptr.To[int32](1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels),
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tsr.Name,
+					Namespace:   namespace,
+					Labels:      labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels),
+					Annotations: tsr.Spec.StatefulSet.Pod.Annotations,
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: tsr.Name,
+					Affinity:           tsr.Spec.StatefulSet.Pod.Affinity,
+					SecurityContext:    tsr.Spec.StatefulSet.Pod.SecurityContext,
+					ImagePullSecrets:   tsr.Spec.StatefulSet.Pod.ImagePullSecrets,
+					NodeSelector:       tsr.Spec.StatefulSet.Pod.NodeSelector,
+					Tolerations:        tsr.Spec.StatefulSet.Pod.Tolerations,
+					Containers: []corev1.Container{
+						{
+							Name: "recorder",
+							Image: func() string {
+								image := tsr.Spec.StatefulSet.Pod.Container.Image
+								if image == "" {
+									image = fmt.Sprintf("tailscale/tsrecorder:%s", selfVersionImageTag())
+								}
+
+								return image
+							}(),
+							ImagePullPolicy: tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy,
+							Resources:       tsr.Spec.StatefulSet.Pod.Container.Resources,
+							SecurityContext: tsr.Spec.StatefulSet.Pod.Container.SecurityContext,
+							Env:             env(tsr),
+							EnvFrom: func() []corev1.EnvFromSource {
+								if tsr.Spec.Storage.S3 == nil || tsr.Spec.Storage.S3.Credentials.Secret.Name == "" {
+									return nil
+								}
+
+								return []corev1.EnvFromSource{{
+									SecretRef: &corev1.SecretEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: tsr.Spec.Storage.S3.Credentials.Secret.Name,
+										},
+									},
+								}}
+							}(),
+							Command: []string{"/tsrecorder"},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "data",
+									MountPath: "/data",
+									ReadOnly:  false,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: "data",
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func tsrServiceAccount(tsr *tsapi.Recorder, namespace string) *corev1.ServiceAccount {
+	return &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+	}
+}
+
+func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"get",
+					"patch",
+					"update",
+				},
+				ResourceNames: []string{
+					tsr.Name,                      // Contains the auth key.
+					fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
+				},
+			},
+		},
+	}
+}
+
+func tsrRoleBinding(tsr *tsapi.Recorder, namespace string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      tsr.Name,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: tsr.Name,
+		},
+	}
+}
+
+func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:       namespace,
+			Name:            tsr.Name,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		StringData: map[string]string{
+			"authkey": authKey,
+		},
+	}
+}
+
+func tsrStateSecret(tsr *tsapi.Recorder, namespace string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            fmt.Sprintf("%s-0", tsr.Name),
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+	}
+}
+
+func env(tsr *tsapi.Recorder) []corev1.EnvVar {
+	envs := []corev1.EnvVar{
+		{
+			Name: "TS_AUTHKEY",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: tsr.Name,
+					},
+					Key: "authkey",
+				},
+			},
+		},
+		{
+			Name: "POD_NAME",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					// Secret is named after the pod.
+					FieldPath: "metadata.name",
+				},
+			},
+		},
+		{
+			Name:  "TS_STATE",
+			Value: "kube:$(POD_NAME)",
+		},
+		{
+			Name:  "TSRECORDER_HOSTNAME",
+			Value: "$(POD_NAME)",
+		},
+	}
+
+	for _, env := range tsr.Spec.StatefulSet.Pod.Container.Env {
+		envs = append(envs, corev1.EnvVar{
+			Name:  string(env.Name),
+			Value: env.Value,
+		})
+	}
+
+	if tsr.Spec.Storage.S3 != nil {
+		envs = append(envs,
+			corev1.EnvVar{
+				Name:  "TSRECORDER_DST",
+				Value: fmt.Sprintf("s3://%s", tsr.Spec.Storage.S3.Endpoint),
+			},
+			corev1.EnvVar{
+				Name:  "TSRECORDER_BUCKET",
+				Value: tsr.Spec.Storage.S3.Bucket,
+			},
+		)
+	} else {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TSRECORDER_DST",
+			Value: "/data/recordings",
+		})
+	}
+
+	if tsr.Spec.EnableUI {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TSRECORDER_UI",
+			Value: "true",
+		})
+	}
+
+	return envs
+}
+
+func labels(app, instance string, customLabels map[string]string) map[string]string {
+	l := make(map[string]string, len(customLabels)+3)
+	for k, v := range customLabels {
+		l[k] = v
+	}
+
+	// ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
+	l["app.kubernetes.io/name"] = app
+	l["app.kubernetes.io/instance"] = instance
+	l["app.kubernetes.io/managed-by"] = "tailscale-operator"
+
+	return l
+}
+
+func tsrOwnerReference(owner metav1.Object) []metav1.OwnerReference {
+	return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("Recorder"))}
+}
+
+// selfVersionImageTag returns the container image tag of the running operator
+// build.
+func selfVersionImageTag() string {
+	meta := version.GetMeta()
+	var versionPrefix string
+	if meta.UnstableBranch {
+		versionPrefix = "unstable-"
+	}
+	return fmt.Sprintf("%sv%s", versionPrefix, meta.MajorMinorPatch)
+}

cmd/k8s-operator/tsrecorder_specs_test.go

@@ -0,0 +1,143 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+)
+
+func TestRecorderSpecs(t *testing.T) {
+	t.Run("ensure spec fields are passed through correctly", func(t *testing.T) {
+		tsr := &tsapi.Recorder{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test",
+			},
+			Spec: tsapi.RecorderSpec{
+				StatefulSet: tsapi.RecorderStatefulSet{
+					Labels: map[string]string{
+						"ss-label-key": "ss-label-value",
+					},
+					Annotations: map[string]string{
+						"ss-annotation-key": "ss-annotation-value",
+					},
+					Pod: tsapi.RecorderPod{
+						Labels: map[string]string{
+							"pod-label-key": "pod-label-value",
+						},
+						Annotations: map[string]string{
+							"pod-annotation-key": "pod-annotation-value",
+						},
+						Affinity: &corev1.Affinity{
+							PodAffinity: &corev1.PodAffinity{
+								RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{
+									LabelSelector: &metav1.LabelSelector{
+										MatchLabels: map[string]string{
+											"match-label": "match-value",
+										},
+									}},
+								},
+							},
+						},
+						SecurityContext: &corev1.PodSecurityContext{
+							RunAsUser: ptr.To[int64](1000),
+						},
+						ImagePullSecrets: []corev1.LocalObjectReference{{
+							Name: "img-pull",
+						}},
+						NodeSelector: map[string]string{
+							"some-node": "selector",
+						},
+						Tolerations: []corev1.Toleration{{
+							Key:               "key",
+							Value:             "value",
+							TolerationSeconds: ptr.To[int64](60),
+						}},
+						Container: tsapi.RecorderContainer{
+							Env: []tsapi.Env{{
+								Name:  "some_env",
+								Value: "env_value",
+							}},
+							Image:           "custom-image",
+							ImagePullPolicy: corev1.PullAlways,
+							SecurityContext: &corev1.SecurityContext{
+								Capabilities: &corev1.Capabilities{
+									Add: []corev1.Capability{
+										"NET_ADMIN",
+									},
+								},
+							},
+							Resources: corev1.ResourceRequirements{
+								Limits: corev1.ResourceList{
+									corev1.ResourceCPU: resource.MustParse("100m"),
+								},
+								Requests: corev1.ResourceList{
+									corev1.ResourceCPU: resource.MustParse("50m"),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		ss := tsrStatefulSet(tsr, tsNamespace)
+
+		// StatefulSet-level.
+		if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Annotations, tsr.Spec.StatefulSet.Pod.Annotations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+
+		// Pod-level.
+		if diff := cmp.Diff(ss.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.SecurityContext, tsr.Spec.StatefulSet.Pod.SecurityContext); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.ImagePullSecrets, tsr.Spec.StatefulSet.Pod.ImagePullSecrets); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.NodeSelector, tsr.Spec.StatefulSet.Pod.NodeSelector); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Tolerations, tsr.Spec.StatefulSet.Pod.Tolerations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+
+		// Container-level.
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].SecurityContext, tsr.Spec.StatefulSet.Pod.Container.SecurityContext); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+	})
+}

cmd/k8s-operator/tsrecorder_test.go

@@ -0,0 +1,162 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+)
+
+const tsNamespace = "tailscale"
+
+func TestRecorder(t *testing.T) {
+	tsr := &tsapi.Recorder{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Finalizers: []string{"tailscale.com/finalizer"},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(tsr).
+		WithStatusSubresource(tsr).
+		Build()
+	tsClient := &fakeTSClient{}
+	zl, _ := zap.NewDevelopment()
+	fr := record.NewFakeRecorder(1)
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	reconciler := &RecorderReconciler{
+		tsNamespace: tsNamespace,
+		Client:      fc,
+		tsClient:    tsClient,
+		recorder:    fr,
+		l:           zl.Sugar(),
+		clock:       cl,
+	}
+
+	t.Run("invalid spec gives an error condition", func(t *testing.T) {
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar())
+		expectEqual(t, fc, tsr, nil)
+		if expected := 0; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		expectRecorderResources(t, fc, tsr, false)
+
+		expectedEvent := "Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
+		expectEvents(t, fr, []string{expectedEvent})
+	})
+
+	t.Run("observe Ready=true status condition for a valid spec", func(t *testing.T) {
+		tsr.Spec.EnableUI = true
+		mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
+			t.Spec = tsr.Spec
+		})
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar())
+		expectEqual(t, fc, tsr, nil)
+		if expected := 1; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		expectRecorderResources(t, fc, tsr, true)
+	})
+
+	t.Run("populate node info in state secret, and see it appear in status", func(t *testing.T) {
+		bytes, err := json.Marshal(map[string]any{
+			"Config": map[string]any{
+				"NodeID": "nodeid-123",
+				"UserProfile": map[string]any{
+					"LoginName": "test-0.example.ts.net",
+				},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		const key = "profile-abc"
+		mustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) {
+			s.Data = map[string][]byte{
+				currentProfileKey: []byte(key),
+				key:               bytes,
+			}
+		})
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+		tsr.Status.Devices = []tsapi.TailnetDevice{
+			{
+				Hostname:   "test-device",
+				TailnetIPs: []string{"1.2.3.4", "::1"},
+				URL:        "https://test-0.example.ts.net",
+			},
+		}
+		expectEqual(t, fc, tsr, nil)
+	})
+
+	t.Run("delete the Recorder and observe cleanup", func(t *testing.T) {
+		if err := fc.Delete(context.Background(), tsr); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		expectMissing[tsapi.Recorder](t, fc, "", tsr.Name)
+		if expected := 0; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-123"}); diff != "" {
+			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
+		}
+		// The fake client does not clean up objects whose owner has been
+		// deleted, so we can't test for the owned resources getting deleted.
+	})
+}
+
+func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) {
+	t.Helper()
+
+	auth := tsrAuthSecret(tsr, tsNamespace, "secret-authkey")
+	state := tsrStateSecret(tsr, tsNamespace)
+	role := tsrRole(tsr, tsNamespace)
+	roleBinding := tsrRoleBinding(tsr, tsNamespace)
+	serviceAccount := tsrServiceAccount(tsr, tsNamespace)
+	statefulSet := tsrStatefulSet(tsr, tsNamespace)
+
+	if shouldExist {
+		expectEqual(t, fc, auth, nil)
+		expectEqual(t, fc, state, nil)
+		expectEqual(t, fc, role, nil)
+		expectEqual(t, fc, roleBinding, nil)
+		expectEqual(t, fc, serviceAccount, nil)
+		expectEqual(t, fc, statefulSet, nil)
+	} else {
+		expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name)
+		expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name)
+		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
+		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
+		expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
+		expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
+	}
+}

cmd/natc/natc.go

@@ -448,7 +448,7 @@ func (c *connector) handleTCPFlow(src, dst netip.AddrPort) (handler func(net.Con
 // in --ignore-destinations
 func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool {
 	for _, a := range dstAddrs {
-		if _, ok := c.ignoreDsts.Get(a); ok {
+		if _, ok := c.ignoreDsts.Lookup(a); ok {
 			return true
 		}
 	}
@@ -456,6 +456,11 @@ func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool {
 }
 
 func proxyTCPConn(c net.Conn, dest string) {
+	if c.RemoteAddr() == nil {
+		log.Printf("proxyTCPConn: nil RemoteAddr")
+		c.Close()
+		return
+	}
 	addrPortStr := c.LocalAddr().String()
 	_, port, err := net.SplitHostPort(addrPortStr)
 	if err != nil {
@@ -489,7 +494,10 @@ type perPeerState struct {
 func (ps *perPeerState) domainForIP(ip netip.Addr) (_ string, ok bool) {
 	ps.mu.Lock()
 	defer ps.mu.Unlock()
-	return ps.addrToDomain.Get(ip)
+	if ps.addrToDomain == nil {
+		return "", false
+	}
+	return ps.addrToDomain.Lookup(ip)
 }
 
 // ipForDomain assigns a pair of unique IP addresses for the given domain and
@@ -515,7 +523,7 @@ func (ps *perPeerState) ipForDomain(domain string) ([]netip.Addr, error) {
 // domain.
 // ps.mu must be held.
 func (ps *perPeerState) isIPUsedLocked(ip netip.Addr) bool {
-	_, ok := ps.addrToDomain.Get(ip)
+	_, ok := ps.addrToDomain.Lookup(ip)
 	return ok
 }
 

cmd/quictest/quictest.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+	"github.com/quic-go/quic-go/qlog"
+)
+
+func main() {
+	count := flag.Int("count", 1, "times to fetch the URL(s)")
+	delay := flag.Duration("duration", time.Second, "delay between --count runs")
+	quiet := flag.Bool("q", false, "don't print the data")
+	keyLogFile := flag.String("keylog", "", "key log file")
+	insecure := flag.Bool("insecure", false, "skip certificate verification")
+	flag.Parse()
+	urls := flag.Args()
+
+	var keyLog io.Writer
+	if len(*keyLogFile) > 0 {
+		f, err := os.Create(*keyLogFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+		keyLog = f
+	}
+
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	roundTripper := &http3.RoundTripper{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            pool,
+			InsecureSkipVerify: *insecure,
+			KeyLogWriter:       keyLog,
+		},
+		QUICConfig: &quic.Config{
+			Tracer: qlog.DefaultConnectionTracer,
+		},
+	}
+	defer roundTripper.Close()
+	hclient := &http.Client{
+		Transport: roundTripper,
+	}
+
+	for range *count {
+
+		var wg sync.WaitGroup
+		wg.Add(len(urls))
+		for _, addr := range urls {
+			log.Printf("GET %s", addr)
+			go func(addr string) {
+				rsp, err := hclient.Get(addr)
+				if err != nil {
+					log.Fatal(err)
+				}
+				log.Printf("Got response for %s: %#v", addr, rsp)
+
+				body := &bytes.Buffer{}
+				_, err = io.Copy(body, rsp.Body)
+				if err != nil {
+					log.Fatal(err)
+				}
+				if *quiet {
+					log.Printf("Response Body: %d bytes", body.Len())
+				} else {
+					log.Printf("Response Body (%d bytes):\n%s", body.Len(), body.Bytes())
+				}
+				wg.Done()
+			}(addr)
+		}
+		wg.Wait()
+
+		time.Sleep(*delay)
+	}
+
+}

cmd/stund/depaware.txt

@@ -2,6 +2,12 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
 
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
+        github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
         github.com/google/uuid                                       from tailscale.com/util/fastuuid
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
@@ -44,6 +50,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/envknob                                        from tailscale.com/tsweb+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/net/stunserver+
         tailscale.com/net/netaddr                                    from tailscale.com/net/tsaddr
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
@@ -59,7 +66,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/types/lazy                                     from tailscale.com/version+
         tailscale.com/types/logger                                   from tailscale.com/tsweb
         tailscale.com/types/opt                                      from tailscale.com/envknob+
-        tailscale.com/types/ptr                                      from tailscale.com/tailcfg
+        tailscale.com/types/ptr                                      from tailscale.com/tailcfg+
         tailscale.com/types/structs                                  from tailscale.com/tailcfg+
         tailscale.com/types/tkatype                                  from tailscale.com/tailcfg+
         tailscale.com/types/views                                    from tailscale.com/net/tsaddr+
@@ -75,15 +82,16 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/version/distro                                 from tailscale.com/envknob
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/net/dns/dnsmessage                              from net
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
+        golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
@@ -128,6 +136,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
+        encoding/base32                                              from github.com/go-json-experiment/json
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
         encoding/hex                                                 from crypto/x509+
@@ -146,6 +155,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from google.golang.org/protobuf/internal/impl
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/tailcfg+
@@ -161,7 +171,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -188,3 +198,4 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/stunstamp/api.go

@@ -1,142 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package main
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"errors"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	sq "github.com/Masterminds/squirrel"
-)
-
-type api struct {
-	db  *db
-	mux *http.ServeMux
-}
-
-func newAPI(db *db) *api {
-	a := &api{
-		db: db,
-	}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/query", a.query)
-	a.mux = mux
-	return a
-}
-
-type apiResult struct {
-	At         int    `json:"at"` // time.Time.Unix()
-	RegionID   int    `json:"regionID"`
-	Hostname   string `json:"hostname"`
-	Af         int    `json:"af"` // 4 or 6
-	Addr       string `json:"addr"`
-	Source     int    `json:"source"` // timestampSourceUserspace (0) or timestampSourceKernel (1)
-	StableConn bool   `json:"stableConn"`
-	DstPort    int    `json:"dstPort"`
-	RttNS      *int   `json:"rttNS"`
-}
-
-func getTimeBounds(vals url.Values) (from time.Time, to time.Time, err error) {
-	lastForm, ok := vals["last"]
-	if ok && len(lastForm) > 0 {
-		dur, err := time.ParseDuration(lastForm[0])
-		if err != nil {
-			return time.Time{}, time.Time{}, err
-		}
-		now := time.Now()
-		return now.Add(-dur), now, nil
-	}
-
-	fromForm, ok := vals["from"]
-	if ok && len(fromForm) > 0 {
-		fromUnixSec, err := strconv.Atoi(fromForm[0])
-		if err != nil {
-			return time.Time{}, time.Time{}, err
-		}
-		from = time.Unix(int64(fromUnixSec), 0)
-		toForm, ok := vals["to"]
-		if ok && len(toForm) > 0 {
-			toUnixSec, err := strconv.Atoi(toForm[0])
-			if err != nil {
-				return time.Time{}, time.Time{}, err
-			}
-			to = time.Unix(int64(toUnixSec), 0)
-		} else {
-			return time.Time{}, time.Time{}, errors.New("from specified without to")
-		}
-		return from, to, nil
-	}
-
-	// no time bounds specified, default to last 1h
-	now := time.Now()
-	return now.Add(-time.Hour), now, nil
-}
-
-func (a *api) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	a.mux.ServeHTTP(w, r)
-}
-
-func (a *api) query(w http.ResponseWriter, r *http.Request) {
-	err := r.ParseForm()
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-	from, to, err := getTimeBounds(r.Form)
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-
-	sb := sq.Select("at_unix", "region_id", "hostname", "af", "address", "timestamp_source", "stable_conn", "dst_port", "rtt_ns").From("rtt")
-	sb = sb.Where(sq.And{
-		sq.GtOrEq{"at_unix": from.Unix()},
-		sq.LtOrEq{"at_unix": to.Unix()},
-	})
-	query, args, err := sb.ToSql()
-	if err != nil {
-		return
-	}
-
-	rows, err := a.db.Query(query, args...)
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-	results := make([]apiResult, 0)
-	for rows.Next() {
-		rtt := 0
-		result := apiResult{
-			RttNS: &rtt,
-		}
-		err = rows.Scan(&result.At, &result.RegionID, &result.Hostname, &result.Af, &result.Addr, &result.Source, &result.StableConn, &result.DstPort, &result.RttNS)
-		if err != nil {
-			http.Error(w, err.Error(), 500)
-			return
-		}
-		results = append(results, result)
-	}
-	if rows.Err() != nil {
-		http.Error(w, rows.Err().Error(), 500)
-		return
-	}
-	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-		gz := gzip.NewWriter(w)
-		defer gz.Close()
-		w.Header().Set("Content-Encoding", "gzip")
-		err = json.NewEncoder(gz).Encode(&results)
-	} else {
-		err = json.NewEncoder(w).Encode(&results)
-	}
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-}

cmd/stunstamp/stunstamp_db_default.go

@@ -1,26 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !(windows && 386)
-
-package main
-
-import (
-	"database/sql"
-
-	_ "modernc.org/sqlite"
-)
-
-type db struct {
-	*sql.DB
-}
-
-func newDB(path string) (*db, error) {
-	d, err := sql.Open("sqlite", *flagOut)
-	if err != nil {
-		return nil, err
-	}
-	return &db{
-		DB: d,
-	}, nil
-}

cmd/stunstamp/stunstamp_db_windows_386.go

@@ -1,17 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package main
-
-import (
-	"database/sql"
-	"errors"
-)
-
-type db struct {
-	*sql.DB
-}
-
-func newDB(path string) (*db, error) {
-	return nil, errors.New("unsupported platform")
-}

cmd/stunstamp/stunstamp_default.go

@@ -8,18 +8,58 @@ package main
 import (
 	"errors"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unimplemented")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	return 0, errors.New("unimplemented")
 }
 
-func supportsKernelTS() bool {
-	return false
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	case protocolICMP:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: false,
+			stableConn:  false,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func getICMPConn(forDst netip.Addr, source timestampSource) (io.ReadWriteCloser, error) {
+	return nil, errors.New("platform unsupported")
+}
+
+func mkICMPMeasureFn(source timestampSource) measureFn {
+	return func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+		return 0, errors.New("platform unsupported")
+	}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	return nil
 }

cmd/stunstamp/stunstamp_linux.go

@@ -10,21 +10,27 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
+	"math"
+	"math/rand/v2"
+	"net/netip"
+	"syscall"
 	"time"
 
 	"github.com/mdlayher/socket"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/stun"
 )
 
 const (
-	flags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
+	timestampingFlags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
 		unix.SOF_TIMESTAMPING_RX_SOFTWARE | // rx timestamp generation in the kernel
 		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
 	if err != nil {
 		return nil, err
@@ -34,7 +40,7 @@ func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, flags)
+	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, timestampingFlags)
 	if err != nil {
 		return nil, err
 	}
@@ -56,24 +62,144 @@ func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
 	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func mkICMPMeasureFn(source timestampSource) measureFn {
+	return func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+		return measureICMPRTT(source, conn, hostname, dst)
+	}
+}
+
+func measureICMPRTT(source timestampSource, conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	sconn, ok := conn.(*socket.Conn)
+	if !ok {
+		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
+	}
+	txBody := &icmp.Echo{
+		// The kernel overrides this and routes appropriately so there is no
+		// point in setting or verifying.
+		ID: 0,
+		// Make this sufficiently random so that we do not account a late
+		// arriving reply in a future probe window.
+		Seq: int(rand.Int32N(math.MaxUint16)),
+		// Fingerprint ourselves.
+		Data: []byte("stunstamp"),
+	}
+	txMsg := icmp.Message{
+		Body: txBody,
+	}
+	var to unix.Sockaddr
+	if dst.Addr().Is4() {
+		txMsg.Type = ipv4.ICMPTypeEcho
+		to = &unix.SockaddrInet4{}
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
+	} else {
+		txMsg.Type = ipv6.ICMPTypeEchoRequest
+		to = &unix.SockaddrInet6{}
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
+	}
+	txBuf, err := txMsg.Marshal(nil)
+	if err != nil {
+		return 0, err
+	}
+	txAt := time.Now()
+	err = sconn.Sendto(context.Background(), txBuf, 0, to)
+	if err != nil {
+		return 0, fmt.Errorf("sendto error: %v", err)
+	}
+
+	if source == timestampSourceKernel {
+		txCtx, txCancel := context.WithTimeout(context.Background(), txRxTimeout)
+		defer txCancel()
+
+		buf := make([]byte, 1024)
+		oob := make([]byte, 1024)
+
+		for {
+			n, oobn, _, _, err := sconn.Recvmsg(txCtx, buf, oob, unix.MSG_ERRQUEUE)
+			if err != nil {
+				return 0, fmt.Errorf("recvmsg (MSG_ERRQUEUE) error: %v", err) // don't wrap
+			}
+
+			buf = buf[:n]
+			// Spin until we find the message we sent. We get the full packet
+			// looped including eth header so match against the tail.
+			if n < len(txBuf) {
+				continue
+			}
+			txLoopedMsg, err := icmp.ParseMessage(txMsg.Type.Protocol(), buf[len(buf)-len(txBuf):])
+			if err != nil {
+				continue
+			}
+			txLoopedBody, ok := txLoopedMsg.Body.(*icmp.Echo)
+			if !ok || txLoopedBody.Seq != txBody.Seq || txLoopedMsg.Code != txMsg.Code ||
+				txLoopedMsg.Type != txLoopedMsg.Type || !bytes.Equal(txLoopedBody.Data, txBody.Data) {
+				continue
+			}
+			txAt, err = parseTimestampFromCmsgs(oob[:oobn])
+			if err != nil {
+				return 0, fmt.Errorf("failed to get tx timestamp: %v", err) // don't wrap
+			}
+			break
+		}
+	}
+
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), txRxTimeout)
+	defer rxCancel()
+
+	rxBuf := make([]byte, 1024)
+	oob := make([]byte, 1024)
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(rxCtx, rxBuf, oob, 0)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg error: %w", err)
+		}
+		rxAt := time.Now()
+		rxMsg, err := icmp.ParseMessage(txMsg.Type.Protocol(), rxBuf[:n])
+		if err != nil {
+			continue
+		}
+		if txMsg.Type == ipv4.ICMPTypeEcho {
+			if rxMsg.Type != ipv4.ICMPTypeEchoReply {
+				continue
+			}
+		} else {
+			if rxMsg.Type != ipv6.ICMPTypeEchoReply {
+				continue
+			}
+		}
+		if rxMsg.Code != txMsg.Code {
+			continue
+		}
+		rxBody, ok := rxMsg.Body.(*icmp.Echo)
+		if !ok || rxBody.Seq != txBody.Seq || !bytes.Equal(rxBody.Data, txBody.Data) {
+			continue
+		}
+		if source == timestampSourceKernel {
+			rxAt, err = parseTimestampFromCmsgs(oob[:oobn])
+			if err != nil {
+				return 0, fmt.Errorf("failed to get rx timestamp: %v", err)
+			}
+		}
+		return rxAt.Sub(txAt), nil
+	}
+}
+
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	sconn, ok := conn.(*socket.Conn)
 	if !ok {
 		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
 	}
 
 	var to unix.Sockaddr
-	to4 := dst.IP.To4()
-	if to4 != nil {
+	if dst.Addr().Is4() {
 		to = &unix.SockaddrInet4{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
 	} else {
 		to = &unix.SockaddrInet6{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
 	}
 
 	txID := stun.NewTxID()
@@ -84,7 +210,7 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 		return 0, fmt.Errorf("sendto error: %v", err) // don't wrap
 	}
 
-	txCtx, txCancel := context.WithTimeout(context.Background(), time.Second*2)
+	txCtx, txCancel := context.WithTimeout(context.Background(), txRxTimeout)
 	defer txCancel()
 
 	buf := make([]byte, 1024)
@@ -110,7 +236,7 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 		break
 	}
 
-	rxCtx, rxCancel := context.WithTimeout(context.Background(), time.Second*2)
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), txRxTimeout)
 	defer rxCancel()
 
 	for {
@@ -138,6 +264,54 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 
 }
 
-func supportsKernelTS() bool {
-	return true
+func getICMPConn(forDst netip.Addr, source timestampSource) (io.ReadWriteCloser, error) {
+	domain := unix.AF_INET
+	proto := unix.IPPROTO_ICMP
+	if forDst.Is6() {
+		domain = unix.AF_INET6
+		proto = unix.IPPROTO_ICMPV6
+	}
+	conn, err := socket.Socket(domain, unix.SOCK_DGRAM, proto, "icmp", nil)
+	if err != nil {
+		return nil, err
+	}
+	if source == timestampSourceKernel {
+		err = conn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, timestampingFlags)
+	}
+	return conn, err
+}
+
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	case protocolICMP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  false,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	// we may restart faster than TIME_WAIT can clear
+	return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
 }

cmd/systray/README.md

@@ -0,0 +1,11 @@
+# systray
+
+The systray command is a minimal Tailscale systray application for Linux.
+It is designed to provide quick access to common operations like profile switching
+and exit node selection.
+
+## Supported platforms
+
+The `fyne.io/systray` package we use supports Windows, macOS, Linux, and many BSDs,
+so the systray application will likely work for the most part on those platforms.
+Notifications currently only work on Linux, as that is the main target.

cmd/systray/logo.go

@@ -0,0 +1,220 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"image/color"
+	"image/png"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/fogleman/gg"
+)
+
+// tsLogo represents the state of the 3x3 dot grid in the Tailscale logo.
+// A 0 represents a gray dot, any other value is a white dot.
+type tsLogo [9]byte
+
+var (
+	// disconnected is all gray dots
+	disconnected = tsLogo{
+		0, 0, 0,
+		0, 0, 0,
+		0, 0, 0,
+	}
+
+	// connected is the normal Tailscale logo
+	connected = tsLogo{
+		0, 0, 0,
+		1, 1, 1,
+		0, 1, 0,
+	}
+
+	// loading is a special tsLogo value that is not meant to be rendered directly,
+	// but indicates that the loading animation should be shown.
+	loading = tsLogo{'l', 'o', 'a', 'd', 'i', 'n', 'g'}
+
+	// loadingIcons are shown in sequence as an animated loading icon.
+	loadingLogos = []tsLogo{
+		{
+			0, 1, 1,
+			1, 0, 1,
+			0, 0, 1,
+		},
+		{
+			0, 1, 1,
+			0, 0, 1,
+			0, 1, 0,
+		},
+		{
+			0, 1, 1,
+			0, 0, 0,
+			0, 0, 1,
+		},
+		{
+			0, 0, 1,
+			0, 1, 0,
+			0, 0, 0,
+		},
+		{
+			0, 1, 0,
+			0, 0, 0,
+			0, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 1,
+			0, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 0,
+			0, 0, 0,
+		},
+		{
+			0, 0, 1,
+			0, 0, 0,
+			0, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 0,
+			1, 0, 0,
+		},
+		{
+			0, 0, 0,
+			0, 0, 0,
+			1, 1, 0,
+		},
+		{
+			0, 0, 0,
+			1, 0, 0,
+			1, 1, 0,
+		},
+		{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 0,
+		},
+		{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 1,
+		},
+		{
+			0, 0, 0,
+			1, 1, 1,
+			0, 0, 1,
+		},
+		{
+			0, 1, 0,
+			0, 1, 1,
+			1, 0, 1,
+		},
+	}
+)
+
+var (
+	black = color.NRGBA{0, 0, 0, 255}
+	white = color.NRGBA{255, 255, 255, 255}
+	gray  = color.NRGBA{255, 255, 255, 102}
+)
+
+// render returns a PNG image of the logo.
+func (logo tsLogo) render() *bytes.Buffer {
+	const radius = 25
+	const borderUnits = 1
+	dim := radius * (8 + borderUnits*2)
+
+	dc := gg.NewContext(dim, dim)
+	dc.DrawRectangle(0, 0, float64(dim), float64(dim))
+	dc.SetColor(black)
+	dc.Fill()
+
+	for y := 0; y < 3; y++ {
+		for x := 0; x < 3; x++ {
+			px := (borderUnits + 1 + 3*x) * radius
+			py := (borderUnits + 1 + 3*y) * radius
+			col := white
+			if logo[y*3+x] == 0 {
+				col = gray
+			}
+			dc.DrawCircle(float64(px), float64(py), radius)
+			dc.SetColor(col)
+			dc.Fill()
+		}
+	}
+
+	b := bytes.NewBuffer(nil)
+	png.Encode(b, dc.Image())
+	return b
+}
+
+// setAppIcon renders logo and sets it as the systray icon.
+func setAppIcon(icon tsLogo) {
+	if icon == loading {
+		startLoadingAnimation()
+	} else {
+		stopLoadingAnimation()
+		systray.SetIcon(icon.render().Bytes())
+	}
+}
+
+var (
+	loadingMu sync.Mutex // protects loadingCancel
+
+	// loadingCancel stops the loading animation in the systray icon.
+	// This is nil if the animation is not currently active.
+	loadingCancel func()
+)
+
+// startLoadingAnimation starts the animated loading icon in the system tray.
+// The animation continues until [stopLoadingAnimation] is called.
+// If the loading animation is already active, this func does nothing.
+func startLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		// loading icon already displayed
+		return
+	}
+
+	ctx := context.Background()
+	ctx, loadingCancel = context.WithCancel(ctx)
+
+	go func() {
+		t := time.NewTicker(500 * time.Millisecond)
+		var i int
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				systray.SetIcon(loadingLogos[i].render().Bytes())
+				i++
+				if i >= len(loadingLogos) {
+					i = 0
+				}
+			}
+		}
+	}()
+}
+
+// stopLoadingAnimation stops the animated loading icon in the system tray.
+// If the loading animation is not currently active, this func does nothing.
+func stopLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		loadingCancel()
+		loadingCancel = nil
+	}
+}

cmd/systray/systray.go

@@ -0,0 +1,258 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+// The systray command is a minimal Tailscale systray application for Linux.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/atotto/clipboard"
+	dbus "github.com/godbus/dbus/v5"
+	"github.com/toqueteos/webbrowser"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+)
+
+var (
+	localClient tailscale.LocalClient
+	chState     chan ipn.State // tailscale state changes
+
+	appIcon *os.File
+)
+
+func main() {
+	systray.Run(onReady, onExit)
+}
+
+// Menu represents the systray menu, its items, and the current Tailscale state.
+type Menu struct {
+	mu     sync.Mutex // protects the entire Menu
+	status *ipnstate.Status
+
+	connect    *systray.MenuItem
+	disconnect *systray.MenuItem
+
+	self *systray.MenuItem
+	more *systray.MenuItem
+	quit *systray.MenuItem
+
+	eventCancel func() // cancel eventLoop
+}
+
+func onReady() {
+	log.Printf("starting")
+	ctx := context.Background()
+
+	setAppIcon(disconnected)
+
+	// dbus wants a file path for notification icons, so copy to a temp file.
+	appIcon, _ = os.CreateTemp("", "tailscale-systray.png")
+	io.Copy(appIcon, connected.render())
+
+	chState = make(chan ipn.State, 1)
+
+	status, err := localClient.Status(ctx)
+	if err != nil {
+		log.Print(err)
+	}
+
+	menu := new(Menu)
+	menu.rebuild(status)
+
+	go watchIPNBus(ctx)
+}
+
+// rebuild the systray menu based on the current Tailscale state.
+//
+// We currently rebuild the entire menu because it is not easy to update the existing menu.
+// You cannot iterate over the items in a menu, nor can you remove some items like separators.
+// So for now we rebuild the whole thing, and can optimize this later if needed.
+func (menu *Menu) rebuild(status *ipnstate.Status) {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+	menu.status = status
+	systray.ResetMenu()
+
+	menu.connect = systray.AddMenuItem("Connect", "")
+	menu.disconnect = systray.AddMenuItem("Disconnect", "")
+	menu.disconnect.Hide()
+	systray.AddSeparator()
+
+	if status != nil && status.Self != nil {
+		title := fmt.Sprintf("This Device: %s (%s)", status.Self.HostName, status.Self.TailscaleIPs[0])
+		menu.self = systray.AddMenuItem(title, "")
+	}
+	systray.AddSeparator()
+
+	menu.more = systray.AddMenuItem("More settings", "")
+	menu.more.Enable()
+
+	menu.quit = systray.AddMenuItem("Quit", "Quit the app")
+	menu.quit.Enable()
+
+	ctx := context.Background()
+	ctx, menu.eventCancel = context.WithCancel(ctx)
+	go menu.eventLoop(ctx)
+}
+
+// eventLoop is the main event loop for handling click events on menu items
+// and responding to Tailscale state changes.
+// This method does not return until ctx.Done is closed.
+func (menu *Menu) eventLoop(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case state := <-chState:
+			switch state {
+			case ipn.Running:
+				setAppIcon(loading)
+				status, err := localClient.Status(ctx)
+				if err != nil {
+					log.Printf("error getting tailscale status: %v", err)
+				}
+				menu.rebuild(status)
+				setAppIcon(connected)
+				menu.connect.SetTitle("Connected")
+				menu.connect.Disable()
+				menu.disconnect.Show()
+				menu.disconnect.Enable()
+			case ipn.NoState, ipn.Stopped:
+				menu.connect.SetTitle("Connect")
+				menu.connect.Enable()
+				menu.disconnect.Hide()
+				setAppIcon(disconnected)
+			case ipn.Starting:
+				setAppIcon(loading)
+			}
+		case <-menu.connect.ClickedCh:
+			_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: true,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Print(err)
+				continue
+			}
+
+		case <-menu.disconnect.ClickedCh:
+			_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: false,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("disconnecting: %v", err)
+				continue
+			}
+
+		case <-menu.self.ClickedCh:
+			copyTailscaleIP(menu.status.Self)
+
+		case <-menu.more.ClickedCh:
+			webbrowser.Open("http://100.100.100.100/")
+
+		case <-menu.quit.ClickedCh:
+			systray.Quit()
+		}
+	}
+}
+
+// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
+// This method does not return.
+func watchIPNBus(ctx context.Context) {
+	for {
+		if err := watchIPNBusInner(ctx); err != nil {
+			log.Println(err)
+			if errors.Is(err, context.Canceled) {
+				// If the context got canceled, we will never be able to
+				// reconnect to IPN bus, so exit the process.
+				log.Fatalf("watchIPNBus: %v", err)
+			}
+		}
+		// If our watch connection breaks, wait a bit before reconnecting. No
+		// reason to spam the logs if e.g. tailscaled is restarting or goes
+		// down.
+		time.Sleep(3 * time.Second)
+	}
+}
+
+func watchIPNBusInner(ctx context.Context) error {
+	watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return fmt.Errorf("watching ipn bus: %w", err)
+	}
+	defer watcher.Close()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			n, err := watcher.Next()
+			if err != nil {
+				return fmt.Errorf("ipnbus error: %w", err)
+			}
+			if n.State != nil {
+				chState <- *n.State
+				log.Printf("new state: %v", n.State)
+			}
+		}
+	}
+}
+
+// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
+// and sends a notification with the copied value.
+func copyTailscaleIP(device *ipnstate.PeerStatus) {
+	if device == nil || len(device.TailscaleIPs) == 0 {
+		return
+	}
+	name := strings.Split(device.DNSName, ".")[0]
+	ip := device.TailscaleIPs[0].String()
+	err := clipboard.WriteAll(ip)
+	if err != nil {
+		log.Printf("clipboard error: %v", err)
+	}
+
+	sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
+}
+
+// sendNotification sends a desktop notification with the given title and content.
+func sendNotification(title, content string) {
+	conn, err := dbus.SessionBus()
+	if err != nil {
+		log.Printf("dbus: %v", err)
+		return
+	}
+	timeout := 3 * time.Second
+	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
+	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
+		appIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
+	if call.Err != nil {
+		log.Printf("dbus: %v", call.Err)
+	}
+}
+
+func onExit() {
+	log.Printf("exiting")
+	os.Remove(appIcon.Name())
+}

cmd/tailscale/cli/cert.go

@@ -16,6 +16,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"software.sslmate.com/src/go-pkcs12"
@@ -34,14 +35,16 @@ var certCmd = &ffcli.Command{
 		fs.StringVar(&certArgs.certFile, "cert-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.crt if --cert-file and --key-file are both unset")
 		fs.StringVar(&certArgs.keyFile, "key-file", "", "output key file or \"-\" for stdout; defaults to DOMAIN.key if --cert-file and --key-file are both unset")
 		fs.BoolVar(&certArgs.serve, "serve-demo", false, "if true, serve on port :443 using the cert as a demo, instead of writing out the files to disk")
+		fs.DurationVar(&certArgs.minValidity, "min-validity", 0, "ensure the certificate is valid for at least this duration; the output certificate is never expired if this flag is unset or 0, but the lifetime may vary; the maximum allowed min-validity depends on the CA")
 		return fs
 	})(),
 }
 
 var certArgs struct {
-	certFile string
-	keyFile  string
-	serve    bool
+	certFile    string
+	keyFile     string
+	serve       bool
+	minValidity time.Duration
 }
 
 func runCert(ctx context.Context, args []string) error {
@@ -102,7 +105,7 @@ func runCert(ctx context.Context, args []string) error {
 		certArgs.certFile = domain + ".crt"
 		certArgs.keyFile = domain + ".key"
 	}
-	certPEM, keyPEM, err := localClient.CertPair(ctx, domain)
+	certPEM, keyPEM, err := localClient.CertPairWithValidity(ctx, domain, certArgs.minValidity)
 	if err != nil {
 		return err
 	}

cmd/tailscale/cli/cli.go

@@ -84,6 +84,13 @@ var localClient = tailscale.LocalClient{
 
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) (err error) {
+	if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 {
+		// We're running on gokrazy and it's the first start.
+		// Don't run the tailscale CLI as a service; just exit.
+		// See https://gokrazy.org/development/process-interface/
+		os.Exit(0)
+	}
+
 	args = CleanUpArgs(args)
 
 	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
@@ -180,6 +187,7 @@ change in the future.
 			configureCmd,
 			netcheckCmd,
 			ipCmd,
+			dnsCmd,
 			statusCmd,
 			pingCmd,
 			ncCmd,

cmd/tailscale/cli/debug.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"time"
@@ -319,9 +320,36 @@ var debugCmd = &ffcli.Command{
 				return fs
 			})(),
 		},
+		{
+			Name:       "resolve",
+			ShortUsage: "tailscale debug resolve <hostname>",
+			Exec:       runDebugResolve,
+			ShortHelp:  "Does a DNS lookup",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("resolve")
+				fs.StringVar(&resolveArgs.net, "net", "ip", "network type to resolve (ip, ip4, ip6)")
+				return fs
+			})(),
+		},
+		{
+			Name:       "go-buildinfo",
+			ShortUsage: "tailscale debug go-buildinfo",
+			ShortHelp:  "Prints Go's runtime/debug.BuildInfo",
+			Exec:       runGoBuildInfo,
+		},
 	},
 }
 
+func runGoBuildInfo(ctx context.Context, args []string) error {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		return errors.New("no Go build info")
+	}
+	e := json.NewEncoder(os.Stdout)
+	e.SetIndent("", "\t")
+	return e.Encode(bi)
+}
+
 var debugArgs struct {
 	file    string
 	cpuSec  int
@@ -1167,3 +1195,26 @@ func runDebugDialTypes(ctx context.Context, args []string) error {
 	fmt.Printf("%s", body)
 	return nil
 }
+
+var resolveArgs struct {
+	net string // "ip", "ip4", "ip6""
+}
+
+func runDebugResolve(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: tailscale debug resolve <hostname>")
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	host := args[0]
+	ips, err := net.DefaultResolver.LookupIP(ctx, resolveArgs.net, host)
+	if err != nil {
+		return err
+	}
+	for _, ip := range ips {
+		fmt.Printf("%s\n", ip)
+	}
+	return nil
+}

cmd/tailscale/cli/dns-query.go

@@ -0,0 +1,163 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/netip"
+	"os"
+	"text/tabwriter"
+
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/types/dnstype"
+)
+
+func runDNSQuery(ctx context.Context, args []string) error {
+	if len(args) < 1 {
+		return flag.ErrHelp
+	}
+	name := args[0]
+	queryType := "A"
+	if len(args) >= 2 {
+		queryType = args[1]
+	}
+	fmt.Printf("DNS query for %q (%s) using internal resolver:\n", name, queryType)
+	fmt.Println()
+	bytes, resolvers, err := localClient.QueryDNS(ctx, name, queryType)
+	if err != nil {
+		fmt.Printf("failed to query DNS: %v\n", err)
+		return nil
+	}
+
+	if len(resolvers) == 1 {
+		fmt.Printf("Forwarding to resolver: %v\n", makeResolverString(*resolvers[0]))
+	} else {
+		fmt.Println("Multiple resolvers available:")
+		for _, r := range resolvers {
+			fmt.Printf("  - %v\n", makeResolverString(*r))
+		}
+	}
+	fmt.Println()
+	var p dnsmessage.Parser
+	header, err := p.Start(bytes)
+	if err != nil {
+		fmt.Printf("failed to parse DNS response: %v\n", err)
+		return err
+	}
+	fmt.Printf("Response code: %v\n", header.RCode.String())
+	fmt.Println()
+	p.SkipAllQuestions()
+	if header.RCode != dnsmessage.RCodeSuccess {
+		fmt.Println("No answers were returned.")
+		return nil
+	}
+	answers, err := p.AllAnswers()
+	if err != nil {
+		fmt.Printf("failed to parse DNS answers: %v\n", err)
+		return err
+	}
+	if len(answers) == 0 {
+		fmt.Println("  (no answers found)")
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "Name\tTTL\tClass\tType\tBody")
+	fmt.Fprintln(w, "----\t---\t-----\t----\t----")
+	for _, a := range answers {
+		fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", a.Header.Name.String(), a.Header.TTL, a.Header.Class.String(), a.Header.Type.String(), makeAnswerBody(a))
+	}
+	w.Flush()
+
+	fmt.Println()
+	return nil
+}
+
+// makeAnswerBody returns a string with the DNS answer body in a human-readable format.
+func makeAnswerBody(a dnsmessage.Resource) string {
+	switch a.Header.Type {
+	case dnsmessage.TypeA:
+		return makeABody(a.Body)
+	case dnsmessage.TypeAAAA:
+		return makeAAAABody(a.Body)
+	case dnsmessage.TypeCNAME:
+		return makeCNAMEBody(a.Body)
+	case dnsmessage.TypeMX:
+		return makeMXBody(a.Body)
+	case dnsmessage.TypeNS:
+		return makeNSBody(a.Body)
+	case dnsmessage.TypeOPT:
+		return makeOPTBody(a.Body)
+	case dnsmessage.TypePTR:
+		return makePTRBody(a.Body)
+	case dnsmessage.TypeSRV:
+		return makeSRVBody(a.Body)
+	case dnsmessage.TypeTXT:
+		return makeTXTBody(a.Body)
+	default:
+		return a.Body.GoString()
+	}
+}
+
+func makeABody(a dnsmessage.ResourceBody) string {
+	if a, ok := a.(*dnsmessage.AResource); ok {
+		return netip.AddrFrom4(a.A).String()
+	}
+	return ""
+}
+func makeAAAABody(aaaa dnsmessage.ResourceBody) string {
+	if a, ok := aaaa.(*dnsmessage.AAAAResource); ok {
+		return netip.AddrFrom16(a.AAAA).String()
+	}
+	return ""
+}
+func makeCNAMEBody(cname dnsmessage.ResourceBody) string {
+	if c, ok := cname.(*dnsmessage.CNAMEResource); ok {
+		return c.CNAME.String()
+	}
+	return ""
+}
+func makeMXBody(mx dnsmessage.ResourceBody) string {
+	if m, ok := mx.(*dnsmessage.MXResource); ok {
+		return fmt.Sprintf("%s (Priority=%d)", m.MX, m.Pref)
+	}
+	return ""
+}
+func makeNSBody(ns dnsmessage.ResourceBody) string {
+	if n, ok := ns.(*dnsmessage.NSResource); ok {
+		return n.NS.String()
+	}
+	return ""
+}
+func makeOPTBody(opt dnsmessage.ResourceBody) string {
+	if o, ok := opt.(*dnsmessage.OPTResource); ok {
+		return o.GoString()
+	}
+	return ""
+}
+func makePTRBody(ptr dnsmessage.ResourceBody) string {
+	if p, ok := ptr.(*dnsmessage.PTRResource); ok {
+		return p.PTR.String()
+	}
+	return ""
+}
+func makeSRVBody(srv dnsmessage.ResourceBody) string {
+	if s, ok := srv.(*dnsmessage.SRVResource); ok {
+		return fmt.Sprintf("Target=%s, Port=%d, Priority=%d, Weight=%d", s.Target.String(), s.Port, s.Priority, s.Weight)
+	}
+	return ""
+}
+func makeTXTBody(txt dnsmessage.ResourceBody) string {
+	if t, ok := txt.(*dnsmessage.TXTResource); ok {
+		return fmt.Sprintf("%q", t.TXT)
+	}
+	return ""
+}
+func makeResolverString(r dnstype.Resolver) string {
+	if len(r.BootstrapResolution) > 0 {
+		return fmt.Sprintf("%s (bootstrap: %v)", r.Addr, r.BootstrapResolution)
+	}
+	return fmt.Sprintf("%s", r.Addr)
+}