util/execqueue: add metrics

Expose enough metrics to get a sense of queue depth, use and if it has
stalled.

Updates tailscale/corp#26058

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I271ac8d03f3db587a33aca6964fe92f2833e1251

.github/workflows/checklocks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: "actions/checkout@v4"
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,18 +23,17 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version-file: go.mod
           cache: false
 
       - name: golangci-lint
-        # Note: this is the 'v3' tag as of 2023-08-14
-        uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299
+        uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
         with:
-          version: v1.56
+          version: v1.60
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -24,13 +24,13 @@ jobs:
 
       - name: Post to slack
         if: failure() && github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@v1.24.0
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
-          channel-id: 'C05PXRM304B'
+          method: chat.postMessage
+          token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
           payload: |
             {
+              "channel": "C05PXRM304B",
               "blocks": [
                 {
                   "type": "section",

.github/workflows/installer.yml

@@ -6,11 +6,13 @@ on:
       - "main"
     paths:
       - scripts/installer.sh
+      - .github/workflows/installer.yml
   pull_request:
     branches:
       - "*"
     paths:
       - scripts/installer.sh
+      - .github/workflows/installer.yml
 
 jobs:
   test:
@@ -29,10 +31,9 @@ jobs:
           - "debian:stable-slim"
           - "debian:testing-slim"
           - "debian:sid-slim"
-          - "ubuntu:18.04"
           - "ubuntu:20.04"
           - "ubuntu:22.04"
-          - "ubuntu:23.04"
+          - "ubuntu:24.04"
           - "elementary/docker:stable"
           - "elementary/docker:unstable"
           - "parrotsec/core:lts-amd64"
@@ -48,7 +49,7 @@ jobs:
           - "opensuse/leap:latest"
           - "opensuse/tumbleweed:latest"
           - "archlinux:latest"
-          - "alpine:3.14"
+          - "alpine:3.21"
           - "alpine:latest"
           - "alpine:edge"
         deps:
@@ -58,10 +59,6 @@ jobs:
           # Check a few images with wget rather than curl.
           - { image: "debian:oldstable-slim", deps: "wget" }
           - { image: "debian:sid-slim", deps: "wget" }
-          - { image: "ubuntu:23.04", deps: "wget" }
-          # Ubuntu 16.04 also needs apt-transport-https installed.
-          - { image: "ubuntu:16.04", deps: "curl apt-transport-https" }
-          - { image: "ubuntu:16.04", deps: "wget apt-transport-https" }
     runs-on: ubuntu-latest
     container:
       image: ${{ matrix.image }}
@@ -98,7 +95,7 @@ jobs:
       # We cannot use v4, as it requires a newer glibc version than some of the
       # tested images provide. See
       # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@v3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -78,9 +78,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -150,16 +150,16 @@ jobs:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -190,7 +190,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +202,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +214,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +258,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -295,7 +295,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -313,13 +313,19 @@ jobs:
           # AIX
           - goos: aix
             goarch: ppc64
+          # Solaris
+          - goos: solaris
+            goarch: amd64
+          # illumos
+          - goos: illumos
+            goarch: amd64
 
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -350,7 +356,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -365,9 +371,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -399,7 +405,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -456,18 +462,22 @@ jobs:
         fuzz-seconds: 300
         dry-run: false
         language: go
+    - name: Set artifacts_path in env (workaround for actions/upload-artifact#176)
+      if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
+      run: |
+        echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
-        path: ./out/artifacts
+        path: ${{ env.artifacts_path }}/out/artifacts
 
   depaware:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
@@ -477,7 +487,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -490,7 +500,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -502,7 +512,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -518,7 +528,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck
@@ -559,8 +569,10 @@ jobs:
       # By having the job always run, but skipping its only step as needed, we
       # let the CI output collapse nicely in PRs.
       if: failure() && github.event_name == 'push'
-      uses: ruby/action-slack@v3.2.1
+      uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
       with:
+        webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+        webhook-type: incoming-webhook
         payload: |
           {
             "attachments": [{
@@ -572,8 +584,6 @@ jobs:
               "color": "danger"
             }]
           }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
   check_mergeability:
     if: always()
@@ -596,6 +606,6 @@ jobs:
     steps:
     - name: Decide if change is okay to merge
       if: github.event_name != 'push'
-      uses: re-actors/alls-green@release/v1
+      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
       with:
         jobs: ${{ toJSON(needs) }}

.github/workflows/update-flake.yml

@@ -21,21 +21,22 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run update-flakes
         run: ./update-flake.sh
 
       - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         id: generate-token
         with:
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_retrieval_mode: "id"
+          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run go get
         run: |
@@ -23,18 +23,19 @@ jobs:
           ./tool/go mod tidy
 
       - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         id: generate-token
         with:
           # TODO(will): this should use the code updater app rather than licensing.
           # It has the same permissions, so not a big deal, but still.
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_retrieval_mode: "id"
+          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

Dockerfile

@@ -27,7 +27,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.22-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 

Makefile

@@ -100,7 +100,7 @@ publishdevoperator: ## Build and publish k8s-operator image to location specifie
 	@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
 	@test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1)
 	@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
-	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=operator ./build_docker.sh
+	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh
 
 publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO}
 	@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
@@ -116,7 +116,6 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
 	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
 

README.md

@@ -37,7 +37,7 @@ not open source.
 
 ## Building
 
-We always require the latest Go release, currently Go 1.22. (While we build
+We always require the latest Go release, currently Go 1.23. (While we build
 releases with our [Go fork](https://github.com/tailscale/go/), its use is not
 required.)
 
@@ -72,7 +72,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
 `Signed-off-by` lines in commits.
 
 See `git log` for our commit message style. It's basically the same as
-[Go's style](https://github.com/golang/go/wiki/CommitMessage).
+[Go's style](https://go.dev/wiki/CommitMessage).
 
 ## About Us
 

VERSION.txt

@@ -1 +1 @@
-1.73.0
+1.79.0

api.md

@@ -1,104 +1,2 @@
 > [!IMPORTANT]
 > The Tailscale API documentation has moved to https://tailscale.com/api
-
-# Tailscale API
-
-The Tailscale API documentation is located in **[tailscale/publicapi](./publicapi/readme.md#tailscale-api)**.
-
-# APIs
-
-**[Overview](./publicapi/readme.md)**
-
-**[Device](./publicapi/device.md#device)**
-
-<a href="device-delete"></a>
-<a href="expire-device-key"></a>
-<a href="device-routes-get">
-<a href="device-routes-post"></a>
-<a href="#device-authorized-post"></a>
-<a href="device-tags-post"></a>
-<a href="device-key-post"></a>
-<a href="tailnet-acl-get"></a>
-
-- Get a device: [`GET /api/v2/device/{deviceid}`](./publicapi/device.md#get-device)
-- Delete a device: [`DELETE /api/v2/device/{deviceID}`](./publicapi/device.md#delete-device)
-- Expire device key: [`POST /api/v2/device/{deviceID}/expire`](./publicapi/device.md#expire-device-key)
-- [**Routes**](./publicapi/device.md#routes)
-  - Get device routes: [`GET /api/v2/device/{deviceID}/routes`](./publicapi/device.md#get-device-routes)
-  - Set device routes: [`POST /api/v2/device/{deviceID}/routes`](./publicapi/device.md#set-device-routes)
-- [**Authorize**](./publicapi/device.md#authorize)
-  - Authorize a device: [`POST /api/v2/device/{deviceID}/authorized`](./publicapi/device.md#authorize-device)
-- [**Tags**](./publicapi/device.md#tags)
-  - Update tags: [`POST /api/v2/device/{deviceID}/tags`](./publicapi/device.md#update-device-tags)
-- [**Keys**](./publicapi/device.md#keys)
-  - Update device key: [`POST /api/v2/device/{deviceID}/key`](./publicapi/device.md#update-device-key)
-- [**IP Addresses**](./publicapi/device.md#ip-addresses)
-  - Set device IPv4 address: [`POST /api/v2/device/{deviceID}/ip`](./publicapi/device.md#set-device-ipv4-address)
-- [**Device posture attributes**](./publicapi/device.md#device-posture-attributes)
-  - Get device posture attributes: [`GET /api/v2/device/{deviceID}/attributes`](./publicapi/device.md#get-device-posture-attributes)
-  - Set custom device posture attributes: [`POST /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#set-device-posture-attributes)
-  - Delete custom device posture attributes: [`DELETE /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#delete-custom-device-posture-attributes)
-- [**Device invites**](./publicapi/device.md#invites-to-a-device)
-  - List device invites: [`GET /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#list-device-invites)
-  - Create device invites: [`POST /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#create-device-invites)
-
-**[Tailnet](./publicapi/tailnet.md#tailnet)**
-
-<a href="tailnet-acl-post"></a>
-<a href="tailnet-acl-preview-post"></a>
-<a href="tailnet-acl-validate-post"></a>
-<a href="tailnet-devices"></a>
-<a href="tailnet-keys-get"></a>
-<a href="tailnet-keys-post"></a>
-<a href="tailnet-keys-key-get"></a>
-<a href="tailnet-keys-key-delete"></a>
-<a href="tailnet-dns"></a>
-<a href="tailnet-dns-nameservers-get"></a>
-<a href="tailnet-dns-nameservers-post"></a>
-<a href="tailnet-dns-preferences-get"></a>
-<a href="tailnet-dns-preferences-post"></a>
-<a href="tailnet-dns-searchpaths-get"></a>
-<a href="tailnet-dns-searchpaths-post"></a>
-
-- [**Policy File**](./publicapi/tailnet.md#policy-file)
-  - Get policy file: [`GET /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#get-policy-file)
-  - Update policy file: [`POST /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#update-policy-file)
-  - Preview rule matches: [`POST /api/v2/tailnet/{tailnet}/acl/preview`](./publicapi/tailnet.md#preview-policy-file-rule-matches)
-  - Validate and test policy file: [`POST /api/v2/tailnet/{tailnet}/acl/validate`](./publicapi/tailnet.md#validate-and-test-policy-file)
-- [**Devices**](./publicapi/tailnet.md#devices)
-  - List tailnet devices: [`GET /api/v2/tailnet/{tailnet}/devices`](./publicapi/tailnet.md#list-tailnet-devices)
-- [**Keys**](./publicapi/tailnet.md#tailnet-keys)
-  - List tailnet keys: [`GET /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#list-tailnet-keys)
-  - Create an auth key: [`POST /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#create-auth-key)
-  - Get a key: [`GET /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#get-key)
-  - Delete a key: [`DELETE /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#delete-key)
-- [**DNS**](./publicapi/tailnet.md#dns)
-  - [**Nameservers**](./publicapi/tailnet.md#nameservers)
-    - Get nameservers: [`GET /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#get-nameservers)
-    - Set nameservers: [`POST /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#set-nameservers)
-  - [**Preferences**](./publicapi/tailnet.md#preferences)
-    - Get DNS preferences: [`GET /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#get-dns-preferences)
-    - Set DNS preferences: [`POST /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#set-dns-preferences)
-  - [**Search Paths**](./publicapi/tailnet.md#search-paths)
-    - Get search paths: [`GET /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#get-search-paths)
-    - Set search paths: [`POST /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#set-search-paths)
-  - [**Split DNS**](./publicapi/tailnet.md#split-dns)
-    - Get split DNS: [`GET /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#get-split-dns)
-    - Update split DNS: [`PATCH /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#update-split-dns)
-    - Set split DNS: [`PUT /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#set-split-dns)
-- [**User invites**](./publicapi/tailnet.md#tailnet-user-invites)
-  - List user invites: [`GET /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#list-user-invites)
-  - Create user invites: [`POST /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#create-user-invites)
-
-**[User invites](./publicapi/userinvites.md#user-invites)**
-
-- Get user invite: [`GET /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#get-user-invite)
-- Delete user invite: [`DELETE /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#delete-user-invite)
-- Resend user invite (by email): [`POST /api/v2/user-invites/{userInviteId}/resend`](#resend-user-invite)
-
-**[Device invites](./publicapi/deviceinvites.md#device-invites)**
-
-- Get device invite: [`GET /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#get-device-invite)
-- Delete device invite: [`DELETE /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#delete-device-invite)
-- Resend device invite (by email): [`POST /api/v2/device-invites/{deviceInviteId}/resend`](./publicapi/deviceinvites.md#resend-device-invite)
-- Accept device invite [`POST /api/v2/device-invites/-/accept`](#accept-device-invite)

appc/appconnector.go

@@ -18,7 +18,6 @@ import (
 	"sync"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
@@ -291,11 +290,11 @@ func (e *AppConnector) updateDomains(domains []string) {
 			}
 		}
 		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
+			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err)
 		}
 	}
 
-	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
+	e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards)
 }
 
 // updateRoutes merges the supplied routes into the currently configured routes. The routes supplied
@@ -354,7 +353,7 @@ func (e *AppConnector) Domains() views.Slice[string] {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	return views.SliceOf(xmaps.Keys(e.domains))
+	return views.SliceOf(slicesx.MapKeys(e.domains))
 }
 
 // DomainRoutes returns a map of domains to resolved IP
@@ -375,13 +374,13 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr {
 // response is being returned over the PeerAPI. The response is parsed and
 // matched against the configured domains, if matched the routeAdvertiser is
 // advised to advertise the discovered route.
-func (e *AppConnector) ObserveDNSResponse(res []byte) {
+func (e *AppConnector) ObserveDNSResponse(res []byte) error {
 	var p dnsmessage.Parser
 	if _, err := p.Start(res); err != nil {
-		return
+		return err
 	}
 	if err := p.SkipAllQuestions(); err != nil {
-		return
+		return err
 	}
 
 	// cnameChain tracks a chain of CNAMEs for a given query in order to reverse
@@ -400,12 +399,12 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			break
 		}
 		if err != nil {
-			return
+			return err
 		}
 
 		if h.Class != dnsmessage.ClassINET {
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 		}
@@ -414,7 +413,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA:
 		default:
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 
@@ -428,7 +427,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		if h.Type == dnsmessage.TypeCNAME {
 			res, err := p.CNAMEResource()
 			if err != nil {
-				return
+				return err
 			}
 			cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".")
 			if len(cname) == 0 {
@@ -442,20 +441,20 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		case dnsmessage.TypeA:
 			r, err := p.AResource()
 			if err != nil {
-				return
+				return err
 			}
 			addr := netip.AddrFrom4(r.A)
 			mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
 		case dnsmessage.TypeAAAA:
 			r, err := p.AAAAResource()
 			if err != nil {
-				return
+				return err
 			}
 			addr := netip.AddrFrom16(r.AAAA)
 			mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
 		default:
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 		}
@@ -486,6 +485,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			e.scheduleAdvertisement(domain, toAdvertise...)
 		}
 	}
+	return nil
 }
 
 // starting from the given domain that resolved to an address, find it, or any

appc/appconnector_test.go

@@ -11,13 +11,13 @@ import (
 	"testing"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
 	"tailscale.com/tstest"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
+	"tailscale.com/util/slicesx"
 )
 
 func fakeStoreRoutes(*RouteInfo) error { return nil }
@@ -50,7 +50,7 @@ func TestUpdateDomains(t *testing.T) {
 		// domains are explicitly downcased on set.
 		a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
 		a.Wait(ctx)
-		if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
+		if got, want := slicesx.MapKeys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
 	}
@@ -69,7 +69,9 @@ func TestUpdateRoutes(t *testing.T) {
 		a.updateDomains([]string{"*.example.com"})
 
 		// This route should be collapsed into the range
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 
 		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
@@ -77,7 +79,9 @@ func TestUpdateRoutes(t *testing.T) {
 		}
 
 		// This route should not be collapsed or removed
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 
 		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
@@ -130,7 +134,9 @@ func TestDomainRoutes(t *testing.T) {
 			a = NewAppConnector(t.Logf, rc, nil, nil)
 		}
 		a.updateDomains([]string{"example.com"})
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(context.Background())
 
 		want := map[string][]netip.Addr{
@@ -155,7 +161,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		}
 
 		// a has no domains configured, so it should not advertise any routes
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
@@ -163,7 +171,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
 		a.updateDomains([]string{"example.com"})
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
@@ -172,7 +182,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		// a CNAME record chain should result in a route being added if the chain
 		// matches a routed domain.
 		a.updateDomains([]string{"www.example.com", "example.com"})
-		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com.")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -181,7 +193,9 @@ func TestObserveDNSResponse(t *testing.T) {
 
 		// a CNAME record chain should result in a route being added if the chain
 		// even if only found in the middle of the chain
-		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org.")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -190,14 +204,18 @@ func TestObserveDNSResponse(t *testing.T) {
 
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
 
 		// don't re-advertise routes that have already been advertised
-		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if !slices.Equal(rc.Routes(), wantRoutes) {
 			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -207,7 +225,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		pfx := netip.MustParsePrefix("192.0.2.0/24")
 		a.updateRoutes([]netip.Prefix{pfx})
 		wantRoutes = append(wantRoutes, pfx)
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if !slices.Equal(rc.Routes(), wantRoutes) {
 			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -230,7 +250,9 @@ func TestWildcardDomains(t *testing.T) {
 		}
 
 		a.updateDomains([]string{"*.example.com"})
-		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
 			t.Errorf("routes: got %v; want %v", got, want)
@@ -438,10 +460,16 @@ func TestUpdateDomainRouteRemoval(t *testing.T) {
 		// adding domains doesn't immediately cause any routes to be advertised
 		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
 
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
+		for _, res := range [][]byte{
+			dnsResponse("a.example.com.", "1.2.3.1"),
+			dnsResponse("a.example.com.", "1.2.3.2"),
+			dnsResponse("b.example.com.", "1.2.3.3"),
+			dnsResponse("b.example.com.", "1.2.3.4"),
+		} {
+			if err := a.ObserveDNSResponse(res); err != nil {
+				t.Errorf("ObserveDNSResponse: %v", err)
+			}
+		}
 		a.Wait(ctx)
 		// observing dns responses causes routes to be advertised
 		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
@@ -487,10 +515,16 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) {
 		// adding domains doesn't immediately cause any routes to be advertised
 		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
 
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
-		a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
-		a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
+		for _, res := range [][]byte{
+			dnsResponse("a.example.com.", "1.2.3.1"),
+			dnsResponse("a.example.com.", "1.2.3.2"),
+			dnsResponse("1.b.example.com.", "1.2.3.3"),
+			dnsResponse("2.b.example.com.", "1.2.3.4"),
+		} {
+			if err := a.ObserveDNSResponse(res); err != nil {
+				t.Errorf("ObserveDNSResponse: %v", err)
+			}
+		}
 		a.Wait(ctx)
 		// observing dns responses causes routes to be advertised
 		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})

assert_ts_toolchain_match.go

@@ -0,0 +1,27 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build tailscale_go
+
+package tailscaleroot
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func init() {
+	tsRev, ok := tailscaleToolchainRev()
+	if !ok {
+		panic("binary built with tailscale_go build tag but failed to read build info or find tailscale.toolchain.rev in build info")
+	}
+	want := strings.TrimSpace(GoToolchainRev)
+	if tsRev != want {
+		if os.Getenv("TS_PERMIT_TOOLCHAIN_MISMATCH") == "1" {
+			fmt.Fprintf(os.Stderr, "tailscale.toolchain.rev = %q, want %q; but ignoring due to TS_PERMIT_TOOLCHAIN_MISMATCH=1\n", tsRev, want)
+			return
+		}
+		panic(fmt.Sprintf("binary built with tailscale_go build tag but Go toolchain %q doesn't match github.com/tailscale/tailscale expected value %q; override this failure with TS_PERMIT_TOOLCHAIN_MISMATCH=1", tsRev, want))
+	}
+}

atomicfile/atomicfile.go

@@ -15,8 +15,9 @@ import (
 )
 
 // WriteFile writes data to filename+some suffix, then renames it into filename.
-// The perm argument is ignored on Windows. If the target filename already
-// exists but is not a regular file, WriteFile returns an error.
+// The perm argument is ignored on Windows, but if the target filename already
+// exists then the target file's attributes and ACLs are preserved. If the target
+// filename already exists but is not a regular file, WriteFile returns an error.
 func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
 	fi, err := os.Stat(filename)
 	if err == nil && !fi.Mode().IsRegular() {
@@ -47,5 +48,5 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
 	if err := f.Close(); err != nil {
 		return err
 	}
-	return os.Rename(tmpName, filename)
+	return rename(tmpName, filename)
 }

atomicfile/atomicfile_notwindows.go

@@ -0,0 +1,14 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !windows
+
+package atomicfile
+
+import (
+	"os"
+)
+
+func rename(srcFile, destFile string) error {
+	return os.Rename(srcFile, destFile)
+}

atomicfile/atomicfile_windows.go

@@ -0,0 +1,33 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package atomicfile
+
+import (
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+func rename(srcFile, destFile string) error {
+	// Use replaceFile when possible to preserve the original file's attributes and ACLs.
+	if err := replaceFile(destFile, srcFile); err == nil || err != windows.ERROR_FILE_NOT_FOUND {
+		return err
+	}
+	// destFile doesn't exist. Just do a normal rename.
+	return os.Rename(srcFile, destFile)
+}
+
+func replaceFile(destFile, srcFile string) error {
+	destFile16, err := windows.UTF16PtrFromString(destFile)
+	if err != nil {
+		return err
+	}
+
+	srcFile16, err := windows.UTF16PtrFromString(srcFile)
+	if err != nil {
+		return err
+	}
+
+	return replaceFileW(destFile16, srcFile16, nil, 0, nil, nil)
+}

atomicfile/atomicfile_windows_test.go

@@ -0,0 +1,146 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package atomicfile
+
+import (
+	"os"
+	"testing"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _SECURITY_RESOURCE_MANAGER_AUTHORITY = windows.SidIdentifierAuthority{[6]byte{0, 0, 0, 0, 0, 9}}
+
+// makeRandomSID generates a SID derived from a v4 GUID.
+// This is basically the same algorithm used by browser sandboxes for generating
+// random SIDs.
+func makeRandomSID() (*windows.SID, error) {
+	guid, err := windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+
+	rids := *((*[4]uint32)(unsafe.Pointer(&guid)))
+
+	var pSID *windows.SID
+	if err := windows.AllocateAndInitializeSid(&_SECURITY_RESOURCE_MANAGER_AUTHORITY, 4, rids[0], rids[1], rids[2], rids[3], 0, 0, 0, 0, &pSID); err != nil {
+		return nil, err
+	}
+	defer windows.FreeSid(pSID)
+
+	// Make a copy that lives on the Go heap
+	return pSID.Copy()
+}
+
+func getExistingFileSD(name string) (*windows.SECURITY_DESCRIPTOR, error) {
+	const infoFlags = windows.DACL_SECURITY_INFORMATION
+	return windows.GetNamedSecurityInfo(name, windows.SE_FILE_OBJECT, infoFlags)
+}
+
+func getExistingFileDACL(name string) (*windows.ACL, error) {
+	sd, err := getExistingFileSD(name)
+	if err != nil {
+		return nil, err
+	}
+
+	dacl, _, err := sd.DACL()
+	return dacl, err
+}
+
+func addDenyACEForRandomSID(dacl *windows.ACL) (*windows.ACL, error) {
+	randomSID, err := makeRandomSID()
+	if err != nil {
+		return nil, err
+	}
+
+	randomSIDTrustee := windows.TRUSTEE{nil, windows.NO_MULTIPLE_TRUSTEE,
+		windows.TRUSTEE_IS_SID, windows.TRUSTEE_IS_UNKNOWN,
+		windows.TrusteeValueFromSID(randomSID)}
+
+	entries := []windows.EXPLICIT_ACCESS{
+		{
+			windows.GENERIC_ALL,
+			windows.DENY_ACCESS,
+			windows.NO_INHERITANCE,
+			randomSIDTrustee,
+		},
+	}
+
+	return windows.ACLFromEntries(entries, dacl)
+}
+
+func setExistingFileDACL(name string, dacl *windows.ACL) error {
+	return windows.SetNamedSecurityInfo(name, windows.SE_FILE_OBJECT,
+		windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)
+}
+
+// makeOrigFileWithCustomDACL creates a new, temporary file with a custom
+// DACL that we can check for later. It returns the name of the temporary
+// file and the security descriptor for the file in SDDL format.
+func makeOrigFileWithCustomDACL() (name, sddl string, err error) {
+	f, err := os.CreateTemp("", "foo*.tmp")
+	if err != nil {
+		return "", "", err
+	}
+	name = f.Name()
+	if err := f.Close(); err != nil {
+		return "", "", err
+	}
+	f = nil
+	defer func() {
+		if err != nil {
+			os.Remove(name)
+		}
+	}()
+
+	dacl, err := getExistingFileDACL(name)
+	if err != nil {
+		return "", "", err
+	}
+
+	// Add a harmless, deny-only ACE for a random SID that isn't used for anything
+	// (but that we can check for later).
+	dacl, err = addDenyACEForRandomSID(dacl)
+	if err != nil {
+		return "", "", err
+	}
+
+	if err := setExistingFileDACL(name, dacl); err != nil {
+		return "", "", err
+	}
+
+	sd, err := getExistingFileSD(name)
+	if err != nil {
+		return "", "", err
+	}
+
+	return name, sd.String(), nil
+}
+
+func TestPreserveSecurityInfo(t *testing.T) {
+	// Make a test file with a custom ACL.
+	origFileName, want, err := makeOrigFileWithCustomDACL()
+	if err != nil {
+		t.Fatalf("makeOrigFileWithCustomDACL returned %v", err)
+	}
+	t.Cleanup(func() {
+		os.Remove(origFileName)
+	})
+
+	if err := WriteFile(origFileName, []byte{}, 0); err != nil {
+		t.Fatalf("WriteFile returned %v", err)
+	}
+
+	// We expect origFileName's security descriptor to be unchanged despite
+	// the WriteFile call.
+	sd, err := getExistingFileSD(origFileName)
+	if err != nil {
+		t.Fatalf("getExistingFileSD(%q) returned %v", origFileName, err)
+	}
+
+	if got := sd.String(); got != want {
+		t.Errorf("security descriptor comparison failed: got %q, want %q", got, want)
+	}
+}

atomicfile/mksyscall.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package atomicfile
+
+//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go mksyscall.go
+
+//sys replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) [int32(failretval)==0] = kernel32.ReplaceFileW

atomicfile/zsyscall_windows.go

@@ -0,0 +1,52 @@
+// Code generated by 'go generate'; DO NOT EDIT.
+
+package atomicfile
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return errERROR_EINVAL
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
+
+	procReplaceFileW = modkernel32.NewProc("ReplaceFileW")
+)
+
+func replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) {
+	r1, _, e1 := syscall.Syscall6(procReplaceFileW.Addr(), 6, uintptr(unsafe.Pointer(replaced)), uintptr(unsafe.Pointer(replacement)), uintptr(unsafe.Pointer(backup)), uintptr(flags), uintptr(exclude), uintptr(reserved))
+	if int32(r1) == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}

build_dist.sh

@@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do
 	--extra-small)
 		shift
 		ldflags="$ldflags -w -s"
-		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion"
+		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan"
 		;;
 	--box)
 		shift

build_docker.sh

@@ -17,12 +17,20 @@ eval "$(./build_dist.sh shellvars)"
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
 DEFAULT_BASE="tailscale/alpine-base:3.18"
+# Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
+# Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
+# annotations defined here will be overriden by release scripts that call this script.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+DEFAULT_ANNOTATIONS="org.opencontainers.image.source=https://github.com/tailscale/tailscale/blob/main/build_docker.sh,org.opencontainers.image.vendor=Tailscale"
 
 PUSH="${PUSH:-false}"
 TARGET="${TARGET:-${DEFAULT_TARGET}}"
 TAGS="${TAGS:-${DEFAULT_TAGS}}"
 BASE="${BASE:-${DEFAULT_BASE}}"
 PLATFORM="${PLATFORM:-}" # default to all platforms
+# OCI annotations that will be added to the image.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
+ANNOTATIONS="${ANNOTATIONS:-${DEFAULT_ANNOTATIONS}}"
 
 case "$TARGET" in
   client)
@@ -43,9 +51,10 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/containerboot
     ;;
-  operator)
+  k8s-operator)
     DEFAULT_REPOS="tailscale/k8s-operator"
     REPOS="${REPOS:-${DEFAULT_REPOS}}"
     go run github.com/tailscale/mkctr \
@@ -56,9 +65,11 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/operator
     ;;
   k8s-nameserver)
@@ -72,9 +83,11 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/k8s-nameserver
     ;;
   *)

client/systray/logo.go

@@ -0,0 +1,319 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+package systray
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"image/color"
+	"image/png"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/fogleman/gg"
+)
+
+// tsLogo represents the Tailscale logo displayed as the systray icon.
+type tsLogo struct {
+	// dots represents the state of the 3x3 dot grid in the logo.
+	// A 0 represents a gray dot, any other value is a white dot.
+	dots [9]byte
+
+	// dotMask returns an image mask to be used when rendering the logo dots.
+	dotMask func(dc *gg.Context, borderUnits int, radius int) *image.Alpha
+
+	// overlay is called after the dots are rendered to draw an additional overlay.
+	overlay func(dc *gg.Context, borderUnits int, radius int)
+}
+
+var (
+	// disconnected is all gray dots
+	disconnected = tsLogo{dots: [9]byte{
+		0, 0, 0,
+		0, 0, 0,
+		0, 0, 0,
+	}}
+
+	// connected is the normal Tailscale logo
+	connected = tsLogo{dots: [9]byte{
+		0, 0, 0,
+		1, 1, 1,
+		0, 1, 0,
+	}}
+
+	// loading is a special tsLogo value that is not meant to be rendered directly,
+	// but indicates that the loading animation should be shown.
+	loading = tsLogo{dots: [9]byte{'l', 'o', 'a', 'd', 'i', 'n', 'g'}}
+
+	// loadingIcons are shown in sequence as an animated loading icon.
+	loadingLogos = []tsLogo{
+		{dots: [9]byte{
+			0, 1, 1,
+			1, 0, 1,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 1, 1,
+			0, 0, 1,
+			0, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 1, 1,
+			0, 0, 0,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 0, 1,
+			0, 1, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 1, 0,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 1,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 1,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			1, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			1, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 0, 0,
+			1, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 1,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 1, 0,
+			0, 1, 1,
+			1, 0, 1,
+		}},
+	}
+
+	// exitNodeOnline is the Tailscale logo with an additional arrow overlay in the corner.
+	exitNodeOnline = tsLogo{
+		dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 1, 0,
+		},
+		// draw an arrow mask in the bottom right corner with a reasonably thick line width.
+		dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 3.5)
+			y := r * (bu + 7)
+			x2 := x1 + (r * 5)
+
+			mc := gg.NewContext(dc.Width(), dc.Height())
+			mc.DrawLine(x1, y, x2, y)                 // arrow center line
+			mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
+			mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
+			mc.SetLineWidth(r * 3)
+			mc.Stroke()
+			return mc.AsMask()
+		},
+		// draw an arrow in the bottom right corner over the masked area.
+		overlay: func(dc *gg.Context, borderUnits int, radius int) {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 3.5)
+			y := r * (bu + 7)
+			x2 := x1 + (r * 5)
+
+			dc.DrawLine(x1, y, x2, y)                 // arrow center line
+			dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
+			dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
+			dc.SetColor(fg)
+			dc.SetLineWidth(r)
+			dc.Stroke()
+		},
+	}
+
+	// exitNodeOffline is the Tailscale logo with a red "x" in the corner.
+	exitNodeOffline = tsLogo{
+		dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 1, 0,
+		},
+		// Draw a square that hides the four dots in the bottom right corner,
+		dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
+			bu, r := float64(borderUnits), float64(radius)
+			x := r * (bu + 3)
+
+			mc := gg.NewContext(dc.Width(), dc.Height())
+			mc.DrawRectangle(x, x, r*6, r*6)
+			mc.Fill()
+			return mc.AsMask()
+		},
+		// draw a red "x" over the bottom right corner.
+		overlay: func(dc *gg.Context, borderUnits int, radius int) {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 4)
+			x2 := x1 + (r * 3.5)
+			dc.DrawLine(x1, x1, x2, x2) // top-left to bottom-right stroke
+			dc.DrawLine(x1, x2, x2, x1) // bottom-left to top-right stroke
+			dc.SetColor(red)
+			dc.SetLineWidth(r)
+			dc.Stroke()
+		},
+	}
+)
+
+var (
+	bg   = color.NRGBA{0, 0, 0, 255}
+	fg   = color.NRGBA{255, 255, 255, 255}
+	gray = color.NRGBA{255, 255, 255, 102}
+	red  = color.NRGBA{229, 111, 74, 255}
+)
+
+// render returns a PNG image of the logo.
+func (logo tsLogo) render() *bytes.Buffer {
+	const borderUnits = 1
+	return logo.renderWithBorder(borderUnits)
+}
+
+// renderWithBorder returns a PNG image of the logo with the specified border width.
+// One border unit is equal to the radius of a tailscale logo dot.
+func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer {
+	const radius = 25
+	dim := radius * (8 + borderUnits*2)
+
+	dc := gg.NewContext(dim, dim)
+	dc.DrawRectangle(0, 0, float64(dim), float64(dim))
+	dc.SetColor(bg)
+	dc.Fill()
+
+	if logo.dotMask != nil {
+		mask := logo.dotMask(dc, borderUnits, radius)
+		dc.SetMask(mask)
+		dc.InvertMask()
+	}
+
+	for y := 0; y < 3; y++ {
+		for x := 0; x < 3; x++ {
+			px := (borderUnits + 1 + 3*x) * radius
+			py := (borderUnits + 1 + 3*y) * radius
+			col := fg
+			if logo.dots[y*3+x] == 0 {
+				col = gray
+			}
+			dc.DrawCircle(float64(px), float64(py), radius)
+			dc.SetColor(col)
+			dc.Fill()
+		}
+	}
+
+	if logo.overlay != nil {
+		dc.ResetClip()
+		logo.overlay(dc, borderUnits, radius)
+	}
+
+	b := bytes.NewBuffer(nil)
+	png.Encode(b, dc.Image())
+	return b
+}
+
+// setAppIcon renders logo and sets it as the systray icon.
+func setAppIcon(icon tsLogo) {
+	if icon.dots == loading.dots {
+		startLoadingAnimation()
+	} else {
+		stopLoadingAnimation()
+		systray.SetIcon(icon.render().Bytes())
+	}
+}
+
+var (
+	loadingMu sync.Mutex // protects loadingCancel
+
+	// loadingCancel stops the loading animation in the systray icon.
+	// This is nil if the animation is not currently active.
+	loadingCancel func()
+)
+
+// startLoadingAnimation starts the animated loading icon in the system tray.
+// The animation continues until [stopLoadingAnimation] is called.
+// If the loading animation is already active, this func does nothing.
+func startLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		// loading icon already displayed
+		return
+	}
+
+	ctx := context.Background()
+	ctx, loadingCancel = context.WithCancel(ctx)
+
+	go func() {
+		t := time.NewTicker(500 * time.Millisecond)
+		var i int
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				systray.SetIcon(loadingLogos[i].render().Bytes())
+				i++
+				if i >= len(loadingLogos) {
+					i = 0
+				}
+			}
+		}
+	}()
+}
+
+// stopLoadingAnimation stops the animated loading icon in the system tray.
+// If the loading animation is not currently active, this func does nothing.
+func stopLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		loadingCancel()
+		loadingCancel = nil
+	}
+}

client/systray/systray.go

@@ -0,0 +1,712 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+// Package systray provides a minimal Tailscale systray application.
+package systray
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/atotto/clipboard"
+	dbus "github.com/godbus/dbus/v5"
+	"github.com/toqueteos/webbrowser"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/slicesx"
+	"tailscale.com/util/stringsx"
+)
+
+var (
+	// newMenuDelay is the amount of time to sleep after creating a new menu,
+	// but before adding items to it. This works around a bug in some dbus implementations.
+	newMenuDelay time.Duration
+
+	// if true, treat all mullvad exit node countries as single-city.
+	// Instead of rendering a submenu with cities, just select the highest-priority peer.
+	hideMullvadCities bool
+)
+
+// Run starts the systray menu and blocks until the menu exits.
+func (menu *Menu) Run() {
+	menu.updateState()
+
+	// exit cleanly on SIGINT and SIGTERM
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		select {
+		case <-interrupt:
+			menu.onExit()
+		case <-menu.bgCtx.Done():
+		}
+	}()
+	go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1)
+
+	systray.Run(menu.onReady, menu.onExit)
+}
+
+// Menu represents the systray menu, its items, and the current Tailscale state.
+type Menu struct {
+	mu sync.Mutex // protects the entire Menu
+
+	lc          tailscale.LocalClient
+	status      *ipnstate.Status
+	curProfile  ipn.LoginProfile
+	allProfiles []ipn.LoginProfile
+
+	bgCtx    context.Context // ctx for background tasks not involving menu item clicks
+	bgCancel context.CancelFunc
+
+	// Top-level menu items
+	connect    *systray.MenuItem
+	disconnect *systray.MenuItem
+	self       *systray.MenuItem
+	exitNodes  *systray.MenuItem
+	more       *systray.MenuItem
+	quit       *systray.MenuItem
+
+	rebuildCh  chan struct{} // triggers a menu rebuild
+	accountsCh chan ipn.ProfileID
+	exitNodeCh chan tailcfg.StableNodeID // ID of selected exit node
+
+	eventCancel context.CancelFunc // cancel eventLoop
+
+	notificationIcon *os.File // icon used for desktop notifications
+}
+
+func (menu *Menu) init() {
+	if menu.bgCtx != nil {
+		// already initialized
+		return
+	}
+
+	menu.rebuildCh = make(chan struct{}, 1)
+	menu.accountsCh = make(chan ipn.ProfileID)
+	menu.exitNodeCh = make(chan tailcfg.StableNodeID)
+
+	// dbus wants a file path for notification icons, so copy to a temp file.
+	menu.notificationIcon, _ = os.CreateTemp("", "tailscale-systray.png")
+	io.Copy(menu.notificationIcon, connected.renderWithBorder(3))
+
+	menu.bgCtx, menu.bgCancel = context.WithCancel(context.Background())
+	go menu.watchIPNBus()
+}
+
+func init() {
+	if runtime.GOOS != "linux" {
+		// so far, these tweaks are only needed on Linux
+		return
+	}
+
+	desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP"))
+	switch desktop {
+	case "gnome":
+		// GNOME expands submenus downward in the main menu, rather than flyouts to the side.
+		// Either as a result of that or another limitation, there seems to be a maximum depth of submenus.
+		// Mullvad countries that have a city submenu are not being rendered, and so can't be selected.
+		// Handle this by simply treating all mullvad countries as single-city and select the best peer.
+		hideMullvadCities = true
+	case "kde":
+		// KDE doesn't need a delay, and actually won't render submenus
+		// if we delay for more than about 400µs.
+		newMenuDelay = 0
+	default:
+		// Add a slight delay to ensure the menu is created before adding items.
+		//
+		// Systray implementations that use libdbusmenu sometimes process messages out of order,
+		// resulting in errors such as:
+		//    (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu'
+		//
+		// See also: https://github.com/fyne-io/systray/issues/12
+		newMenuDelay = 10 * time.Millisecond
+	}
+}
+
+// onReady is called by the systray package when the menu is ready to be built.
+func (menu *Menu) onReady() {
+	log.Printf("starting")
+	setAppIcon(disconnected)
+	menu.rebuild()
+}
+
+// updateState updates the Menu state from the Tailscale local client.
+func (menu *Menu) updateState() {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+	menu.init()
+
+	var err error
+	menu.status, err = menu.lc.Status(menu.bgCtx)
+	if err != nil {
+		log.Print(err)
+	}
+	menu.curProfile, menu.allProfiles, err = menu.lc.ProfileStatus(menu.bgCtx)
+	if err != nil {
+		log.Print(err)
+	}
+}
+
+// rebuild the systray menu based on the current Tailscale state.
+//
+// We currently rebuild the entire menu because it is not easy to update the existing menu.
+// You cannot iterate over the items in a menu, nor can you remove some items like separators.
+// So for now we rebuild the whole thing, and can optimize this later if needed.
+func (menu *Menu) rebuild() {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+	menu.init()
+
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+	ctx := context.Background()
+	ctx, menu.eventCancel = context.WithCancel(ctx)
+
+	systray.ResetMenu()
+
+	menu.connect = systray.AddMenuItem("Connect", "")
+	menu.disconnect = systray.AddMenuItem("Disconnect", "")
+	menu.disconnect.Hide()
+	systray.AddSeparator()
+
+	// delay to prevent race setting icon on first start
+	time.Sleep(newMenuDelay)
+
+	// Set systray menu icon and title.
+	// Also adjust connect/disconnect menu items if needed.
+	var backendState string
+	if menu.status != nil {
+		backendState = menu.status.BackendState
+	}
+	switch backendState {
+	case ipn.Running.String():
+		if menu.status.ExitNodeStatus != nil && !menu.status.ExitNodeStatus.ID.IsZero() {
+			if menu.status.ExitNodeStatus.Online {
+				setTooltip("Using exit node")
+				setAppIcon(exitNodeOnline)
+			} else {
+				setTooltip("Exit node offline")
+				setAppIcon(exitNodeOffline)
+			}
+		} else {
+			setTooltip(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name))
+			setAppIcon(connected)
+		}
+		menu.connect.SetTitle("Connected")
+		menu.connect.Disable()
+		menu.disconnect.Show()
+		menu.disconnect.Enable()
+	case ipn.Starting.String():
+		setTooltip("Connecting")
+		setAppIcon(loading)
+	default:
+		setTooltip("Disconnected")
+		setAppIcon(disconnected)
+	}
+
+	account := "Account"
+	if pt := profileTitle(menu.curProfile); pt != "" {
+		account = pt
+	}
+	accounts := systray.AddMenuItem(account, "")
+	setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL)
+	time.Sleep(newMenuDelay)
+	for _, profile := range menu.allProfiles {
+		title := profileTitle(profile)
+		var item *systray.MenuItem
+		if profile.ID == menu.curProfile.ID {
+			item = accounts.AddSubMenuItemCheckbox(title, "", true)
+		} else {
+			item = accounts.AddSubMenuItem(title, "")
+		}
+		setRemoteIcon(item, profile.UserProfile.ProfilePicURL)
+		onClick(ctx, item, func(ctx context.Context) {
+			select {
+			case <-ctx.Done():
+			case menu.accountsCh <- profile.ID:
+			}
+		})
+	}
+
+	if menu.status != nil && menu.status.Self != nil && len(menu.status.Self.TailscaleIPs) > 0 {
+		title := fmt.Sprintf("This Device: %s (%s)", menu.status.Self.HostName, menu.status.Self.TailscaleIPs[0])
+		menu.self = systray.AddMenuItem(title, "")
+	} else {
+		menu.self = systray.AddMenuItem("This Device: not connected", "")
+		menu.self.Disable()
+	}
+	systray.AddSeparator()
+
+	menu.rebuildExitNodeMenu(ctx)
+
+	if menu.status != nil {
+		menu.more = systray.AddMenuItem("More settings", "")
+		onClick(ctx, menu.more, func(_ context.Context) {
+			webbrowser.Open("http://100.100.100.100/")
+		})
+	}
+
+	menu.quit = systray.AddMenuItem("Quit", "Quit the app")
+	menu.quit.Enable()
+
+	go menu.eventLoop(ctx)
+}
+
+// profileTitle returns the title string for a profile menu item.
+func profileTitle(profile ipn.LoginProfile) string {
+	title := profile.Name
+	if profile.NetworkProfile.DomainName != "" {
+		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+			// windows and mac don't support multi-line menu
+			title += " (" + profile.NetworkProfile.DomainName + ")"
+		} else {
+			title += "\n" + profile.NetworkProfile.DomainName
+		}
+	}
+	return title
+}
+
+var (
+	cacheMu   sync.Mutex
+	httpCache = map[string][]byte{} // URL => response body
+)
+
+// setRemoteIcon sets the icon for menu to the specified remote image.
+// Remote images are fetched as needed and cached.
+func setRemoteIcon(menu *systray.MenuItem, urlStr string) {
+	if menu == nil || urlStr == "" {
+		return
+	}
+
+	cacheMu.Lock()
+	b, ok := httpCache[urlStr]
+	if !ok {
+		resp, err := http.Get(urlStr)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			b, _ = io.ReadAll(resp.Body)
+			httpCache[urlStr] = b
+			resp.Body.Close()
+		}
+	}
+	cacheMu.Unlock()
+
+	if len(b) > 0 {
+		menu.SetIcon(b)
+	}
+}
+
+// setTooltip sets the tooltip text for the systray icon.
+func setTooltip(text string) {
+	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+		systray.SetTooltip(text)
+	} else {
+		// on Linux, SetTitle actually sets the tooltip
+		systray.SetTitle(text)
+	}
+}
+
+// eventLoop is the main event loop for handling click events on menu items
+// and responding to Tailscale state changes.
+// This method does not return until ctx.Done is closed.
+func (menu *Menu) eventLoop(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-menu.rebuildCh:
+			menu.updateState()
+			menu.rebuild()
+		case <-menu.connect.ClickedCh:
+			_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: true,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("error connecting: %v", err)
+			}
+
+		case <-menu.disconnect.ClickedCh:
+			_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: false,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("error disconnecting: %v", err)
+			}
+
+		case <-menu.self.ClickedCh:
+			menu.copyTailscaleIP(menu.status.Self)
+
+		case id := <-menu.accountsCh:
+			if err := menu.lc.SwitchProfile(ctx, id); err != nil {
+				log.Printf("error switching to profile ID %v: %v", id, err)
+			}
+
+		case exitNode := <-menu.exitNodeCh:
+			if exitNode.IsZero() {
+				log.Print("disable exit node")
+				if err := menu.lc.SetUseExitNode(ctx, false); err != nil {
+					log.Printf("error disabling exit node: %v", err)
+				}
+			} else {
+				log.Printf("enable exit node: %v", exitNode)
+				mp := &ipn.MaskedPrefs{
+					Prefs: ipn.Prefs{
+						ExitNodeID: exitNode,
+					},
+					ExitNodeIDSet: true,
+				}
+				if _, err := menu.lc.EditPrefs(ctx, mp); err != nil {
+					log.Printf("error setting exit node: %v", err)
+				}
+			}
+
+		case <-menu.quit.ClickedCh:
+			systray.Quit()
+		}
+	}
+}
+
+// onClick registers a click handler for a menu item.
+func onClick(ctx context.Context, item *systray.MenuItem, fn func(ctx context.Context)) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-item.ClickedCh:
+				fn(ctx)
+			}
+		}
+	}()
+}
+
+// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
+// This method does not return.
+func (menu *Menu) watchIPNBus() {
+	for {
+		if err := menu.watchIPNBusInner(); err != nil {
+			log.Println(err)
+			if errors.Is(err, context.Canceled) {
+				// If the context got canceled, we will never be able to
+				// reconnect to IPN bus, so exit the process.
+				log.Fatalf("watchIPNBus: %v", err)
+			}
+		}
+		// If our watch connection breaks, wait a bit before reconnecting. No
+		// reason to spam the logs if e.g. tailscaled is restarting or goes
+		// down.
+		time.Sleep(3 * time.Second)
+	}
+}
+
+func (menu *Menu) watchIPNBusInner() error {
+	watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return fmt.Errorf("watching ipn bus: %w", err)
+	}
+	defer watcher.Close()
+	for {
+		select {
+		case <-menu.bgCtx.Done():
+			return nil
+		default:
+			n, err := watcher.Next()
+			if err != nil {
+				return fmt.Errorf("ipnbus error: %w", err)
+			}
+			var rebuild bool
+			if n.State != nil {
+				log.Printf("new state: %v", n.State)
+				rebuild = true
+			}
+			if n.Prefs != nil {
+				rebuild = true
+			}
+			if rebuild {
+				menu.rebuildCh <- struct{}{}
+			}
+		}
+	}
+}
+
+// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
+// and sends a notification with the copied value.
+func (menu *Menu) copyTailscaleIP(device *ipnstate.PeerStatus) {
+	if device == nil || len(device.TailscaleIPs) == 0 {
+		return
+	}
+	name := strings.Split(device.DNSName, ".")[0]
+	ip := device.TailscaleIPs[0].String()
+	err := clipboard.WriteAll(ip)
+	if err != nil {
+		log.Printf("clipboard error: %v", err)
+	}
+
+	menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
+}
+
+// sendNotification sends a desktop notification with the given title and content.
+func (menu *Menu) sendNotification(title, content string) {
+	conn, err := dbus.SessionBus()
+	if err != nil {
+		log.Printf("dbus: %v", err)
+		return
+	}
+	timeout := 3 * time.Second
+	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
+	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
+		menu.notificationIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
+	if call.Err != nil {
+		log.Printf("dbus: %v", call.Err)
+	}
+}
+
+func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) {
+	if menu.status == nil {
+		return
+	}
+
+	status := menu.status
+	menu.exitNodes = systray.AddMenuItem("Exit Nodes", "")
+	time.Sleep(newMenuDelay)
+
+	// register a click handler for a menu item to set nodeID as the exit node.
+	setExitNodeOnClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) {
+		onClick(ctx, item, func(ctx context.Context) {
+			select {
+			case <-ctx.Done():
+			case menu.exitNodeCh <- nodeID:
+			}
+		})
+	}
+
+	noExitNodeMenu := menu.exitNodes.AddSubMenuItemCheckbox("None", "", status.ExitNodeStatus == nil)
+	setExitNodeOnClick(noExitNodeMenu, "")
+
+	// Show recommended exit node if available.
+	if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) {
+		sugg, err := menu.lc.SuggestExitNode(ctx)
+		if err == nil {
+			title := "Recommended: "
+			if loc := sugg.Location; loc.Valid() && loc.Country() != "" {
+				flag := countryFlag(loc.CountryCode())
+				title += fmt.Sprintf("%s %s: %s", flag, loc.Country(), loc.City())
+			} else {
+				title += strings.Split(sugg.Name, ".")[0]
+			}
+			menu.exitNodes.AddSeparator()
+			rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false)
+			setExitNodeOnClick(rm, sugg.ID)
+			if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID {
+				rm.Check()
+			}
+		}
+	}
+
+	// Add tailnet exit nodes if present.
+	var tailnetExitNodes []*ipnstate.PeerStatus
+	for _, ps := range status.Peer {
+		if ps.ExitNodeOption && ps.Location == nil {
+			tailnetExitNodes = append(tailnetExitNodes, ps)
+		}
+	}
+	if len(tailnetExitNodes) > 0 {
+		menu.exitNodes.AddSeparator()
+		menu.exitNodes.AddSubMenuItem("Tailnet Exit Nodes", "").Disable()
+		for _, ps := range status.Peer {
+			if !ps.ExitNodeOption || ps.Location != nil {
+				continue
+			}
+			name := strings.Split(ps.DNSName, ".")[0]
+			if !ps.Online {
+				name += " (offline)"
+			}
+			sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false)
+			if !ps.Online {
+				sm.Disable()
+			}
+			if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID {
+				sm.Check()
+			}
+			setExitNodeOnClick(sm, ps.ID)
+		}
+	}
+
+	// Add mullvad exit nodes if present.
+	var mullvadExitNodes mullvadPeers
+	if status.Self.CapMap.Contains("mullvad") {
+		mullvadExitNodes = newMullvadPeers(status)
+	}
+	if len(mullvadExitNodes.countries) > 0 {
+		menu.exitNodes.AddSeparator()
+		menu.exitNodes.AddSubMenuItem("Location-based Exit Nodes", "").Disable()
+		mullvadMenu := menu.exitNodes.AddSubMenuItemCheckbox("Mullvad VPN", "", false)
+
+		for _, country := range mullvadExitNodes.sortedCountries() {
+			flag := countryFlag(country.code)
+			countryMenu := mullvadMenu.AddSubMenuItemCheckbox(flag+" "+country.name, "", false)
+
+			// single-city country, no submenu
+			if len(country.cities) == 1 || hideMullvadCities {
+				setExitNodeOnClick(countryMenu, country.best.ID)
+				if status.ExitNodeStatus != nil {
+					for _, city := range country.cities {
+						for _, ps := range city.peers {
+							if status.ExitNodeStatus.ID == ps.ID {
+								mullvadMenu.Check()
+								countryMenu.Check()
+							}
+						}
+					}
+				}
+				continue
+			}
+
+			// multi-city country, build submenu with "best available" option and cities.
+			time.Sleep(newMenuDelay)
+			bm := countryMenu.AddSubMenuItemCheckbox("Best Available", "", false)
+			setExitNodeOnClick(bm, country.best.ID)
+			countryMenu.AddSeparator()
+
+			for _, city := range country.sortedCities() {
+				cityMenu := countryMenu.AddSubMenuItemCheckbox(city.name, "", false)
+				setExitNodeOnClick(cityMenu, city.best.ID)
+				if status.ExitNodeStatus != nil {
+					for _, ps := range city.peers {
+						if status.ExitNodeStatus.ID == ps.ID {
+							mullvadMenu.Check()
+							countryMenu.Check()
+							cityMenu.Check()
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// TODO: "Allow Local Network Access" and "Run Exit Node" menu items
+}
+
+// mullvadPeers contains all mullvad peer nodes, sorted by country and city.
+type mullvadPeers struct {
+	countries map[string]*mvCountry // country code (uppercase) => country
+}
+
+// sortedCountries returns countries containing mullvad nodes, sorted by name.
+func (mp mullvadPeers) sortedCountries() []*mvCountry {
+	countries := slicesx.MapValues(mp.countries)
+	slices.SortFunc(countries, func(a, b *mvCountry) int {
+		return stringsx.CompareFold(a.name, b.name)
+	})
+	return countries
+}
+
+type mvCountry struct {
+	code   string
+	name   string
+	best   *ipnstate.PeerStatus // highest priority peer in the country
+	cities map[string]*mvCity   // city code => city
+}
+
+// sortedCities returns cities containing mullvad nodes, sorted by name.
+func (mc *mvCountry) sortedCities() []*mvCity {
+	cities := slicesx.MapValues(mc.cities)
+	slices.SortFunc(cities, func(a, b *mvCity) int {
+		return stringsx.CompareFold(a.name, b.name)
+	})
+	return cities
+}
+
+// countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag.
+// It returns the empty string on error.
+func countryFlag(code string) string {
+	if len(code) != 2 {
+		return ""
+	}
+	runes := make([]rune, 0, 2)
+	for i := range 2 {
+		b := code[i] | 32 // lowercase
+		if b < 'a' || b > 'z' {
+			return ""
+		}
+		// https://en.wikipedia.org/wiki/Regional_indicator_symbol
+		runes = append(runes, 0x1F1E6+rune(b-'a'))
+	}
+	return string(runes)
+}
+
+type mvCity struct {
+	name  string
+	best  *ipnstate.PeerStatus // highest priority peer in the city
+	peers []*ipnstate.PeerStatus
+}
+
+func newMullvadPeers(status *ipnstate.Status) mullvadPeers {
+	countries := make(map[string]*mvCountry)
+	for _, ps := range status.Peer {
+		if !ps.ExitNodeOption || ps.Location == nil {
+			continue
+		}
+		loc := ps.Location
+		country, ok := countries[loc.CountryCode]
+		if !ok {
+			country = &mvCountry{
+				code:   loc.CountryCode,
+				name:   loc.Country,
+				cities: make(map[string]*mvCity),
+			}
+			countries[loc.CountryCode] = country
+		}
+		city, ok := countries[loc.CountryCode].cities[loc.CityCode]
+		if !ok {
+			city = &mvCity{
+				name: loc.City,
+			}
+			countries[loc.CountryCode].cities[loc.CityCode] = city
+		}
+		city.peers = append(city.peers, ps)
+		if city.best == nil || ps.Location.Priority > city.best.Location.Priority {
+			city.best = ps
+		}
+		if country.best == nil || ps.Location.Priority > country.best.Location.Priority {
+			country.best = ps
+		}
+	}
+	return mullvadPeers{countries}
+}
+
+// onExit is called by the systray package when the menu is exiting.
+func (menu *Menu) onExit() {
+	log.Printf("exiting")
+	if menu.bgCancel != nil {
+		menu.bgCancel()
+	}
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+
+	os.Remove(menu.notificationIcon.Name())
+}

client/tailscale/acl.go

@@ -19,6 +19,7 @@ import (
 // Only one of Src/Dst or Users/Ports may be specified.
 type ACLRow struct {
 	Action string   `json:"action,omitempty"` // valid values: "accept"
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Users  []string `json:"users,omitempty"`  // old name for src
 	Ports  []string `json:"ports,omitempty"`  // old name for dst
 	Src    []string `json:"src,omitempty"`
@@ -31,6 +32,7 @@ type ACLRow struct {
 type ACLTest struct {
 	Src    string   `json:"src,omitempty"`    // source
 	User   string   `json:"user,omitempty"`   // old name for source
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Accept []string `json:"accept,omitempty"` // expected destination ip:port that user can access
 	Deny   []string `json:"deny,omitempty"`   // expected destination ip:port that user cannot access
 

client/tailscale/apitype/apitype.go

@@ -4,7 +4,10 @@
 // Package apitype contains types for the Tailscale LocalAPI and control plane API.
 package apitype
 
-import "tailscale.com/tailcfg"
+import (
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/dnstype"
+)
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
@@ -57,3 +60,19 @@ type ExitNodeSuggestionResponse struct {
 	Name     string
 	Location tailcfg.LocationView `json:",omitempty"`
 }
+
+// DNSOSConfig mimics dns.OSConfig without forcing us to import the entire dns package
+// into the CLI.
+type DNSOSConfig struct {
+	Nameservers   []string
+	SearchDomains []string
+	MatchDomains  []string
+}
+
+// DNSQueryResponse is the response to a DNS query request sent via LocalAPI.
+type DNSQueryResponse struct {
+	// Bytes is the raw DNS response bytes.
+	Bytes []byte
+	// Resolvers is the list of resolvers that the forwarder deemed able to resolve the query.
+	Resolvers []*dnstype.Resolver
+}

client/tailscale/localclient.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build go1.19
+//go:build go1.22
 
 package tailscale
 
@@ -37,8 +37,10 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
+	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
+	"tailscale.com/util/syspolicy/setting"
 )
 
 // defaultLocalClient is the default LocalClient when using the legacy
@@ -60,6 +62,12 @@ type LocalClient struct {
 	// machine's tailscaled or equivalent. If nil, a default is used.
 	Dial func(ctx context.Context, network, addr string) (net.Conn, error)
 
+	// Transport optionally specifies an alternate [http.RoundTripper]
+	// used to execute HTTP requests. If nil, a default [http.Transport] is used,
+	// potentially with custom dialing logic from [Dial].
+	// It is primarily used for testing.
+	Transport http.RoundTripper
+
 	// Socket specifies an alternate path to the local Tailscale socket.
 	// If empty, a platform-specific default is used.
 	Socket string
@@ -127,9 +135,9 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 	req.Header.Set("Tailscale-Cap", strconv.Itoa(int(tailcfg.CurrentCapabilityVersion)))
 	lc.tsClientOnce.Do(func() {
 		lc.tsClient = &http.Client{
-			Transport: &http.Transport{
-				DialContext: lc.dialer(),
-			},
+			Transport: cmp.Or(lc.Transport, http.RoundTripper(
+				&http.Transport{DialContext: lc.dialer()}),
+			),
 		}
 	})
 	if !lc.OmitAuth {
@@ -353,6 +361,12 @@ func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
+// UserMetrics returns the user metrics in
+// the Prometheus text exposition format.
+func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
+	return lc.get200(ctx, "/localapi/v0/usermetrics")
+}
+
 // IncrementCounter increments the value of a Tailscale daemon's counter
 // metric by the given delta. If the metric has yet to exist, a new counter
 // metric is created and initialized to delta.
@@ -485,6 +499,17 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
 	return nil
 }
 
+// DebugActionBody invokes a debug action with a body parameter, such as
+// "debug-force-prefer-derp".
+// These are development tools and subject to change or removal over time.
+func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
+	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, rbody)
+	if err != nil {
+		return fmt.Errorf("error %w: %s", err, body)
+	}
+	return nil
+}
+
 // DebugResultJSON invokes a debug action and returns its result as something JSON-able.
 // These are development tools and subject to change or removal over time.
 func (lc *LocalClient) DebugResultJSON(ctx context.Context, action string) (any, error) {
@@ -807,6 +832,62 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
+// GetEffectivePolicy returns the effective policy for the specified scope.
+func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+	scopeID, err := scope.MarshalText()
+	if err != nil {
+		return nil, err
+	}
+	body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID))
+	if err != nil {
+		return nil, err
+	}
+	return decodeJSON[*setting.Snapshot](body)
+}
+
+// ReloadEffectivePolicy reloads the effective policy for the specified scope
+// by reading and merging policy settings from all applicable policy sources.
+func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+	scopeID, err := scope.MarshalText()
+	if err != nil {
+		return nil, err
+	}
+	body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody)
+	if err != nil {
+		return nil, err
+	}
+	return decodeJSON[*setting.Snapshot](body)
+}
+
+// GetDNSOSConfig returns the system DNS configuration for the current device.
+// That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
+func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
+	if err != nil {
+		return nil, err
+	}
+	var osCfg apitype.DNSOSConfig
+	if err := json.Unmarshal(body, &osCfg); err != nil {
+		return nil, fmt.Errorf("invalid dns.OSConfig: %w", err)
+	}
+	return &osCfg, nil
+}
+
+// QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
+// It returns the raw DNS response bytes and the resolvers that were used to answer the query
+// (often just one, but can be more if we raced multiple resolvers).
+func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
+	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
+	if err != nil {
+		return nil, nil, err
+	}
+	var res apitype.DNSQueryResponse
+	if err := json.Unmarshal(body, &res); err != nil {
+		return nil, nil, fmt.Errorf("invalid query response: %w", err)
+	}
+	return res.Bytes, res.Resolvers, nil
+}
+
 // StartLoginInteractive starts an interactive login.
 func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)
@@ -1263,6 +1344,17 @@ func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConf
 	return nil
 }
 
+// DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be
+// run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over
+// to another replica whilst there is still some grace period for the existing connections to terminate.
+func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
+	_, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil)
+	if err != nil {
+		return fmt.Errorf("error disconnecting control: %w", err)
+	}
+	return nil
+}
+
 // NetworkLockDisable shuts down network-lock across the tailnet.
 func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) error {
 	if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil {

client/tailscale/required_version.go

@@ -1,10 +1,10 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !go1.21
+//go:build !go1.23
 
 package tailscale
 
 func init() {
-	you_need_Go_1_21_to_compile_Tailscale()
+	you_need_Go_1_23_to_compile_Tailscale()
 }

client/tailscale/tailscale.go

@@ -51,6 +51,9 @@ type Client struct {
 	// HTTPClient optionally specifies an alternate HTTP client to use.
 	// If nil, http.DefaultClient is used.
 	HTTPClient *http.Client
+
+	// UserAgent optionally specifies an alternate User-Agent header
+	UserAgent string
 }
 
 func (c *Client) httpClient() *http.Client {
@@ -97,8 +100,9 @@ func (c *Client) setAuth(r *http.Request) {
 // and can be changed manually by the user.
 func NewClient(tailnet string, auth AuthMethod) *Client {
 	return &Client{
-		tailnet: tailnet,
-		auth:    auth,
+		tailnet:   tailnet,
+		auth:      auth,
+		UserAgent: "tailscale-client-oss",
 	}
 }
 
@@ -110,17 +114,16 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 		return nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
 	}
 	c.setAuth(req)
+	if c.UserAgent != "" {
+		req.Header.Set("User-Agent", c.UserAgent)
+	}
 	return c.httpClient().Do(req)
 }
 
 // sendRequest add the authentication key to the request and sends it. It
 // receives the response and reads up to 10MB of it.
 func (c *Client) sendRequest(req *http.Request) ([]byte, *http.Response, error) {
-	if !I_Acknowledge_This_API_Is_Unstable {
-		return nil, nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
-	}
-	c.setAuth(req)
-	resp, err := c.httpClient().Do(req)
+	resp, err := c.Do(req)
 	if err != nil {
 		return nil, resp, err
 	}

client/web/web.go

@@ -17,7 +17,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -27,6 +26,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
+	"tailscale.com/envknob/featureknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -35,6 +35,7 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/views"
 	"tailscale.com/util/httpm"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -88,8 +89,8 @@ type Server struct {
 type ServerMode string
 
 const (
-	// LoginServerMode serves a readonly login client for logging a
-	// node into a tailnet, and viewing a readonly interface of the
+	// LoginServerMode serves a read-only login client for logging a
+	// node into a tailnet, and viewing a read-only interface of the
 	// node's current Tailscale settings.
 	//
 	// In this mode, API calls are authenticated via platform auth.
@@ -109,15 +110,10 @@ const (
 	// This mode restricts the app to only being assessible over Tailscale,
 	// and API calls are authenticated via browser sessions associated with
 	// the source's Tailscale identity. If the source browser does not have
-	// a valid session, a readonly version of the app is displayed.
+	// a valid session, a read-only version of the app is displayed.
 	ManageServerMode ServerMode = "manage"
 )
 
-var (
-	exitNodeRouteV4 = netip.MustParsePrefix("0.0.0.0/0")
-	exitNodeRouteV6 = netip.MustParsePrefix("::/0")
-)
-
 // ServerOpts contains options for constructing a new Server.
 type ServerOpts struct {
 	// Mode specifies the mode of web client being constructed.
@@ -283,6 +279,12 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if r.URL.Path == "/metrics" {
+		r.URL.Path = "/api/local/v0/usermetrics"
+		s.proxyRequestToLocalAPI(w, r)
+		return
+	}
+
 	if strings.HasPrefix(r.URL.Path, "/api/") {
 		switch {
 		case r.URL.Path == "/api/auth" && r.Method == httpm.GET:
@@ -693,16 +695,16 @@ func (s *Server) serveAPIAuth(w http.ResponseWriter, r *http.Request) {
 	switch {
 	case sErr != nil && errors.Is(sErr, errNotUsingTailscale):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_local", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && errors.Is(sErr, errNotOwner):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_not_owner", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && errors.Is(sErr, errTaggedLocalSource):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_local_tag", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && errors.Is(sErr, errTaggedRemoteSource):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_remote_tag", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && !errors.Is(sErr, errNoSession):
 		// Any other error.
 		http.Error(w, sErr.Error(), http.StatusInternalServerError)
@@ -802,8 +804,8 @@ type nodeData struct {
 	DeviceName  string
 	TailnetName string // TLS cert name
 	DomainName  string
-	IPv4        string
-	IPv6        string
+	IPv4        netip.Addr
+	IPv6        netip.Addr
 	OS          string
 	IPNVersion  string
 
@@ -862,10 +864,14 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	filterRules, _ := s.lc.DebugPacketFilterRules(r.Context())
+	ipv4, ipv6 := s.selfNodeAddresses(r, st)
+
 	data := &nodeData{
 		ID:               st.Self.ID,
 		Status:           st.BackendState,
 		DeviceName:       strings.Split(st.Self.DNSName, ".")[0],
+		IPv4:             ipv4,
+		IPv6:             ipv6,
 		OS:               st.Self.OS,
 		IPNVersion:       strings.Split(st.Version, "-")[0],
 		Profile:          st.User[st.Self.UserID],
@@ -885,10 +891,6 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 		ACLAllowsAnyIncomingTraffic: s.aclsAllowAccess(filterRules),
 	}
 
-	ipv4, ipv6 := s.selfNodeAddresses(r, st)
-	data.IPv4 = ipv4.String()
-	data.IPv6 = ipv6.String()
-
 	if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn && data.URLPrefix == "" {
 		// X-Ingress-Path is the path prefix in use for Home Assistant
 		// https://developers.home-assistant.io/docs/add-ons/presentation#ingress
@@ -921,10 +923,10 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 			return p == route
 		})
 	}
-	data.AdvertisingExitNodeApproved = routeApproved(exitNodeRouteV4) || routeApproved(exitNodeRouteV6)
+	data.AdvertisingExitNodeApproved = routeApproved(tsaddr.AllIPv4()) || routeApproved(tsaddr.AllIPv6())
 
 	for _, r := range prefs.AdvertiseRoutes {
-		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
+		if tsaddr.IsExitRoute(r) {
 			data.AdvertisingExitNode = true
 		} else {
 			data.AdvertisedRoutes = append(data.AdvertisedRoutes, subnetRoute{
@@ -959,37 +961,16 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 }
 
 func availableFeatures() map[string]bool {
-	env := hostinfo.GetEnvType()
 	features := map[string]bool{
 		"advertise-exit-node": true, // available on all platforms
 		"advertise-routes":    true, // available on all platforms
-		"use-exit-node":       canUseExitNode(env) == nil,
-		"ssh":                 envknob.CanRunTailscaleSSH() == nil,
+		"use-exit-node":       featureknob.CanUseExitNode() == nil,
+		"ssh":                 featureknob.CanRunTailscaleSSH() == nil,
 		"auto-update":         version.IsUnstableBuild() && clientupdate.CanAutoUpdate(),
 	}
-	if env == hostinfo.HomeAssistantAddOn {
-		// Setting SSH on Home Assistant causes trouble on startup
-		// (since the flag is not being passed to `tailscale up`).
-		// Although Tailscale SSH does work here,
-		// it's not terribly useful since it's running in a separate container.
-		features["ssh"] = false
-	}
 	return features
 }
 
-func canUseExitNode(env hostinfo.EnvType) error {
-	switch dist := distro.Get(); dist {
-	case distro.Synology, // see https://github.com/tailscale/tailscale/issues/1995
-		distro.QNAP,
-		distro.Unraid:
-		return fmt.Errorf("Tailscale exit nodes cannot be used on %s.", dist)
-	}
-	if env == hostinfo.HomeAssistantAddOn {
-		return errors.New("Tailscale exit nodes cannot be used on Home Assistant.")
-	}
-	return nil
-}
-
 // aclsAllowAccess returns whether tailnet ACLs (as expressed in the provided filter rules)
 // permit any devices to access the local web client.
 // This does not currently check whether a specific device can connect, just any device.
@@ -1065,7 +1046,7 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 	var currNonExitRoutes []string
 	var currAdvertisingExitNode bool
 	for _, r := range prefs.AdvertiseRoutes {
-		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
+		if tsaddr.IsExitRoute(r) {
 			currAdvertisingExitNode = true
 			continue
 		}
@@ -1086,12 +1067,7 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 		return err
 	}
 
-	hasExitNodeRoute := func(all []netip.Prefix) bool {
-		return slices.Contains(all, exitNodeRouteV4) ||
-			slices.Contains(all, exitNodeRouteV6)
-	}
-
-	if !data.UseExitNode.IsZero() && hasExitNodeRoute(routes) {
+	if !data.UseExitNode.IsZero() && tsaddr.ContainsExitRoutes(views.SliceOf(routes)) {
 		return errors.New("cannot use and advertise exit node at same time")
 	}
 

client/web/yarn.lock

@@ -5382,9 +5382,9 @@ wrappy@1:
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
 ws@^8.14.2:
-  version "8.14.2"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f"
-  integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
+  integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
 
 xml-name-validator@^5.0.0:
   version "5.0.0"

clientupdate/clientupdate.go

@@ -27,11 +27,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/uuid"
-	"tailscale.com/clientupdate/distsign"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
-	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
@@ -756,164 +753,6 @@ func (up *Updater) updateMacAppStore() error {
 	return nil
 }
 
-const (
-	// winMSIEnv is the environment variable that, if set, is the MSI file for
-	// the update command to install. It's passed like this so we can stop the
-	// tailscale.exe process from running before the msiexec process runs and
-	// tries to overwrite ourselves.
-	winMSIEnv = "TS_UPDATE_WIN_MSI"
-	// winExePathEnv is the environment variable that is set along with
-	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
-	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
-	// install is complete.
-	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
-)
-
-var (
-	verifyAuthenticode func(string) error // set non-nil only on Windows
-	markTempFileFunc   func(string) error // set non-nil only on Windows
-)
-
-func (up *Updater) updateWindows() error {
-	if msi := os.Getenv(winMSIEnv); msi != "" {
-		// stdout/stderr from this part of the install could be lost since the
-		// parent tailscaled is replaced. Create a temp log file to have some
-		// output to debug with in case update fails.
-		close, err := up.switchOutputToFile()
-		if err != nil {
-			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
-		} else {
-			defer close.Close()
-		}
-
-		up.Logf("installing %v ...", msi)
-		if err := up.installMSI(msi); err != nil {
-			up.Logf("MSI install failed: %v", err)
-			return err
-		}
-
-		up.Logf("success.")
-		return nil
-	}
-
-	if !winutil.IsCurrentProcessElevated() {
-		return errors.New(`update must be run as Administrator
-
-you can run the command prompt as Administrator one of these ways:
-* right-click cmd.exe, select 'Run as administrator'
-* press Windows+x, then press a
-* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
-	}
-	ver, err := requestedTailscaleVersion(up.Version, up.Track)
-	if err != nil {
-		return err
-	}
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	if !up.confirm(ver) {
-		return nil
-	}
-
-	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
-	msiDir := filepath.Join(tsDir, "MSICache")
-	if fi, err := os.Stat(tsDir); err != nil {
-		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
-	} else if !fi.IsDir() {
-		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
-	}
-	if err := os.MkdirAll(msiDir, 0700); err != nil {
-		return err
-	}
-	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
-	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
-	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
-	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
-		return err
-	}
-
-	up.Logf("verifying MSI authenticode...")
-	if err := verifyAuthenticode(msiTarget); err != nil {
-		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
-	}
-	up.Logf("authenticode verification succeeded")
-
-	up.Logf("making tailscale.exe copy to switch to...")
-	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
-	selfOrig, selfCopy, err := makeSelfCopy()
-	if err != nil {
-		return err
-	}
-	defer os.Remove(selfCopy)
-	up.Logf("running tailscale.exe copy for final install...")
-
-	cmd := exec.Command(selfCopy, "update")
-	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
-	cmd.Stdout = up.Stderr
-	cmd.Stderr = up.Stderr
-	cmd.Stdin = os.Stdin
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	// Once it's started, exit ourselves, so the binary is free
-	// to be replaced.
-	os.Exit(0)
-	panic("unreachable")
-}
-
-func (up *Updater) switchOutputToFile() (io.Closer, error) {
-	var logFilePath string
-	exePath, err := os.Executable()
-	if err != nil {
-		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
-	} else {
-		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
-	}
-
-	up.Logf("writing update output to %q", logFilePath)
-	logFile, err := os.Create(logFilePath)
-	if err != nil {
-		return nil, err
-	}
-
-	up.Logf = func(m string, args ...any) {
-		fmt.Fprintf(logFile, m+"\n", args...)
-	}
-	up.Stdout = logFile
-	up.Stderr = logFile
-	return logFile, nil
-}
-
-func (up *Updater) installMSI(msi string) error {
-	var err error
-	for tries := 0; tries < 2; tries++ {
-		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
-		cmd.Dir = filepath.Dir(msi)
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		if err == nil {
-			break
-		}
-		up.Logf("Install attempt failed: %v", err)
-		uninstallVersion := up.currentVersion
-		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
-			uninstallVersion = v
-		}
-		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
-		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
-		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		up.Logf("msiexec uninstall: %v", err)
-	}
-	return err
-}
-
 // cleanupOldDownloads removes all files matching glob (see filepath.Glob).
 // Only regular files are removed, so the glob must match specific files and
 // not directories.
@@ -938,53 +777,6 @@ func (up *Updater) cleanupOldDownloads(glob string) {
 	}
 }
 
-func msiUUIDForVersion(ver string) string {
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	track, err := versionToTrack(ver)
-	if err != nil {
-		track = UnstableTrack
-	}
-	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
-	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
-}
-
-func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
-	selfExe, err := os.Executable()
-	if err != nil {
-		return "", "", err
-	}
-	f, err := os.Open(selfExe)
-	if err != nil {
-		return "", "", err
-	}
-	defer f.Close()
-	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
-	if err != nil {
-		return "", "", err
-	}
-	if f := markTempFileFunc; f != nil {
-		if err := f(f2.Name()); err != nil {
-			return "", "", err
-		}
-	}
-	if _, err := io.Copy(f2, f); err != nil {
-		f2.Close()
-		return "", "", err
-	}
-	return selfExe, f2.Name(), f2.Close()
-}
-
-func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
-	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
-	if err != nil {
-		return err
-	}
-	return c.Download(context.Background(), pathSrc, fileDst)
-}
-
 func (up *Updater) updateFreeBSD() (err error) {
 	if up.Version != "" {
 		return errors.New("installing a specific version on FreeBSD is not supported")

clientupdate/clientupdate_downloads.go

@@ -0,0 +1,20 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build (linux && !android) || windows
+
+package clientupdate
+
+import (
+	"context"
+
+	"tailscale.com/clientupdate/distsign"
+)
+
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
+	if err != nil {
+		return err
+	}
+	return c.Download(context.Background(), pathSrc, fileDst)
+}

clientupdate/clientupdate_not_downloads.go

@@ -0,0 +1,10 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !((linux && !android) || windows)
+
+package clientupdate
+
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	panic("unreachable")
+}

clientupdate/clientupdate_notwindows.go

@@ -0,0 +1,10 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !windows
+
+package clientupdate
+
+func (up *Updater) updateWindows() error {
+	panic("unreachable")
+}

clientupdate/clientupdate_windows.go

@@ -7,13 +7,57 @@
 package clientupdate
 
 import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/google/uuid"
 	"golang.org/x/sys/windows"
+	"tailscale.com/util/winutil"
 	"tailscale.com/util/winutil/authenticode"
 )
 
-func init() {
-	markTempFileFunc = markTempFileWindows
-	verifyAuthenticode = verifyTailscale
+const (
+	// winMSIEnv is the environment variable that, if set, is the MSI file for
+	// the update command to install. It's passed like this so we can stop the
+	// tailscale.exe process from running before the msiexec process runs and
+	// tries to overwrite ourselves.
+	winMSIEnv = "TS_UPDATE_WIN_MSI"
+	// winExePathEnv is the environment variable that is set along with
+	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
+	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
+	// install is complete.
+	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
+)
+
+func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
+	selfExe, err := os.Executable()
+	if err != nil {
+		return "", "", err
+	}
+	f, err := os.Open(selfExe)
+	if err != nil {
+		return "", "", err
+	}
+	defer f.Close()
+	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
+	if err != nil {
+		return "", "", err
+	}
+	if err := markTempFileWindows(f2.Name()); err != nil {
+		return "", "", err
+	}
+	if _, err := io.Copy(f2, f); err != nil {
+		f2.Close()
+		return "", "", err
+	}
+	return selfExe, f2.Name(), f2.Close()
 }
 
 func markTempFileWindows(name string) error {
@@ -23,6 +67,159 @@ func markTempFileWindows(name string) error {
 
 const certSubjectTailscale = "Tailscale Inc."
 
-func verifyTailscale(path string) error {
+func verifyAuthenticode(path string) error {
 	return authenticode.Verify(path, certSubjectTailscale)
 }
+
+func (up *Updater) updateWindows() error {
+	if msi := os.Getenv(winMSIEnv); msi != "" {
+		// stdout/stderr from this part of the install could be lost since the
+		// parent tailscaled is replaced. Create a temp log file to have some
+		// output to debug with in case update fails.
+		close, err := up.switchOutputToFile()
+		if err != nil {
+			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
+		} else {
+			defer close.Close()
+		}
+
+		up.Logf("installing %v ...", msi)
+		if err := up.installMSI(msi); err != nil {
+			up.Logf("MSI install failed: %v", err)
+			return err
+		}
+
+		up.Logf("success.")
+		return nil
+	}
+
+	if !winutil.IsCurrentProcessElevated() {
+		return errors.New(`update must be run as Administrator
+
+you can run the command prompt as Administrator one of these ways:
+* right-click cmd.exe, select 'Run as administrator'
+* press Windows+x, then press a
+* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
+	}
+	ver, err := requestedTailscaleVersion(up.Version, up.Track)
+	if err != nil {
+		return err
+	}
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	if !up.confirm(ver) {
+		return nil
+	}
+
+	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
+	msiDir := filepath.Join(tsDir, "MSICache")
+	if fi, err := os.Stat(tsDir); err != nil {
+		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
+	}
+	if err := os.MkdirAll(msiDir, 0700); err != nil {
+		return err
+	}
+	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
+	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
+	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
+		return err
+	}
+
+	up.Logf("verifying MSI authenticode...")
+	if err := verifyAuthenticode(msiTarget); err != nil {
+		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
+	}
+	up.Logf("authenticode verification succeeded")
+
+	up.Logf("making tailscale.exe copy to switch to...")
+	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
+	selfOrig, selfCopy, err := makeSelfCopy()
+	if err != nil {
+		return err
+	}
+	defer os.Remove(selfCopy)
+	up.Logf("running tailscale.exe copy for final install...")
+
+	cmd := exec.Command(selfCopy, "update")
+	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
+	cmd.Stdout = up.Stderr
+	cmd.Stderr = up.Stderr
+	cmd.Stdin = os.Stdin
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	// Once it's started, exit ourselves, so the binary is free
+	// to be replaced.
+	os.Exit(0)
+	panic("unreachable")
+}
+
+func (up *Updater) installMSI(msi string) error {
+	var err error
+	for tries := 0; tries < 2; tries++ {
+		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
+		cmd.Dir = filepath.Dir(msi)
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		if err == nil {
+			break
+		}
+		up.Logf("Install attempt failed: %v", err)
+		uninstallVersion := up.currentVersion
+		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
+			uninstallVersion = v
+		}
+		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
+		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
+		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		up.Logf("msiexec uninstall: %v", err)
+	}
+	return err
+}
+
+func msiUUIDForVersion(ver string) string {
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	track, err := versionToTrack(ver)
+	if err != nil {
+		track = UnstableTrack
+	}
+	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
+	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
+}
+
+func (up *Updater) switchOutputToFile() (io.Closer, error) {
+	var logFilePath string
+	exePath, err := os.Executable()
+	if err != nil {
+		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
+	} else {
+		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
+	}
+
+	up.Logf("writing update output to %q", logFilePath)
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	up.Logf = func(m string, args ...any) {
+		fmt.Fprintf(logFile, m+"\n", args...)
+	}
+	up.Stdout = logFile
+	up.Stderr = logFile
+	return logFile, nil
+}

cmd/addlicense/main.go

@@ -18,12 +18,12 @@ var (
 )
 
 func usage() {
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 usage: addlicense -file FILE <subcommand args...>
 `[1:])
 
 	flag.PrintDefaults()
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 addlicense adds a Tailscale license to the beginning of file.
 
 It is intended for use with 'go generate', so it also runs a subcommand,

cmd/checkmetrics/checkmetrics.go

@@ -0,0 +1,131 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// checkmetrics validates that all metrics in the tailscale client-metrics
+// are documented in a given path or URL.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"time"
+
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tsnet"
+	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/util/httpm"
+)
+
+var (
+	kbPath = flag.String("kb-path", "", "filepath to the client-metrics knowledge base")
+	kbUrl  = flag.String("kb-url", "", "URL to the client-metrics knowledge base page")
+)
+
+func main() {
+	flag.Parse()
+	if *kbPath == "" && *kbUrl == "" {
+		log.Fatalf("either -kb-path or -kb-url must be set")
+	}
+
+	var control testcontrol.Server
+	ts := httptest.NewServer(&control)
+	defer ts.Close()
+
+	td, err := os.MkdirTemp("", "testcontrol")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	// tsnet is used not used as a Tailscale client, but as a way to
+	// boot up Tailscale, have all the metrics registered, and then
+	// verifiy that all the metrics are documented.
+	tsn := &tsnet.Server{
+		Dir:        td,
+		Store:      new(mem.Store),
+		UserLogf:   log.Printf,
+		Ephemeral:  true,
+		ControlURL: ts.URL,
+	}
+	if err := tsn.Start(); err != nil {
+		log.Fatal(err)
+	}
+	defer tsn.Close()
+
+	log.Printf("checking that all metrics are documented, looking for: %s", tsn.Sys().UserMetricsRegistry().MetricNames())
+
+	if *kbPath != "" {
+		kb, err := readKB(*kbPath)
+		if err != nil {
+			log.Fatalf("reading kb: %v", err)
+		}
+		missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
+
+		if len(missing) > 0 {
+			log.Fatalf("found undocumented metrics in %q: %v", *kbPath, missing)
+		}
+	}
+
+	if *kbUrl != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		kb, err := getKB(ctx, *kbUrl)
+		if err != nil {
+			log.Fatalf("getting kb: %v", err)
+		}
+		missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
+
+		if len(missing) > 0 {
+			log.Fatalf("found undocumented metrics in %q: %v", *kbUrl, missing)
+		}
+	}
+}
+
+func readKB(path string) (string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("reading file: %w", err)
+	}
+
+	return string(b), nil
+}
+
+func getKB(ctx context.Context, url string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, url, nil)
+	if err != nil {
+		return "", fmt.Errorf("creating request: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("getting kb page: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("reading body: %w", err)
+	}
+	return string(b), nil
+}
+
+func undocumentedMetrics(b string, metrics []string) []string {
+	var missing []string
+	for _, metric := range metrics {
+		if !strings.Contains(b, metric) {
+			missing = append(missing, metric)
+		}
+	}
+	return missing
+}

cmd/cloner/cloner.go

@@ -47,7 +47,7 @@ func main() {
 	it := codegen.NewImportTracker(pkg.Types)
 	buf := new(bytes.Buffer)
 	for _, typeName := range typeNames {
-		typ, ok := namedTypes[typeName]
+		typ, ok := namedTypes[typeName].(*types.Named)
 		if !ok {
 			log.Fatalf("could not find type %s", typeName)
 		}
@@ -115,7 +115,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		if !codegen.ContainsPointers(ft) || codegen.HasNoClone(t.Tag(i)) {
 			continue
 		}
-		if named, _ := ft.(*types.Named); named != nil {
+		if named, _ := codegen.NamedTypeOf(ft); named != nil {
 			if codegen.IsViewType(ft) {
 				writef("dst.%s = src.%s", fname, fname)
 				continue
@@ -161,7 +161,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		case *types.Pointer:
 			base := ft.Elem()
 			hasPtrs := codegen.ContainsPointers(base)
-			if named, _ := base.(*types.Named); named != nil && hasPtrs {
+			if named, _ := codegen.NamedTypeOf(base); named != nil && hasPtrs {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}

cmd/containerboot/forwarding.go

@@ -0,0 +1,262 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"tailscale.com/util/linuxfw"
+)
+
+// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
+	var (
+		v4Forwarding, v6Forwarding bool
+	)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	// Currently we only proxy traffic to the IPv4 address of the tailnet
+	// target.
+	if tailnetTargetFQDN != "" {
+		v4Forwarding = true
+	}
+	if routes != nil && *routes != "" {
+		for _, route := range strings.Split(*routes, ",") {
+			cidr, err := netip.ParsePrefix(route)
+			if err != nil {
+				return fmt.Errorf("invalid subnet route: %v", err)
+			}
+			if cidr.Addr().Is4() {
+				v4Forwarding = true
+			} else {
+				v6Forwarding = true
+			}
+		}
+	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
+
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
+	var paths []string
+	if v4Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
+	}
+	if v6Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
+	}
+
+	// In some common configurations (e.g. default docker,
+	// kubernetes), the container environment denies write access to
+	// most sysctls, including IP forwarding controls. Check the
+	// sysctl values before trying to change them, so that we
+	// gracefully do nothing if the container's already been set up
+	// properly by e.g. a k8s initContainer.
+	for _, path := range paths {
+		bs, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %q: %w", path, err)
+		}
+		if v := strings.TrimSpace(string(bs)); v != "1" {
+			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
+				return fmt.Errorf("enabling %q: %w", path, err)
+			}
+		}
+	}
+	return nil
+}
+
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.EnsureSNATForDst(local, dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	return nil
+}
+
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(_ context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	proxyHasIPv4Address := false
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.AddDNATRule(local, dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRuleForDNSTarget(_ context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}

cmd/containerboot/healthz.go

@@ -0,0 +1,50 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"sync"
+)
+
+// healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type healthz struct {
+	sync.Mutex
+	hasAddrs bool
+}
+
+func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+
+	if h.hasAddrs {
+		w.Write([]byte("ok"))
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable)
+	}
+}
+
+func (h *healthz) update(healthy bool) {
+	h.Lock()
+	defer h.Unlock()
+
+	if h.hasAddrs != healthy {
+		log.Println("Setting healthy", healthy)
+	}
+	h.hasAddrs = healthy
+}
+
+// healthHandlers registers a simple health handler at /healthz.
+// A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func healthHandlers(mux *http.ServeMux) *healthz {
+	h := &healthz{}
+	mux.Handle("GET /healthz", h)
+	return h
+}

cmd/containerboot/kube.go

@@ -8,31 +8,57 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 )
 
-// storeDeviceID writes deviceID to 'device_id' data field of the named
-// Kubernetes Secret.
-func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
-	s := &kube.Secret{
-		Data: map[string][]byte{
-			"device_id": []byte(deviceID),
-		},
-	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+// kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
+// this rather than any of the upstream Kubernetes client libaries to avoid extra imports.
+type kubeClient struct {
+	kubeclient.Client
+	stateSecret string
+	canPatch    bool // whether the client has permissions to patch Kubernetes Secrets
 }
 
-// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields
-// 'device_ips', 'device_fqdn' of the named Kubernetes Secret.
-func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, addresses []netip.Prefix) error {
+func newKubeClient(root string, stateSecret string) (*kubeClient, error) {
+	if root != "/" {
+		// If we are running in a test, we need to set the root path to the fake
+		// service account directory.
+		kubeclient.SetRootPathForTesting(root)
+	}
+	var err error
+	kc, err := kubeclient.New("tailscale-container")
+	if err != nil {
+		return nil, fmt.Errorf("Error creating kube client: %w", err)
+	}
+	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		// Used to set http server in tests, or optionally enabled by flag
+		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	}
+	return &kubeClient{Client: kc, stateSecret: stateSecret}, nil
+}
+
+// storeDeviceID writes deviceID to 'device_id' data field of the client's state Secret.
+func (kc *kubeClient) storeDeviceID(ctx context.Context, deviceID tailcfg.StableNodeID) error {
+	s := &kubeapi.Secret{
+		Data: map[string][]byte{
+			kubetypes.KeyDeviceID: []byte(deviceID),
+		},
+	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
+}
+
+// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields 'device_ips', 'device_fqdn' of client's
+// state Secret.
+func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, addresses []netip.Prefix) error {
 	var ips []string
 	for _, addr := range addresses {
 		ips = append(ips, addr.Addr().String())
@@ -42,27 +68,39 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 		return err
 	}
 
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
-			"device_fqdn": []byte(fqdn),
-			"device_ips":  deviceIPs,
+			kubetypes.KeyDeviceFQDN: []byte(fqdn),
+			kubetypes.KeyDeviceIPs:  deviceIPs,
 		},
 	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
+}
+
+// storeHTTPSEndpoint writes an HTTPS endpoint exposed by this device via 'tailscale serve' to the client's state
+// Secret. In practice this will be the same value that gets written to 'device_fqdn', but this should only be called
+// when the serve config has been successfully set up.
+func (kc *kubeClient) storeHTTPSEndpoint(ctx context.Context, ep string) error {
+	s := &kubeapi.Secret{
+		Data: map[string][]byte{
+			kubetypes.KeyHTTPSEndpoint: []byte(ep),
+		},
+	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }
 
 // deleteAuthKey deletes the 'authkey' field of the given kube
 // secret. No-op if there is no authkey in the secret.
-func deleteAuthKey(ctx context.Context, secretName string) error {
+func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
-	m := []kube.JSONPatch{
+	m := []kubeclient.JSONPatch{
 		{
 			Op:   "remove",
 			Path: "/data/authkey",
 		},
 	}
-	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
-		if s, ok := err.(*kube.Status); ok && s.Code == http.StatusUnprocessableEntity {
+	if err := kc.JSONPatchResource(ctx, kc.stateSecret, kubeclient.TypeSecrets, m); err != nil {
+		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
 			return nil
@@ -72,72 +110,19 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kube.Client
-
-// setupKube is responsible for doing any necessary configuration and checks to
-// ensure that tailscale state storage and authentication mechanism will work on
-// Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
-	if cfg.KubeSecret == "" {
-		return nil
+// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale
+// state Secret.
+// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container.
+func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error {
+	capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
+	d := map[string][]byte{
+		kubetypes.KeyCapVer: []byte(capVerS),
 	}
-	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
-	if err != nil {
-		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	if podUID != "" {
+		d[kubetypes.KeyPodUID] = []byte(podUID)
 	}
-	cfg.KubernetesCanPatch = canPatch
-
-	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
-	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
-		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
-			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
-			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
-			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
-	} else if err != nil && !kube.IsNotFoundErr(err) {
-		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
-	}
-
-	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-		if s == nil {
-			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
-			return nil
-		}
-		keyBytes, _ := s.Data["authkey"]
-		key := string(keyBytes)
-
-		if key != "" {
-			// This behavior of pulling authkeys from kube secrets was added
-			// at the same time as the patch permission, so we can enforce
-			// that we must be able to patch out the authkey after
-			// authenticating if you want to use this feature. This avoids
-			// us having to deal with the case where we might leave behind
-			// an unnecessary reusable authkey in a secret, like a rake in
-			// the grass.
-			if !cfg.KubernetesCanPatch {
-				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-			}
-			cfg.AuthKey = key
-		} else {
-			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-		}
-	}
-	return nil
-}
-
-func initKubeClient(root string) {
-	if root != "/" {
-		// If we are running in a test, we need to set the root path to the fake
-		// service account directory.
-		kube.SetRootPathForTesting(root)
-	}
-	var err error
-	kc, err = kube.New()
-	if err != nil {
-		log.Fatalf("Error creating kube client: %v", err)
-	}
-	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
-		// Derive the API server address from the environment variables
-		// Used to set http server in tests, or optionally enabled by flag
-		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	s := &kubeapi.Secret{
+		Data: d,
 	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }

cmd/containerboot/kube_test.go

@@ -11,7 +11,8 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 )
 
 func TestSetupKube(t *testing.T) {
@@ -20,7 +21,7 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kube.Client
+		kc      *kubeClient
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
@@ -28,14 +29,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -47,14 +48,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -66,14 +67,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -86,14 +87,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 403}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 403}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -110,11 +111,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
-			},
+			}},
 			wantErr: true,
 		},
 		{
@@ -126,14 +127,14 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 		},
 		{
 			// Interactive login using URL in Pod logs
@@ -144,28 +145,28 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{}, nil
 				},
-			},
+			}},
 		},
 		{
 			name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
@@ -176,14 +177,14 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				KubeSecret:         "foo",
 				AuthKey:            "foo",
@@ -193,9 +194,9 @@ func TestSetupKube(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		kc = tt.kc
+		kc := tt.kc
 		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
+			if err := tt.cfg.setupKube(context.Background(), kc); (err != nil) != tt.wantErr {
 				t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
 			}
 			if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {

cmd/containerboot/main_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn"
+	"tailscale.com/kube/egressservices"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/netmap"
@@ -57,6 +58,16 @@ func TestContainerBoot(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error unmarshaling tailscaled config: %v", err)
 	}
+	serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}}
+	serveConfBytes, err := json.Marshal(serveConf)
+	if err != nil {
+		t.Fatalf("error unmarshaling serve config: %v", err)
+	}
+	egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}}
+	egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg)
+	if err != nil {
+		t.Fatalf("error unmarshaling egress services config: %v", err)
+	}
 
 	dirs := []string{
 		"var/lib",
@@ -73,14 +84,16 @@ func TestContainerBoot(t *testing.T) {
 		}
 	}
 	files := map[string][]byte{
-		"usr/bin/tailscaled":                    fakeTailscaled,
-		"usr/bin/tailscale":                     fakeTailscale,
-		"usr/bin/iptables":                      fakeTailscale,
-		"usr/bin/ip6tables":                     fakeTailscale,
-		"dev/net/tun":                           []byte(""),
-		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
-		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
-		"etc/tailscaled/cap-95.hujson":          tailscaledConfBytes,
+		"usr/bin/tailscaled":                         fakeTailscaled,
+		"usr/bin/tailscale":                          fakeTailscale,
+		"usr/bin/iptables":                           fakeTailscale,
+		"usr/bin/ip6tables":                          fakeTailscale,
+		"dev/net/tun":                                []byte(""),
+		"proc/sys/net/ipv4/ip_forward":               []byte("0"),
+		"proc/sys/net/ipv6/conf/all/forwarding":      []byte("0"),
+		"etc/tailscaled/cap-95.hujson":               tailscaledConfBytes,
+		"etc/tailscaled/serve-config.json":           serveConfBytes,
+		"etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes,
 	}
 	resetFiles := func() {
 		for path, content := range files {
@@ -101,6 +114,26 @@ func TestContainerBoot(t *testing.T) {
 
 	argFile := filepath.Join(d, "args")
 	runningSockPath := filepath.Join(d, "tmp/tailscaled.sock")
+	var localAddrPort, healthAddrPort int
+	for _, p := range []*int{&localAddrPort, &healthAddrPort} {
+		ln, err := net.Listen("tcp", ":0")
+		if err != nil {
+			t.Fatalf("Failed to open listener: %v", err)
+		}
+		if err := ln.Close(); err != nil {
+			t.Fatalf("Failed to close listener: %v", err)
+		}
+		port := ln.Addr().(*net.TCPAddr).Port
+		*p = port
+	}
+	metricsURL := func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d/metrics", port)
+	}
+	healthURL := func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
+	}
+
+	capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
 
 	type phase struct {
 		// If non-nil, send this IPN bus notification (and remember it as the
@@ -119,6 +152,8 @@ func TestContainerBoot(t *testing.T) {
 		// WantFatalLog is the fatal log message we expect from containerboot.
 		// If set for a phase, the test will finish on that phase.
 		WantFatalLog string
+
+		EndpointStatuses map[string]int
 	}
 	runningNotify := &ipn.Notify{
 		State: ptr.To(ipn.Running),
@@ -147,6 +182,11 @@ func TestContainerBoot(t *testing.T) {
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
 					},
+					// No metrics or health by default.
+					EndpointStatuses: map[string]int{
+						metricsURL(9002): -1,
+						healthURL(9002):  -1,
+					},
 				},
 				{
 					Notify: runningNotify,
@@ -453,10 +493,11 @@ func TestContainerBoot(t *testing.T) {
 				{
 					Notify: runningNotify,
 					WantKubeSecret: map[string]string{
-						"authkey":     "tskey-key",
-						"device_fqdn": "test-node.test.ts.net",
-						"device_id":   "myID",
-						"device_ips":  `["100.64.0.1"]`,
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 			},
@@ -546,9 +587,10 @@ func TestContainerBoot(t *testing.T) {
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
 					},
 					WantKubeSecret: map[string]string{
-						"device_fqdn": "test-node.test.ts.net",
-						"device_id":   "myID",
-						"device_ips":  `["100.64.0.1"]`,
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 			},
@@ -575,10 +617,11 @@ func TestContainerBoot(t *testing.T) {
 				{
 					Notify: runningNotify,
 					WantKubeSecret: map[string]string{
-						"authkey":     "tskey-key",
-						"device_fqdn": "test-node.test.ts.net",
-						"device_id":   "myID",
-						"device_ips":  `["100.64.0.1"]`,
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 				{
@@ -593,10 +636,11 @@ func TestContainerBoot(t *testing.T) {
 						},
 					},
 					WantKubeSecret: map[string]string{
-						"authkey":     "tskey-key",
-						"device_fqdn": "new-name.test.ts.net",
-						"device_id":   "newID",
-						"device_ips":  `["100.64.0.1"]`,
+						"authkey":          "tskey-key",
+						"device_fqdn":      "new-name.test.ts.net",
+						"device_id":        "newID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 			},
@@ -700,6 +744,199 @@ func TestContainerBoot(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "metrics_enabled",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_METRICS":  "true",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(localAddrPort):  -1,
+					},
+				}, {
+					Notify: runningNotify,
+				},
+			},
+		},
+		{
+			Name: "health_enabled",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT":     fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_HEALTH_CHECK": "true",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): -1,
+						healthURL(localAddrPort):  503, // Doesn't start passing until the next phase.
+					},
+				}, {
+					Notify: runningNotify,
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): -1,
+						healthURL(localAddrPort):  200,
+					},
+				},
+			},
+		},
+		{
+			Name: "metrics_and_health_on_same_port",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT":     fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_METRICS":      "true",
+				"TS_ENABLE_HEALTH_CHECK": "true",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(localAddrPort):  503, // Doesn't start passing until the next phase.
+					},
+				}, {
+					Notify: runningNotify,
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(localAddrPort):  200,
+					},
+				},
+			},
+		},
+		{
+			Name: "local_metrics_and_deprecated_health",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT":       fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_METRICS":        "true",
+				"TS_HEALTHCHECK_ADDR_PORT": fmt.Sprintf("[::]:%d", healthAddrPort),
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(healthAddrPort): 503, // Doesn't start passing until the next phase.
+					},
+				}, {
+					Notify: runningNotify,
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(healthAddrPort): 200,
+					},
+				},
+			},
+		},
+		{
+			Name: "serve_config_no_kube",
+			Env: map[string]string{
+				"TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"),
+				"TS_AUTHKEY":      "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+				},
+			},
+		},
+		{
+			Name: "serve_config_kube",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_SERVE_CONFIG":               filepath.Join(d, "etc/tailscaled/serve-config.json"),
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+					WantKubeSecret: map[string]string{
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"https_endpoint":   "no-https",
+						"tailscale_capver": capver,
+					},
+				},
+			},
+		},
+		{
+			Name: "egress_svcs_config_kube",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":        kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS":  kube.Port,
+				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+					WantKubeSecret: map[string]string{
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
+					},
+				},
+			},
+		},
+		{
+			Name: "egress_svcs_config_no_kube",
+			Env: map[string]string{
+				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
+				"TS_AUTHKEY":                     "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -796,7 +1033,26 @@ func TestContainerBoot(t *testing.T) {
 					return nil
 				})
 				if err != nil {
-					t.Fatal(err)
+					t.Fatalf("phase %d: %v", i, err)
+				}
+
+				for url, want := range p.EndpointStatuses {
+					err := tstest.WaitFor(2*time.Second, func() error {
+						resp, err := http.Get(url)
+						if err != nil && want != -1 {
+							return fmt.Errorf("GET %s: %v", url, err)
+						}
+						if want > 0 && resp.StatusCode != want {
+							defer resp.Body.Close()
+							body, _ := io.ReadAll(resp.Body)
+							return fmt.Errorf("GET %s, want %d, got %d\n%s", url, want, resp.StatusCode, string(body))
+						}
+
+						return nil
+					})
+					if err != nil {
+						t.Fatalf("phase %d: %v", i, err)
+					}
 				}
 			}
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
@@ -955,6 +1211,12 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if r.Method != "GET" {
 			panic(fmt.Sprintf("unsupported method %q", r.Method))
 		}
+	case "/localapi/v0/usermetrics":
+		if r.Method != "GET" {
+			panic(fmt.Sprintf("unsupported method %q", r.Method))
+		}
+		w.Write([]byte("fake metrics"))
+		return
 	default:
 		panic(fmt.Sprintf("unsupported path %q", r.URL.Path))
 	}

cmd/containerboot/metrics.go

@@ -0,0 +1,79 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
+)
+
+// metrics is a simple metrics HTTP server, if enabled it forwards requests to
+// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
+type metrics struct {
+	debugEndpoint string
+	lc            *tailscale.LocalClient
+}
+
+func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+
+	resp, err := do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	for key, val := range resp.Header {
+		for _, v := range val {
+			w.Header().Add(key, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
+	localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
+	proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
+}
+
+func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
+	if m.debugEndpoint == "" {
+		http.Error(w, "debug endpoint not configured", http.StatusNotFound)
+		return
+	}
+
+	debugURL := "http://" + m.debugEndpoint + r.URL.Path
+	proxy(w, r, debugURL, http.DefaultClient.Do)
+}
+
+// metricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding
+// requests to tailscaled's /localapi/v0/usermetrics API.
+//
+// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
+// endpoint if configured to ease migration for a breaking change serving user
+// metrics instead of debug metrics on the "metrics" port.
+func metricsHandlers(mux *http.ServeMux, lc *tailscale.LocalClient, debugAddrPort string) {
+	m := &metrics{
+		lc:            lc,
+		debugEndpoint: debugAddrPort,
+	}
+
+	mux.HandleFunc("GET /metrics", m.handleMetrics)
+	mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
+}

cmd/containerboot/serve.go

@@ -0,0 +1,155 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/types/netmap"
+)
+
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient, kc *kubeClient) {
+	if certDomainAtomic == nil {
+		panic("certDomainAtomic must not be nil")
+	}
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("serve proxy: failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(path)); err != nil {
+			log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
+		}
+		eventChan = w.Events
+	}
+
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-eventChan:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("serve proxy: failed to read serve config: %v", err)
+		}
+		if sc == nil {
+			log.Printf("serve proxy: no serve config at %q, skipping", path)
+			continue
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil {
+			log.Fatalf("serve proxy: error updating serve config: %v", err)
+		}
+		if kc != nil && kc.canPatch {
+			if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil {
+				log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err)
+			}
+		}
+		prevServeConfig = sc
+	}
+}
+
+func certDomainFromNetmap(nm *netmap.NetworkMap) string {
+	if len(nm.DNS.CertDomains) == 0 {
+		return ""
+	}
+	return nm.DNS.CertDomains[0]
+}
+
+// localClient is a subset of tailscale.LocalClient that can be mocked for testing.
+type localClient interface {
+	SetServeConfig(context.Context, *ipn.ServeConfig) error
+}
+
+func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error {
+	if !isValidHTTPSConfig(certDomain, sc) {
+		return nil
+	}
+	log.Printf("serve proxy: applying serve config")
+	return lc.SetServeConfig(ctx, sc)
+}
+
+func isValidHTTPSConfig(certDomain string, sc *ipn.ServeConfig) bool {
+	if certDomain == kubetypes.ValueNoHTTPS && hasHTTPSEndpoint(sc) {
+		log.Printf(
+			`serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet,
+		(perhaps a Kubernetes operator Ingress proxy) but it is not able to issue TLS certs, so this will likely not work.
+		To make it work, ensure that HTTPS is enabled for your tailnet, see https://tailscale.com/kb/1153/enabling-https for more details.`)
+		return false
+	}
+	return true
+}
+
+func hasHTTPSEndpoint(cfg *ipn.ServeConfig) bool {
+	if cfg == nil {
+		return false
+	}
+	for _, tcpCfg := range cfg.TCP {
+		if tcpCfg.HTTPS {
+			return true
+		}
+	}
+	return false
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	// Serve config can be provided by users as well as the Kubernetes Operator (for its proxies). User-provided
+	// config could be empty for reasons.
+	if len(j) == 0 {
+		log.Printf("serve proxy: serve config file is empty, skipping")
+		return nil, nil
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}

cmd/containerboot/serve_test.go

@@ -0,0 +1,267 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
+)
+
+func TestUpdateServeConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		sc         *ipn.ServeConfig
+		certDomain string
+		wantCall   bool
+	}{
+		{
+			name: "no_https_no_cert_domain",
+			sc: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80: {HTTP: true},
+				},
+			},
+			certDomain: kubetypes.ValueNoHTTPS, // tailnet has HTTPS disabled
+			wantCall:   true,                   // should set serve config as it doesn't have HTTPS endpoints
+		},
+		{
+			name: "https_with_cert_domain",
+			sc: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {HTTPS: true},
+				},
+				Web: map[ipn.HostPort]*ipn.WebServerConfig{
+					"${TS_CERT_DOMAIN}:443": {
+						Handlers: map[string]*ipn.HTTPHandler{
+							"/": {Proxy: "http://10.0.1.100:8080"},
+						},
+					},
+				},
+			},
+			certDomain: "test-node.tailnet.ts.net",
+			wantCall:   true,
+		},
+		{
+			name: "https_without_cert_domain",
+			sc: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {HTTPS: true},
+				},
+			},
+			certDomain: kubetypes.ValueNoHTTPS,
+			wantCall:   false, // incorrect configuration- should not set serve config
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fakeLC := &fakeLocalClient{}
+			err := updateServeConfig(context.Background(), tt.sc, tt.certDomain, fakeLC)
+			if err != nil {
+				t.Errorf("updateServeConfig() error = %v", err)
+			}
+			if fakeLC.setServeCalled != tt.wantCall {
+				t.Errorf("SetServeConfig() called = %v, want %v", fakeLC.setServeCalled, tt.wantCall)
+			}
+		})
+	}
+}
+
+func TestReadServeConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		gotSC      string
+		certDomain string
+		wantSC     *ipn.ServeConfig
+		wantErr    bool
+	}{
+		{
+			name: "empty_file",
+		},
+		{
+			name: "valid_config_with_cert_domain_placeholder",
+			gotSC: `{
+				"TCP": {
+					"443": {
+						"HTTPS": true
+					}
+				},
+				"Web": {
+					"${TS_CERT_DOMAIN}:443": {
+					"Handlers": {
+						"/api": {
+							"Proxy": "https://10.2.3.4/api"
+						}}}}}`,
+			certDomain: "example.com",
+			wantSC: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {
+						HTTPS: true,
+					},
+				},
+				Web: map[ipn.HostPort]*ipn.WebServerConfig{
+					ipn.HostPort("example.com:443"): {
+						Handlers: map[string]*ipn.HTTPHandler{
+							"/api": {
+								Proxy: "https://10.2.3.4/api",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "valid_config_for_http_proxy",
+			gotSC: `{
+				"TCP": {
+					"80": {
+						"HTTP": true
+					}
+				}}`,
+			wantSC: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80: {
+						HTTP: true,
+					},
+				},
+			},
+		},
+		{
+			name: "config_without_cert_domain",
+			gotSC: `{
+				"TCP": {
+					"443": {
+						"HTTPS": true
+					}
+				},
+				"Web": {
+					"localhost:443": {
+					"Handlers": {
+						"/api": {
+							"Proxy": "https://10.2.3.4/api"
+						}}}}}`,
+			certDomain: "",
+			wantErr:    false,
+			wantSC: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {
+						HTTPS: true,
+					},
+				},
+				Web: map[ipn.HostPort]*ipn.WebServerConfig{
+					ipn.HostPort("localhost:443"): {
+						Handlers: map[string]*ipn.HTTPHandler{
+							"/api": {
+								Proxy: "https://10.2.3.4/api",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:    "invalid_json",
+			gotSC:   "invalid json",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "serve-config.json")
+			if err := os.WriteFile(path, []byte(tt.gotSC), 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			got, err := readServeConfig(path, tt.certDomain)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("readServeConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.wantSC) {
+				t.Errorf("readServeConfig() diff (-got +want):\n%s", cmp.Diff(got, tt.wantSC))
+			}
+		})
+	}
+}
+
+type fakeLocalClient struct {
+	*tailscale.LocalClient
+	setServeCalled bool
+}
+
+func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error {
+	m.setServeCalled = true
+	return nil
+}
+
+func TestHasHTTPSEndpoint(t *testing.T) {
+	tests := []struct {
+		name string
+		cfg  *ipn.ServeConfig
+		want bool
+	}{
+		{
+			name: "nil_config",
+			cfg:  nil,
+			want: false,
+		},
+		{
+			name: "empty_config",
+			cfg:  &ipn.ServeConfig{},
+			want: false,
+		},
+		{
+			name: "no_https_endpoints",
+			cfg: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80: {
+						HTTPS: false,
+					},
+				},
+			},
+			want: false,
+		},
+		{
+			name: "has_https_endpoint",
+			cfg: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {
+						HTTPS: true,
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "mixed_endpoints",
+			cfg: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80:  {HTTPS: false},
+					443: {HTTPS: true},
+				},
+			},
+			want: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := hasHTTPSEndpoint(tt.cfg)
+			if got != tt.want {
+				t.Errorf("hasHTTPSEndpoint() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/containerboot/services.go

@@ -0,0 +1,571 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/ipn"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubeclient"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/linuxfw"
+	"tailscale.com/util/mak"
+)
+
+const tailscaleTunInterface = "tailscale0"
+
+// This file contains functionality to run containerboot as a proxy that can
+// route cluster traffic to one or more tailnet targets, based on portmapping
+// rules read from a configfile. Currently (9/2024) this is only used for the
+// Kubernetes operator egress proxies.
+
+// egressProxy knows how to configure firewall rules to route cluster traffic to
+// one or more tailnet services.
+type egressProxy struct {
+	cfgPath string // path to egress service config file
+
+	nfr linuxfw.NetfilterRunner // never nil
+
+	kc          kubeclient.Client // never nil
+	stateSecret string            // name of the kube state Secret
+
+	netmapChan chan ipn.Notify // chan to receive netmap updates on
+
+	podIPv4 string // never empty string, currently only IPv4 is supported
+
+	// tailnetFQDNs is the egress service FQDN to tailnet IP mappings that
+	// were last used to configure firewall rules for this proxy.
+	// TODO(irbekrm): target addresses are also stored in the state Secret.
+	// Evaluate whether we should retrieve them from there and not store in
+	// memory at all.
+	targetFQDNs map[string][]netip.Prefix
+
+	// used to configure firewall rules.
+	tailnetAddrs []netip.Prefix
+}
+
+// run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when:
+// - the mounted egress config has changed
+// - the proxy's tailnet IP addresses have changed
+// - tailnet IPs have changed for any backend targets specified by tailnet FQDN
+func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	// TODO (irbekrm): take a look if this can be pulled into a single func
+	// shared with serve config loader.
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil {
+			return fmt.Errorf("failed to add fsnotify watch: %w", err)
+		}
+		eventChan = w.Events
+	}
+
+	if err := ep.sync(ctx, n); err != nil {
+		return err
+	}
+	for {
+		var err error
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-tickChan:
+			err = ep.sync(ctx, n)
+		case <-eventChan:
+			log.Printf("config file change detected, ensuring firewall config is up to date...")
+			err = ep.sync(ctx, n)
+		case n = <-ep.netmapChan:
+			shouldResync := ep.shouldResync(n)
+			if shouldResync {
+				log.Printf("netmap change detected, ensuring firewall config is up to date...")
+				err = ep.sync(ctx, n)
+			}
+		}
+		if err != nil {
+			return fmt.Errorf("error syncing egress service config: %w", err)
+		}
+	}
+}
+
+// sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if
+// any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current
+// firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such
+// as failed firewall update
+func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error {
+	cfgs, err := ep.getConfigs()
+	if err != nil {
+		return fmt.Errorf("error retrieving egress service configs: %w", err)
+	}
+	status, err := ep.getStatus(ctx)
+	if err != nil {
+		return fmt.Errorf("error retrieving current egress proxy status: %w", err)
+	}
+	newStatus, err := ep.syncEgressConfigs(cfgs, status, n)
+	if err != nil {
+		return fmt.Errorf("error syncing egress service configs: %w", err)
+	}
+	if !servicesStatusIsEqual(newStatus, status) {
+		if err := ep.setStatus(ctx, newStatus, n); err != nil {
+			return fmt.Errorf("error setting egress proxy status: %w", err)
+		}
+	}
+	return nil
+}
+
+// addrsHaveChanged returns true if the provided netmap update contains tailnet address change for this proxy node.
+// Netmap must not be nil.
+func (ep *egressProxy) addrsHaveChanged(n ipn.Notify) bool {
+	return !reflect.DeepEqual(ep.tailnetAddrs, n.NetMap.SelfNode.Addresses())
+}
+
+// syncEgressConfigs adds and deletes firewall rules to match the desired
+// configuration. It uses the provided status to determine what is currently
+// applied and updates the status after a successful sync.
+func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *egressservices.Status, n ipn.Notify) (*egressservices.Status, error) {
+	if !(wantsServicesConfigured(cfgs) || hasServicesConfigured(status)) {
+		return nil, nil
+	}
+
+	// Delete unnecessary services.
+	if err := ep.deleteUnnecessaryServices(cfgs, status); err != nil {
+		return nil, fmt.Errorf("error deleting services: %w", err)
+
+	}
+	newStatus := &egressservices.Status{}
+	if !wantsServicesConfigured(cfgs) {
+		return newStatus, nil
+	}
+
+	// Add new services, update rules for any that have changed.
+	rulesPerSvcToAdd := make(map[string][]rule, 0)
+	rulesPerSvcToDelete := make(map[string][]rule, 0)
+	for svcName, cfg := range *cfgs {
+		tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, n)
+		if err != nil {
+			return nil, fmt.Errorf("error determining tailnet target IPs: %w", err)
+		}
+		rulesToAdd, rulesToDelete, err := updatesForCfg(svcName, cfg, status, tailnetTargetIPs)
+		if err != nil {
+			return nil, fmt.Errorf("error validating service changes: %v", err)
+		}
+		log.Printf("syncegressservices: looking at svc %s rulesToAdd %d rulesToDelete %d", svcName, len(rulesToAdd), len(rulesToDelete))
+		if len(rulesToAdd) != 0 {
+			mak.Set(&rulesPerSvcToAdd, svcName, rulesToAdd)
+		}
+		if len(rulesToDelete) != 0 {
+			mak.Set(&rulesPerSvcToDelete, svcName, rulesToDelete)
+		}
+		if len(rulesToAdd) != 0 || ep.addrsHaveChanged(n) {
+			// For each tailnet target, set up SNAT from the local tailnet device address of the matching
+			// family.
+			for _, t := range tailnetTargetIPs {
+				var local netip.Addr
+				for _, pfx := range n.NetMap.SelfNode.Addresses().All() {
+					if !pfx.IsSingleIP() {
+						continue
+					}
+					if pfx.Addr().Is4() != t.Is4() {
+						continue
+					}
+					local = pfx.Addr()
+					break
+				}
+				if !local.IsValid() {
+					return nil, fmt.Errorf("no valid local IP: %v", local)
+				}
+				if err := ep.nfr.EnsureSNATForDst(local, t); err != nil {
+					return nil, fmt.Errorf("error setting up SNAT rule: %w", err)
+				}
+			}
+		}
+		// Update the status. Status will be written back to the state Secret by the caller.
+		mak.Set(&newStatus.Services, svcName, &egressservices.ServiceStatus{TailnetTargetIPs: tailnetTargetIPs, TailnetTarget: cfg.TailnetTarget, Ports: cfg.Ports})
+	}
+
+	// Actually apply the firewall rules.
+	if err := ensureRulesAdded(rulesPerSvcToAdd, ep.nfr); err != nil {
+		return nil, fmt.Errorf("error adding rules: %w", err)
+	}
+	if err := ensureRulesDeleted(rulesPerSvcToDelete, ep.nfr); err != nil {
+		return nil, fmt.Errorf("error deleting rules: %w", err)
+	}
+
+	return newStatus, nil
+}
+
+// updatesForCfg calculates any rules that need to be added or deleted for an individucal egress service config.
+func updatesForCfg(svcName string, cfg egressservices.Config, status *egressservices.Status, tailnetTargetIPs []netip.Addr) ([]rule, []rule, error) {
+	rulesToAdd := make([]rule, 0)
+	rulesToDelete := make([]rule, 0)
+	currentConfig, ok := lookupCurrentConfig(svcName, status)
+
+	// If no rules for service are present yet, add them all.
+	if !ok {
+		for _, t := range tailnetTargetIPs {
+			for ports := range cfg.Ports {
+				log.Printf("syncegressservices: svc %s adding port %v", svcName, ports)
+				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: t})
+			}
+		}
+		return rulesToAdd, rulesToDelete, nil
+	}
+
+	// If there are no backend targets available, delete any currently configured rules.
+	if len(tailnetTargetIPs) == 0 {
+		log.Printf("tailnet target for egress service %s does not have any backend addresses, deleting all rules", svcName)
+		for _, ip := range currentConfig.TailnetTargetIPs {
+			for ports := range currentConfig.Ports {
+				rulesToDelete = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
+			}
+		}
+		return rulesToAdd, rulesToDelete, nil
+	}
+
+	// If there are rules present for backend targets that no longer match, delete them.
+	for _, ip := range currentConfig.TailnetTargetIPs {
+		var found bool
+		for _, wantsIP := range tailnetTargetIPs {
+			if reflect.DeepEqual(ip, wantsIP) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			for ports := range currentConfig.Ports {
+				rulesToDelete = append(rulesToDelete, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
+			}
+		}
+	}
+
+	// Sync rules for the currently wanted backend targets.
+	for _, ip := range tailnetTargetIPs {
+
+		// If the backend target is not yet present in status, add all rules.
+		var found bool
+		for _, gotIP := range currentConfig.TailnetTargetIPs {
+			if reflect.DeepEqual(ip, gotIP) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			for ports := range cfg.Ports {
+				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
+			}
+			continue
+		}
+
+		// If the backend target is present in status, check that the
+		// currently applied rules are up to date.
+
+		// Delete any current portmappings that are no longer present in config.
+		for port := range currentConfig.Ports {
+			if _, ok := cfg.Ports[port]; ok {
+				continue
+			}
+			rulesToDelete = append(rulesToDelete, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
+		}
+
+		// Add any new portmappings.
+		for port := range cfg.Ports {
+			if _, ok := currentConfig.Ports[port]; ok {
+				continue
+			}
+			rulesToAdd = append(rulesToAdd, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
+		}
+	}
+	return rulesToAdd, rulesToDelete, nil
+}
+
+// deleteUnneccessaryServices ensure that any services found on status, but not
+// present in config are deleted.
+func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, status *egressservices.Status) error {
+	if !hasServicesConfigured(status) {
+		return nil
+	}
+	if !wantsServicesConfigured(cfgs) {
+		for svcName, svc := range status.Services {
+			log.Printf("service %s is no longer required, deleting", svcName)
+			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
+				return fmt.Errorf("error deleting service %s: %w", svcName, err)
+			}
+		}
+		return nil
+	}
+
+	for svcName, svc := range status.Services {
+		if _, ok := (*cfgs)[svcName]; !ok {
+			log.Printf("service %s is no longer required, deleting", svcName)
+			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
+				return fmt.Errorf("error deleting service %s: %w", svcName, err)
+			}
+			// TODO (irbekrm): also delete the SNAT rule here
+		}
+	}
+	return nil
+}
+
+// getConfigs gets the mounted egress service configuration.
+func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) {
+	j, err := os.ReadFile(ep.cfgPath)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(j) == 0 || string(j) == "" {
+		return nil, nil
+	}
+	cfg := &egressservices.Configs{}
+	if err := json.Unmarshal(j, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// getStatus gets the current status of the configured firewall. The current
+// status is stored in state Secret. Returns nil status if no status that
+// applies to the current proxy Pod was found. Uses the Pod IP to determine if a
+// status found in the state Secret applies to this proxy Pod.
+func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, error) {
+	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving state secret: %w", err)
+	}
+	status := &egressservices.Status{}
+	raw, ok := secret.Data[egressservices.KeyEgressServices]
+	if !ok {
+		return nil, nil
+	}
+	if err := json.Unmarshal([]byte(raw), status); err != nil {
+		return nil, fmt.Errorf("error unmarshalling previous config: %w", err)
+	}
+	if reflect.DeepEqual(status.PodIPv4, ep.podIPv4) {
+		return status, nil
+	}
+	return nil, nil
+}
+
+// setStatus writes egress proxy's currently configured firewall to the state
+// Secret and updates proxy's tailnet addresses.
+func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, n ipn.Notify) error {
+	// Pod IP is used to determine if a stored status applies to THIS proxy Pod.
+	if status == nil {
+		status = &egressservices.Status{}
+	}
+	status.PodIPv4 = ep.podIPv4
+	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
+	if err != nil {
+		return fmt.Errorf("error retrieving state Secret: %w", err)
+	}
+	bs, err := json.Marshal(status)
+	if err != nil {
+		return fmt.Errorf("error marshalling service config: %w", err)
+	}
+	secret.Data[egressservices.KeyEgressServices] = bs
+	patch := kubeclient.JSONPatch{
+		Op:    "replace",
+		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
+		Value: bs,
+	}
+	if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil {
+		return fmt.Errorf("error patching state Secret: %w", err)
+	}
+	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
+	return nil
+}
+
+// tailnetTargetIPsForSvc returns the tailnet IPs to which traffic for this
+// egress service should be proxied. The egress service can be configured by IP
+// or by FQDN. If it's configured by IP, just return that. If it's configured by
+// FQDN, resolve the FQDN and return the resolved IPs. It checks if the
+// netfilter runner supports IPv6 NAT and skips any IPv6 addresses if it
+// doesn't.
+func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.Notify) (addrs []netip.Addr, err error) {
+	if svc.TailnetTarget.IP != "" {
+		addr, err := netip.ParseAddr(svc.TailnetTarget.IP)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing tailnet target IP: %w", err)
+		}
+		if addr.Is6() && !ep.nfr.HasIPV6NAT() {
+			log.Printf("tailnet target is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode. This will probably not work.")
+			return addrs, nil
+		}
+		return []netip.Addr{addr}, nil
+	}
+
+	if svc.TailnetTarget.FQDN == "" {
+		return nil, errors.New("unexpected egress service config- neither tailnet target IP nor FQDN is set")
+	}
+	if n.NetMap == nil {
+		log.Printf("netmap is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN)
+		return addrs, nil
+	}
+	var (
+		node      tailcfg.NodeView
+		nodeFound bool
+	)
+	for _, nn := range n.NetMap.Peers {
+		if equalFQDNs(nn.Name(), svc.TailnetTarget.FQDN) {
+			node = nn
+			nodeFound = true
+			break
+		}
+	}
+	if nodeFound {
+		for _, addr := range node.Addresses().AsSlice() {
+			if addr.Addr().Is6() && !ep.nfr.HasIPV6NAT() {
+				log.Printf("tailnet target %v is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode, skipping.", addr.Addr().String())
+				continue
+			}
+			addrs = append(addrs, addr.Addr())
+		}
+		// Egress target endpoints configured via FQDN are stored, so
+		// that we can determine if a netmap update should trigger a
+		// resync.
+		mak.Set(&ep.targetFQDNs, svc.TailnetTarget.FQDN, node.Addresses().AsSlice())
+	}
+	return addrs, nil
+}
+
+// shouldResync parses netmap update and returns true if the update contains
+// changes for which the egress proxy's firewall should be reconfigured.
+func (ep *egressProxy) shouldResync(n ipn.Notify) bool {
+	if n.NetMap == nil {
+		return false
+	}
+
+	// If proxy's tailnet addresses have changed, resync.
+	if !reflect.DeepEqual(n.NetMap.SelfNode.Addresses().AsSlice(), ep.tailnetAddrs) {
+		log.Printf("node addresses have changed, trigger egress config resync")
+		ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
+		return true
+	}
+
+	// If the IPs for any of the egress services configured via FQDN have
+	// changed, resync.
+	for fqdn, ips := range ep.targetFQDNs {
+		for _, nn := range n.NetMap.Peers {
+			if equalFQDNs(nn.Name(), fqdn) {
+				if !reflect.DeepEqual(ips, nn.Addresses().AsSlice()) {
+					log.Printf("backend addresses for egress target %q have changed old IPs %v, new IPs %v trigger egress config resync", nn.Name(), ips, nn.Addresses().AsSlice())
+				}
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// ensureServiceDeleted ensures that any rules for an egress service are removed
+// from the firewall configuration.
+func ensureServiceDeleted(svcName string, svc *egressservices.ServiceStatus, nfr linuxfw.NetfilterRunner) error {
+
+	// Note that the portmap is needed for iptables based firewall only.
+	// Nftables group rules for a service in a chain, so there is no need to
+	// specify individual portmapping based rules.
+	pms := make([]linuxfw.PortMap, 0)
+	for pm := range svc.Ports {
+		pms = append(pms, linuxfw.PortMap{MatchPort: pm.MatchPort, TargetPort: pm.TargetPort, Protocol: pm.Protocol})
+	}
+
+	if err := nfr.DeleteSvc(svcName, tailscaleTunInterface, svc.TailnetTargetIPs, pms); err != nil {
+		return fmt.Errorf("error deleting service %s: %w", svcName, err)
+	}
+	return nil
+}
+
+// ensureRulesAdded ensures that all portmapping rules are added to the firewall
+// configuration. For any rules that already exist, calling this function is a
+// no-op. In case of nftables, a service consists of one or two (one per IP
+// family) chains that conain the portmapping rules for the service and the
+// chains as needed when this function is called.
+func ensureRulesAdded(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
+	for svc, rules := range rulesPerSvc {
+		for _, rule := range rules {
+			log.Printf("ensureRulesAdded svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
+			if err := nfr.EnsurePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
+				return fmt.Errorf("error ensuring rule: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+// ensureRulesDeleted ensures that the given rules are deleted from the firewall
+// configuration. For any rules that do not exist, calling this funcion is a
+// no-op.
+func ensureRulesDeleted(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
+	for svc, rules := range rulesPerSvc {
+		for _, rule := range rules {
+			log.Printf("ensureRulesDeleted svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
+			if err := nfr.DeletePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
+				return fmt.Errorf("error deleting rule: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func lookupCurrentConfig(svcName string, status *egressservices.Status) (*egressservices.ServiceStatus, bool) {
+	if status == nil || len(status.Services) == 0 {
+		return nil, false
+	}
+	c, ok := status.Services[svcName]
+	return c, ok
+}
+
+func equalFQDNs(s, s1 string) bool {
+	s, _ = strings.CutSuffix(s, ".")
+	s1, _ = strings.CutSuffix(s1, ".")
+	return strings.EqualFold(s, s1)
+}
+
+// rule contains configuration for an egress proxy firewall rule.
+type rule struct {
+	containerPort uint16     // port to match incoming traffic
+	tailnetPort   uint16     // tailnet service port
+	tailnetIP     netip.Addr // tailnet service IP
+	protocol      string
+}
+
+func wantsServicesConfigured(cfgs *egressservices.Configs) bool {
+	return cfgs != nil && len(*cfgs) != 0
+}
+
+func hasServicesConfigured(status *egressservices.Status) bool {
+	return status != nil && len(status.Services) != 0
+}
+
+func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
+	if st == nil && st1 == nil {
+		return true
+	}
+	if st == nil || st1 == nil {
+		return false
+	}
+	st.PodIPv4 = ""
+	st1.PodIPv4 = ""
+	return reflect.DeepEqual(*st, *st1)
+}

cmd/containerboot/services_test.go

@@ -0,0 +1,175 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"net/netip"
+	"reflect"
+	"testing"
+
+	"tailscale.com/kube/egressservices"
+)
+
+func Test_updatesForSvc(t *testing.T) {
+	tailnetIPv4, tailnetIPv6 := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
+	tailnetIPv4_1, tailnetIPv6_1 := netip.MustParseAddr("100.88.88.88"), netip.MustParseAddr("fd7a:115c:a1e0::4101:512f")
+	ports := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {}}
+	ports1 := map[egressservices.PortMap]struct{}{{Protocol: "udp", MatchPort: 4004, TargetPort: 53}: {}}
+	ports2 := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {},
+		{Protocol: "tcp", MatchPort: 4005, TargetPort: 443}: {}}
+	fqdnSpec := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
+		Ports:         ports,
+	}
+	fqdnSpec1 := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
+		Ports:         ports1,
+	}
+	fqdnSpec2 := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+		Ports:         ports,
+	}
+	fqdnSpec3 := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+		Ports:         ports2,
+	}
+	r := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4}
+	r1 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6}
+	r2 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv4}
+	r3 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv6}
+	r4 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4_1}
+	r5 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6_1}
+	r6 := rule{containerPort: 4005, tailnetPort: 443, protocol: "tcp", tailnetIP: tailnetIPv4}
+
+	tests := []struct {
+		name              string
+		svcName           string
+		tailnetTargetIPs  []netip.Addr
+		podIP             string
+		spec              egressservices.Config
+		status            *egressservices.Status
+		wantRulesToAdd    []rule
+		wantRulesToDelete []rule
+	}{
+		{
+			name:              "add_fqdn_svc_that_does_not_yet_exist",
+			svcName:           "test",
+			tailnetTargetIPs:  []netip.Addr{tailnetIPv4, tailnetIPv6},
+			spec:              fqdnSpec,
+			status:            &egressservices.Status{},
+			wantRulesToAdd:    []rule{r, r1},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "fqdn_svc_already_exists",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+			spec:             fqdnSpec,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "fqdn_svc_already_exists_add_port_remove_port",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+			spec:             fqdnSpec1,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{r2, r3},
+			wantRulesToDelete: []rule{r, r1},
+		},
+		{
+			name:             "fqdn_svc_already_exists_change_fqdn_backend_ips",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4_1, tailnetIPv6_1},
+			spec:             fqdnSpec,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{r4, r5},
+			wantRulesToDelete: []rule{r, r1},
+		},
+		{
+			name:              "add_ip_service",
+			svcName:           "test",
+			tailnetTargetIPs:  []netip.Addr{tailnetIPv4},
+			spec:              fqdnSpec2,
+			status:            &egressservices.Status{},
+			wantRulesToAdd:    []rule{r},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "add_ip_service_already_exists",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
+			spec:             fqdnSpec2,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
+					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "ip_service_add_port",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
+			spec:             fqdnSpec3,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
+					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{r6},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "ip_service_delete_port",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
+			spec:             fqdnSpec,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
+					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+					Ports:            ports2,
+				}}},
+			wantRulesToAdd:    []rule{},
+			wantRulesToDelete: []rule{r6},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotRulesToAdd, gotRulesToDelete, err := updatesForCfg(tt.svcName, tt.spec, tt.status, tt.tailnetTargetIPs)
+			if err != nil {
+				t.Errorf("updatesForSvc() unexpected error %v", err)
+				return
+			}
+			if !reflect.DeepEqual(gotRulesToAdd, tt.wantRulesToAdd) {
+				t.Errorf("updatesForSvc() got rulesToAdd = \n%v\n want rulesToAdd \n%v", gotRulesToAdd, tt.wantRulesToAdd)
+			}
+			if !reflect.DeepEqual(gotRulesToDelete, tt.wantRulesToDelete) {
+				t.Errorf("updatesForSvc() got rulesToDelete = \n%v\n want rulesToDelete \n%v", gotRulesToDelete, tt.wantRulesToDelete)
+			}
+		})
+	}
+}

cmd/containerboot/settings.go

@@ -0,0 +1,360 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"tailscale.com/ipn/conffile"
+	"tailscale.com/kube/kubeclient"
+)
+
+// settings is all the configuration for containerboot.
+type settings struct {
+	AuthKey  string
+	Hostname string
+	Routes   *string
+	// ProxyTargetIP is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. This is typically a
+	// Tailscale IP.
+	TailnetTargetIP string
+	// TailnetTargetFQDN is an MagicDNS name to which all incoming
+	// non-Tailscale traffic should be proxied. This must be a full Tailnet
+	// node FQDN.
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	// Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters
+	PodIP               string
+	PodIPv4             string
+	PodIPv6             string
+	PodUID              string
+	HealthCheckAddrPort string
+	LocalAddrPort       string
+	MetricsEnabled      bool
+	HealthCheckEnabled  bool
+	DebugAddrPort       string
+	EgressSvcsCfgPath   string
+}
+
+func configFromEnv() (*settings, error) {
+	cfg := &settings{
+		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
+		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
+		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
+		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
+		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
+		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
+		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
+		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
+		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
+		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
+		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
+		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
+		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
+		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
+		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
+		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
+		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
+		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
+		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
+		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
+		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
+		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
+		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
+		LocalAddrPort:                         defaultEnv("TS_LOCAL_ADDR_PORT", "[::]:9002"),
+		MetricsEnabled:                        defaultBool("TS_ENABLE_METRICS", false),
+		HealthCheckEnabled:                    defaultBool("TS_ENABLE_HEALTH_CHECK", false),
+		DebugAddrPort:                         defaultEnv("TS_DEBUG_ADDR_PORT", ""),
+		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
+		PodUID:                                defaultEnv("POD_UID", ""),
+	}
+	podIPs, ok := os.LookupEnv("POD_IPS")
+	if ok {
+		ips := strings.Split(podIPs, ",")
+		if len(ips) > 2 {
+			return nil, fmt.Errorf("POD_IPs can contain at most 2 IPs, got %d (%v)", len(ips), ips)
+		}
+		for _, ip := range ips {
+			parsed, err := netip.ParseAddr(ip)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing IP address %s: %w", ip, err)
+			}
+			if parsed.Is4() {
+				cfg.PodIPv4 = parsed.String()
+				continue
+			}
+			cfg.PodIPv6 = parsed.String()
+		}
+	}
+	if err := cfg.validate(); err != nil {
+		return nil, fmt.Errorf("invalid configuration: %v", err)
+	}
+	return cfg, nil
+}
+
+func (s *settings) validate() error {
+	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
+		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
+		}
+	}
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
+	if s.TailnetTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
+		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
+	}
+	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
+	if s.HealthCheckAddrPort != "" {
+		log.Printf("[warning] TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0. Please use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT instead.")
+		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_HEALTHCHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+		}
+	}
+	if s.localMetricsEnabled() || s.localHealthEnabled() {
+		if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
+		}
+	}
+	if s.DebugAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.DebugAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_DEBUG_ADDR_PORT value %q: %w", s.DebugAddrPort, err)
+		}
+	}
+	if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" {
+		return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT")
+	}
+	if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
+		return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
+	}
+	return nil
+}
+
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context, kc *kubeClient) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+	kc.canPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil {
+		if !kubeclient.IsNotFoundErr(err) {
+			return fmt.Errorf("getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+		}
+
+		if !canCreate {
+			return fmt.Errorf("tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+				"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+				"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+				"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+		}
+	}
+
+	// Return early if we already have an auth key.
+	if cfg.AuthKey != "" || isOneStepConfig(cfg) {
+		return nil
+	}
+
+	if s == nil {
+		log.Print("TS_AUTHKEY not provided and state Secret does not exist, login will be interactive if needed.")
+		return nil
+	}
+
+	keyBytes, _ := s.Data["authkey"]
+	key := string(keyBytes)
+
+	if key != "" {
+		// Enforce that we must be able to patch out the authkey after
+		// authenticating if you want to use this feature. This avoids
+		// us having to deal with the case where we might leave behind
+		// an unnecessary reusable authkey in a secret, like a rake in
+		// the grass.
+		if !cfg.KubernetesCanPatch {
+			return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the Secret to manage the authkey.")
+		}
+		cfg.AuthKey = key
+	}
+
+	log.Print("No authkey found in state Secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+
+	return nil
+}
+
+// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
+// in two steps and login should only happen once.
+// Step 1: run 'tailscaled'
+// Step 2):
+// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
+// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
+func isTwoStepConfigAuthOnce(cfg *settings) bool {
+	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
+// in two steps and we should log in every time it starts.
+// Step 1: run 'tailscaled'
+// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
+func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
+	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isOneStepConfig returns true if the Tailscale node should always be ran and
+// configured in a single step by running 'tailscaled <config opts>'
+func isOneStepConfig(cfg *settings) bool {
+	return cfg.TailscaledConfigFilePath != ""
+}
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != ""
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
+func (cfg *settings) localMetricsEnabled() bool {
+	return cfg.LocalAddrPort != "" && cfg.MetricsEnabled
+}
+
+func (cfg *settings) localHealthEnabled() bool {
+	return cfg.LocalAddrPort != "" && cfg.HealthCheckEnabled
+}
+
+// defaultEnv returns the value of the given envvar name, or defVal if
+// unset.
+func defaultEnv(name, defVal string) string {
+	if v, ok := os.LookupEnv(name); ok {
+		return v
+	}
+	return defVal
+}
+
+// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being set to empty string vs unset.
+func defaultEnvStringPointer(name string) *string {
+	if v, ok := os.LookupEnv(name); ok {
+		return &v
+	}
+	return nil
+}
+
+// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being explicitly set to false vs unset.
+func defaultEnvBoolPointer(name string) *bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return nil
+	}
+	return &ret
+}
+
+func defaultEnvs(names []string, defVal string) string {
+	for _, name := range names {
+		if v, ok := os.LookupEnv(name); ok {
+			return v
+		}
+	}
+	return defVal
+}
+
+// defaultBool returns the boolean value of the given envvar name, or
+// defVal if unset or not a bool.
+func defaultBool(name string, defVal bool) bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return defVal
+	}
+	return ret
+}

cmd/containerboot/tailscaled.go

@@ -0,0 +1,238 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/tailscale"
+)
+
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+	args := tailscaledArgs(cfg)
+	// tailscaled runs without context, since it needs to persist
+	// beyond the startup timeout in ctx.
+	cmd := exec.Command("tailscaled", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	log.Printf("Starting tailscaled")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
+	}
+
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
+	log.Printf("Waiting for tailscaled socket")
+	for {
+		if ctx.Err() != nil {
+			log.Fatalf("Timed out waiting for tailscaled socket")
+		}
+		_, err := os.Stat(cfg.Socket)
+		if errors.Is(err, fs.ErrNotExist) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		} else if err != nil {
+			log.Fatalf("Waiting for tailscaled socket: %v", err)
+		}
+		break
+	}
+
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process, nil
+}
+
+// tailscaledArgs uses cfg to construct the argv for tailscaled.
+func tailscaledArgs(cfg *settings) []string {
+	args := []string{"--socket=" + cfg.Socket}
+	switch {
+	case cfg.InKubernetes && cfg.KubeSecret != "":
+		args = append(args, "--state=kube:"+cfg.KubeSecret)
+		if cfg.StateDir == "" {
+			cfg.StateDir = "/tmp"
+		}
+		fallthrough
+	case cfg.StateDir != "":
+		args = append(args, "--statedir="+cfg.StateDir)
+	default:
+		args = append(args, "--state=mem:", "--statedir=/tmp")
+	}
+
+	if cfg.UserspaceMode {
+		args = append(args, "--tun=userspace-networking")
+	} else if err := ensureTunFile(cfg.Root); err != nil {
+		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
+	}
+
+	if cfg.SOCKSProxyAddr != "" {
+		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
+	}
+	if cfg.HTTPProxyAddr != "" {
+		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
+	}
+	if cfg.TailscaledConfigFilePath != "" {
+		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
+	}
+	// Once enough proxy versions have been released for all the supported
+	// versions to understand this cfg setting, the operator can stop
+	// setting TS_TAILSCALED_EXTRA_ARGS for the debug flag.
+	if cfg.DebugAddrPort != "" && !strings.Contains(cfg.DaemonExtraArgs, cfg.DebugAddrPort) {
+		args = append(args, "--debug="+cfg.DebugAddrPort)
+	}
+	if cfg.DaemonExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
+	}
+	return args
+}
+
+// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
+// if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleUp(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "up"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale up'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale up failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state and only if TS_AUTH_ONCE is set.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	log.Printf("Running 'tailscale set'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale set failed: %v", err)
+	}
+	return nil
+}
+
+func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscale.LocalClient, errCh chan<- error) {
+	var (
+		tickChan          <-chan time.Time
+		tailscaledCfgDir  = filepath.Dir(path)
+		prevTailscaledCfg []byte
+	)
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Printf("tailscaled config watch: failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(tailscaledCfgDir); err != nil {
+			errCh <- fmt.Errorf("failed to add fsnotify watch: %w", err)
+			return
+		}
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		errCh <- fmt.Errorf("error reading configfile: %w", err)
+		return
+	}
+	prevTailscaledCfg = b
+	// kubelet mounts Secrets to Pods using a series of symlinks, one of
+	// which is <mount-dir>/..data that Kubernetes recommends consumers to
+	// use if they need to monitor changes
+	// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
+	const kubeletMountedCfg = "..data"
+	toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedCfg)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case err := <-w.Errors:
+			errCh <- fmt.Errorf("watcher error: %w", err)
+			return
+		case <-tickChan:
+		case event := <-w.Events:
+			if event.Name != toWatch {
+				continue
+			}
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			errCh <- fmt.Errorf("error reading configfile: %w", err)
+			return
+		}
+		// For some proxy types the mounted volume also contains tailscaled state and other files. We
+		// don't want to reload config unnecessarily on unrelated changes to these files.
+		if reflect.DeepEqual(b, prevTailscaledCfg) {
+			continue
+		}
+		prevTailscaledCfg = b
+		log.Printf("tailscaled config watch: ensuring that config is up to date")
+		ok, err := lc.ReloadConfig(ctx)
+		if err != nil {
+			errCh <- fmt.Errorf("error reloading tailscaled config: %w", err)
+			return
+		}
+		if ok {
+			log.Printf("tailscaled config watch: config was reloaded")
+		}
+	}
+}

cmd/derper/bootstrap_dns_test.go

@@ -20,10 +20,10 @@ import (
 )
 
 func BenchmarkHandleBootstrapDNS(b *testing.B) {
-	tstest.Replace(b, bootstrapDNS, "log.tailscale.io,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
+	tstest.Replace(b, bootstrapDNS, "log.tailscale.com,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
 	refreshBootstrapDNS()
 	w := new(bitbucketResponseWriter)
-	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.io"), nil)
+	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.com"), nil)
 	b.ReportAllocs()
 	b.ResetTimer()
 	b.RunParallel(func(b *testing.PB) {
@@ -63,7 +63,7 @@ func TestUnpublishedDNS(t *testing.T) {
 	nettest.SkipIfNoNetwork(t)
 
 	const published = "login.tailscale.com"
-	const unpublished = "log.tailscale.io"
+	const unpublished = "log.tailscale.com"
 
 	prev1, prev2 := *bootstrapDNS, *unpublishedDNS
 	*bootstrapDNS = published
@@ -119,18 +119,18 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 
 	unpublishedDNSCache.Store(&dnsEntryMap{
 		IPs: map[string][]net.IP{
-			"log.tailscale.io":           {},
+			"log.tailscale.com":          {},
 			"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
 		},
 		Percent: map[string]float64{
-			"log.tailscale.io":           1.0,
+			"log.tailscale.com":          1.0,
 			"controlplane.tailscale.com": 1.0,
 		},
 	})
 
 	t.Run("CacheMiss", func(t *testing.T) {
 		// One domain in map but empty, one not in map at all
-		for _, q := range []string{"log.tailscale.io", "login.tailscale.com"} {
+		for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} {
 			resetMetrics()
 			ips := getBootstrapDNS(t, q)
 

cmd/derper/cert.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"path/filepath"
 	"regexp"
@@ -53,8 +54,9 @@ func certProviderByCertMode(mode, dir, hostname string) (certProvider, error) {
 }
 
 type manualCertManager struct {
-	cert     *tls.Certificate
-	hostname string
+	cert       *tls.Certificate
+	hostname   string // hostname or IP address of server
+	noHostname bool   // whether hostname is an IP address
 }
 
 // NewManualCertManager returns a cert provider which read certificate by given hostname on create.
@@ -74,7 +76,11 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	if err := x509Cert.VerifyHostname(hostname); err != nil {
 		return nil, fmt.Errorf("cert invalid for hostname %q: %w", hostname, err)
 	}
-	return &manualCertManager{cert: &cert, hostname: hostname}, nil
+	return &manualCertManager{
+		cert:       &cert,
+		hostname:   hostname,
+		noHostname: net.ParseIP(hostname) != nil,
+	}, nil
 }
 
 func (m *manualCertManager) TLSConfig() *tls.Config {
@@ -88,7 +94,7 @@ func (m *manualCertManager) TLSConfig() *tls.Config {
 }
 
 func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	if hi.ServerName != m.hostname {
+	if hi.ServerName != m.hostname && !m.noHostname {
 		return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
 	}
 

cmd/derper/cert_test.go

@@ -0,0 +1,97 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// Verify that in --certmode=manual mode, we can use a bare IP address
+// as the --hostname and that GetCertificate will return it.
+func TestCertIP(t *testing.T) {
+	dir := t.TempDir()
+	const hostname = "1.2.3.4"
+
+	priv, err := ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ip := net.ParseIP(hostname)
+	if ip == nil {
+		t.Fatalf("invalid IP address %q", hostname)
+	}
+	template := &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"Tailscale Test Corp"},
+		},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(30 * 24 * time.Hour),
+
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IPAddresses:           []net.IP{ip},
+	}
+	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	certOut, err := os.Create(filepath.Join(dir, hostname+".crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		t.Fatalf("Failed to write data to cert.pem: %v", err)
+	}
+	if err := certOut.Close(); err != nil {
+		t.Fatalf("Error closing cert.pem: %v", err)
+	}
+
+	keyOut, err := os.OpenFile(filepath.Join(dir, hostname+".key"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	privBytes, err := x509.MarshalPKCS8PrivateKey(priv)
+	if err != nil {
+		t.Fatalf("Unable to marshal private key: %v", err)
+	}
+	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
+		t.Fatalf("Failed to write data to key.pem: %v", err)
+	}
+	if err := keyOut.Close(); err != nil {
+		t.Fatalf("Error closing key.pem: %v", err)
+	}
+
+	cp, err := certProviderByCertMode("manual", dir, hostname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	back, err := cp.TLSConfig().GetCertificate(&tls.ClientHelloInfo{
+		ServerName: "", // no SNI
+	})
+	if err != nil {
+		t.Fatalf("GetCertificate: %v", err)
+	}
+	if back == nil {
+		t.Fatalf("GetCertificate returned nil")
+	}
+}

cmd/derper/depaware.txt

@@ -27,9 +27,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
-   L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
    L 💣 github.com/mdlayher/netlink                                  from github.com/google/nftables+
@@ -37,11 +35,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
+        github.com/munnerz/goautoneg                                 from github.com/prometheus/common/expfmt
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
         github.com/prometheus/common/expfmt                          from github.com/prometheus/client_golang/prometheus+
-        github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
         github.com/prometheus/common/model                           from github.com/prometheus/client_golang/prometheus+
   LD    github.com/prometheus/procfs                                 from github.com/prometheus/client_golang/prometheus
   LD    github.com/prometheus/procfs/internal/fs                     from github.com/prometheus/procfs
@@ -52,7 +50,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
@@ -87,7 +85,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
-        tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
+     💣 tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
         tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale
         tailscale.com/derp                                           from tailscale.com/cmd/derper+
@@ -99,7 +97,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
@@ -112,9 +112,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
-        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
+        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper
         tailscale.com/paths                                          from tailscale.com/client/tailscale
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
@@ -127,7 +128,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -138,6 +139,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/ipn
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/ipn+
@@ -149,22 +151,29 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/set                                       from tailscale.com/derp+
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/testenv                                   from tailscale.com/util/syspolicy+
+        tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
@@ -175,16 +184,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from tailscale.com/tka
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
-        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -194,7 +204,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
         golang.org/x/sync/errgroup                                   from github.com/mdlayher/socket+
-        golang.org/x/sys/cpu                                         from github.com/josharian/native+
+        golang.org/x/sys/cpu                                         from golang.org/x/crypto/argon2+
   LD    golang.org/x/sys/unix                                        from github.com/google/nftables+
    W    golang.org/x/sys/windows                                     from github.com/dblohm7/wingoes+
    W    golang.org/x/sys/windows/registry                            from github.com/dblohm7/wingoes+
@@ -233,7 +243,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         crypto/tls                                                   from golang.org/x/crypto/acme+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -245,7 +254,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper
+        flag                                                         from tailscale.com/cmd/derper+
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -253,9 +262,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
+        html/template                                                from tailscale.com/cmd/derper
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
-        io/ioutil                                                    from github.com/mitchellh/go-ps+
+   L    io/ioutil                                                    from github.com/mitchellh/go-ps+
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/ipn+
@@ -263,7 +274,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -271,14 +282,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
-   W    os/user                                                      from tailscale.com/util/winutil
+   W    os/user                                                      from tailscale.com/util/winutil+
         path                                                         from github.com/prometheus/client_golang/prometheus/internal+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
@@ -296,7 +307,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
         text/tabwriter                                               from runtime/pprof
+        text/template                                                from html/template
+        text/template/parse                                          from html/template+
         time                                                         from compress/gzip+
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/derper/derper.go

@@ -19,6 +19,7 @@ import (
 	"expvar"
 	"flag"
 	"fmt"
+	"html/template"
 	"io"
 	"log"
 	"math"
@@ -57,12 +58,12 @@ var (
 	configPath  = flag.String("c", "", "config file path")
 	certMode    = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
 	certDir     = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
-	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
+	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks")
 	runSTUN     = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
 	runDERP     = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
 
 	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
-	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
+	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.")
 	bootstrapDNS    = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
 	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
 	verifyClients   = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
@@ -76,6 +77,8 @@ var (
 	tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time")
 	// tcpUserTimeout is intentionally short, so that hung connections are cleaned up promptly. DERPs should be nearby users.
 	tcpUserTimeout = flag.Duration("tcp-user-timeout", 15*time.Second, "TCP user timeout")
+	// tcpWriteTimeout is the timeout for writing to client TCP connections. It does not apply to mesh connections.
+	tcpWriteTimeout = flag.Duration("tcp-write-timeout", derp.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes")
 )
 
 var (
@@ -172,6 +175,7 @@ func main() {
 	s.SetVerifyClient(*verifyClients)
 	s.SetVerifyClientURL(*verifyClientURL)
 	s.SetVerifyClientURLFailOpen(*verifyFailOpen)
+	s.SetTCPWriteTimeout(*tcpWriteTimeout)
 
 	if *meshPSKFile != "" {
 		b, err := os.ReadFile(*meshPSKFile)
@@ -212,25 +216,16 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
 		w.WriteHeader(200)
-		io.WriteString(w, `<html><body>
-<h1>DERP</h1>
-<p>
-  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
-</p>
-<p>
-  Documentation:
-</p>
-<ul>
-  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
-  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
-  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
-</ul>
-`)
-		if !*runDERP {
-			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
-		}
-		if tsweb.AllowDebugAccess(r) {
-			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
+		err := homePageTemplate.Execute(w, templateData{
+			ShowAbuseInfo: validProdHostname.MatchString(*hostname),
+			Disabled:      !*runDERP,
+			AllowDebug:    tsweb.AllowDebugAccess(r),
+		})
+		if err != nil {
+			if r.Context().Err() == nil {
+				log.Printf("homePageTemplate.Execute: %v", err)
+			}
+			return
 		}
 	}))
 	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -468,3 +463,52 @@ func init() {
 		return 0
 	}))
 }
+
+type templateData struct {
+	ShowAbuseInfo bool
+	Disabled      bool
+	AllowDebug    bool
+}
+
+// homePageTemplate renders the home page using [templateData].
+var homePageTemplate = template.Must(template.New("home").Parse(`<html><body>
+<h1>DERP</h1>
+<p>
+  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
+</p>
+
+<p>
+  It provides STUN, interactive connectivity establishment, and relaying of end-to-end encrypted traffic
+  for Tailscale clients.
+</p>
+
+{{if .ShowAbuseInfo }}
+<p>
+  If you suspect abuse, please contact <a href="mailto:security@tailscale.com">security@tailscale.com</a>.
+</p>
+{{end}}
+
+<p>
+  Documentation:
+</p>
+
+<ul>
+{{if .ShowAbuseInfo }}
+  <li><a href="https://tailscale.com/security-policies">Tailscale Security Policies</a></li>
+  <li><a href="https://tailscale.com/tailscale-aup">Tailscale Acceptable Use Policies</a></li>
+{{end}}
+  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
+  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
+  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
+</ul>
+
+{{if .Disabled}}
+<p>Status: <b>disabled</b></p>
+{{end}}
+
+{{if .AllowDebug}}
+<p>Debug info at <a href='/debug/'>/debug/</a>.</p>
+{{end}}
+</body>
+</html>
+`))

cmd/derper/derper_test.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"net/http/httptest"
@@ -107,6 +108,33 @@ func TestDeps(t *testing.T) {
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
 			"tailscale.com/net/packet":           "not needed in derper",
 			"github.com/gaissmai/bart":           "not needed in derper",
+			"database/sql/driver":                "not needed in derper", // previously came in via github.com/google/uuid
 		},
 	}.Check(t)
 }
+
+func TestTemplate(t *testing.T) {
+	buf := &bytes.Buffer{}
+	err := homePageTemplate.Execute(buf, templateData{
+		ShowAbuseInfo: true,
+		Disabled:      true,
+		AllowDebug:    true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	str := buf.String()
+	if !strings.Contains(str, "If you suspect abuse") {
+		t.Error("Output is missing abuse mailto")
+	}
+	if !strings.Contains(str, "Tailscale Security Policies") {
+		t.Error("Output is missing Tailscale Security Policies link")
+	}
+	if !strings.Contains(str, "Status:") {
+		t.Error("Output is missing disabled status")
+	}
+	if !strings.Contains(str, "Debug info") {
+		t.Error("Output is missing debug info")
+	}
+}

cmd/derper/mesh.go

@@ -10,7 +10,6 @@ import (
 	"log"
 	"net"
 	"strings"
-	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -25,15 +24,28 @@ func startMesh(s *derp.Server) error {
 	if !s.HasMeshKey() {
 		return errors.New("--mesh-with requires --mesh-psk-file")
 	}
-	for _, host := range strings.Split(*meshWith, ",") {
-		if err := startMeshWithHost(s, host); err != nil {
+	for _, hostTuple := range strings.Split(*meshWith, ",") {
+		if err := startMeshWithHost(s, hostTuple); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func startMeshWithHost(s *derp.Server, host string) error {
+func startMeshWithHost(s *derp.Server, hostTuple string) error {
+	var host string
+	var dialHost string
+	hostParts := strings.Split(hostTuple, "/")
+	if len(hostParts) > 2 {
+		return fmt.Errorf("too many components in host tuple %q", hostTuple)
+	}
+	host = hostParts[0]
+	if len(hostParts) == 2 {
+		dialHost = hostParts[1]
+	} else {
+		dialHost = hostParts[0]
+	}
+
 	logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
 	netMon := netmon.NewStatic() // good enough for cmd/derper; no need for netns fanciness
 	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf, netMon)
@@ -43,31 +55,20 @@ func startMeshWithHost(s *derp.Server, host string) error {
 	c.MeshKey = s.MeshKey()
 	c.WatchConnectionChanges = true
 
-	// For meshed peers within a region, connect via VPC addresses.
-	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
-		host, port, err := net.SplitHostPort(addr)
-		if err != nil {
-			return nil, err
-		}
+	logf("will dial %q for %q", dialHost, host)
+	if dialHost != host {
 		var d net.Dialer
-		var r net.Resolver
-		if base, ok := strings.CutSuffix(host, ".tailscale.com"); ok && port == "443" {
-			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
-			defer cancel()
-			vpcHost := base + "-vpc.tailscale.com"
-			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
-			if len(ips) > 0 {
-				vpcAddr := net.JoinHostPort(ips[0].String(), port)
-				c, err := d.DialContext(subCtx, network, vpcAddr)
-				if err == nil {
-					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
-					return c, nil
-				}
-				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+		c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
+			_, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				logf("failed to split %q: %v", addr, err)
+				return nil, err
 			}
-		}
-		return d.DialContext(ctx, network, addr)
-	})
+			dialAddr := net.JoinHostPort(dialHost, port)
+			logf("dialing %q instead of %q", dialAddr, addr)
+			return d.DialContext(ctx, network, dialAddr)
+		})
+	}
 
 	add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) }
 	remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) }

cmd/derpprobe/derpprobe.go

@@ -18,17 +18,21 @@ import (
 )
 
 var (
-	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
-	versionFlag  = flag.Bool("version", false, "print version and exit")
-	listen       = flag.String("listen", ":8030", "HTTP listen address")
-	probeOnce    = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
-	spread       = flag.Bool("spread", true, "whether to spread probing over time")
-	interval     = flag.Duration("interval", 15*time.Second, "probe interval")
-	meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
-	stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
-	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
-	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
-	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	derpMapURL         = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
+	versionFlag        = flag.Bool("version", false, "print version and exit")
+	listen             = flag.String("listen", ":8030", "HTTP listen address")
+	probeOnce          = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
+	spread             = flag.Bool("spread", true, "whether to spread probing over time")
+	interval           = flag.Duration("interval", 15*time.Second, "probe interval")
+	meshInterval       = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
+	stunInterval       = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
+	tlsInterval        = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
+	bwInterval         = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
+	bwSize             = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	bwTUNIPv4Address   = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP; we will use a /30 subnet including this IP address")
+	qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second")
+	qdPacketTimeout    = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings")
+	regionCodeOrID     = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -43,9 +47,13 @@ func main() {
 		prober.WithMeshProbing(*meshInterval),
 		prober.WithSTUNProbing(*stunInterval),
 		prober.WithTLSProbing(*tlsInterval),
+		prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout),
 	}
 	if *bwInterval > 0 {
-		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
+		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address))
+	}
+	if *regionCodeOrID != "" {
+		opts = append(opts, prober.WithRegionCodeOrID(*regionCodeOrID))
 	}
 	dp, err := prober.DERP(p, *derpMapURL, opts...)
 	if err != nil {
@@ -75,6 +83,11 @@ func main() {
 		prober.WithPageLink("Prober metrics", "/debug/varz"),
 		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
 	), tsweb.HandlerOptions{Logf: log.Printf}))
+	mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok\n"))
+	}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }
@@ -97,7 +110,7 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 			// Do not show probes that have not finished yet.
 			continue
 		}
-		if i.Result {
+		if i.Status == prober.ProbeStatusSucceeded {
 			o.addGoodf("%s: %s", p, i.Latency)
 		} else {
 			o.addBadf("%s: %s", p, i.Error)

cmd/get-authkey/main.go

@@ -46,11 +46,11 @@ func main() {
 		ClientID:     clientID,
 		ClientSecret: clientSecret,
 		TokenURL:     baseURL + "/api/v2/oauth/token",
-		Scopes:       []string{"device"},
 	}
 
 	ctx := context.Background()
 	tsClient := tailscale.NewClient("-", nil)
+	tsClient.UserAgent = "tailscale-get-authkey"
 	tsClient.HTTPClient = credentials.Client(ctx)
 	tsClient.BaseURL = baseURL
 

cmd/gitops-pusher/gitops-pusher.go

@@ -58,8 +58,8 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		}
 
 		if cache.PrevETag == "" {
-			log.Println("no previous etag found, assuming local file is correct and recording that")
-			cache.PrevETag = localEtag
+			log.Println("no previous etag found, assuming the latest control etag")
+			cache.PrevETag = controlEtag
 		}
 
 		log.Printf("control: %s", controlEtag)
@@ -105,8 +105,8 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		}
 
 		if cache.PrevETag == "" {
-			log.Println("no previous etag found, assuming local file is correct and recording that")
-			cache.PrevETag = localEtag
+			log.Println("no previous etag found, assuming the latest control etag")
+			cache.PrevETag = controlEtag
 		}
 
 		log.Printf("control: %s", controlEtag)
@@ -148,8 +148,8 @@ func getChecksums(cache *Cache, client *http.Client, tailnet, apiKey string) fun
 		}
 
 		if cache.PrevETag == "" {
-			log.Println("no previous etag found, assuming local file is correct and recording that")
-			cache.PrevETag = Shuck(localEtag)
+			log.Println("no previous etag found, assuming control etag")
+			cache.PrevETag = Shuck(controlEtag)
 		}
 
 		log.Printf("control: %s", controlEtag)

cmd/k8s-operator/connector.go

@@ -10,10 +10,12 @@ import (
 	"fmt"
 	"net/netip"
 	"slices"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/pkg/errors"
+	"errors"
+
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
@@ -26,6 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -33,6 +36,7 @@ import (
 
 const (
 	reasonConnectorCreationFailed = "ConnectorCreationFailed"
+	reasonConnectorCreating       = "ConnectorCreating"
 	reasonConnectorCreated        = "ConnectorCreated"
 	reasonConnectorInvalid        = "ConnectorInvalid"
 
@@ -57,15 +61,18 @@ type ConnectorReconciler struct {
 
 	subnetRouters set.Slice[types.UID] // for subnet routers gauge
 	exitNodes     set.Slice[types.UID] // for exit nodes gauge
+	appConnectors set.Slice[types.UID] // for app connectors gauge
 }
 
 var (
 	// gaugeConnectorResources tracks the overall number of Connectors currently managed by this operator instance.
-	gaugeConnectorResources = clientmetric.NewGauge("k8s_connector_resources")
+	gaugeConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorResourceCount)
 	// gaugeConnectorSubnetRouterResources tracks the number of Connectors managed by this operator instance that are subnet routers.
-	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
+	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
-	gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
+	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
+	// gaugeConnectorAppConnectorResources tracks the number of Connectors currently managed by this operator instance that are app connectors.
+	gaugeConnectorAppConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithAppConnectorCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -107,13 +114,12 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	oldCnStatus := cn.Status.DeepCopy()
 	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
+		var updateErr error
+		if !apiequality.Semantic.DeepEqual(oldCnStatus, &cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
+			updateErr = a.Client.Status().Update(ctx, cn)
 		}
-		return res, err
+		return res, errors.Join(err, updateErr)
 	}
 
 	if !slices.Contains(cn.Finalizers, FinalizerName) {
@@ -130,17 +136,24 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	}
 
 	if err := a.validate(cn); err != nil {
-		logger.Errorf("error validating Connector spec: %w", err)
 		message := fmt.Sprintf(messageConnectorInvalid, err)
 		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorInvalid, message)
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reasonConnectorInvalid, message)
 	}
 
 	if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil {
-		logger.Errorf("error creating Connector resources: %w", err)
+		reason := reasonConnectorCreationFailed
 		message := fmt.Sprintf(messageConnectorCreationFailed, err)
-		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorCreationFailed, message)
-		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reasonConnectorCreationFailed, message)
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			reason = reasonConnectorCreating
+			message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
+			err = nil
+			logger.Info(message)
+		} else {
+			a.recorder.Eventf(cn, corev1.EventTypeWarning, reason, message)
+		}
+
+		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reason, message)
 	}
 
 	logger.Info("Connector resources synced")
@@ -149,6 +162,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 	}
+	if cn.Spec.AppConnector != nil {
+		cn.Status.IsAppConnector = true
+	}
 	cn.Status.SubnetRoutes = ""
 	return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 }
@@ -182,29 +198,44 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 			isExitNode: cn.Spec.ExitNode,
 		},
 		ProxyClassName: proxyClass,
+		proxyType:      proxyTypeConnector,
 	}
 
 	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
 		sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 	}
 
+	if cn.Spec.AppConnector != nil {
+		sts.Connector.isAppConnector = true
+		if len(cn.Spec.AppConnector.Routes) != 0 {
+			sts.Connector.routes = cn.Spec.AppConnector.Routes.Stringify()
+		}
+	}
+
 	a.mu.Lock()
-	if sts.Connector.isExitNode {
+	if cn.Spec.ExitNode {
 		a.exitNodes.Add(cn.UID)
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
-	if sts.Connector.routes != "" {
+	if cn.Spec.SubnetRouter != nil {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
+	if cn.Spec.AppConnector != nil {
+		a.appConnectors.Add(cn.GetUID())
+	} else {
+		a.appConnectors.Remove(cn.GetUID())
+	}
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
@@ -212,27 +243,27 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		return err
 	}
 
-	_, tsHost, ips, err := a.ssr.DeviceInfo(ctx, crl)
+	dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
 	if err != nil {
 		return err
 	}
 
-	if tsHost == "" {
-		logger.Debugf("no Tailscale hostname known yet, waiting for connector pod to finish auth")
+	if dev == nil || dev.hostname == "" {
+		logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth")
 		// No hostname yet. Wait for the connector pod to auth.
 		cn.Status.TailnetIPs = nil
 		cn.Status.Hostname = ""
 		return nil
 	}
 
-	cn.Status.TailnetIPs = ips
-	cn.Status.Hostname = tsHost
+	cn.Status.TailnetIPs = dev.ips
+	cn.Status.Hostname = dev.hostname
 
 	return nil
 }
 
 func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
-	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector")); err != nil {
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil {
 		return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
 	} else if !done {
 		logger.Debugf("Connector cleanup not done yet, waiting for next reconcile")
@@ -247,12 +278,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
 	a.mu.Lock()
 	a.subnetRouters.Remove(cn.UID)
 	a.exitNodes.Remove(cn.UID)
+	a.appConnectors.Remove(cn.UID)
 	a.mu.Unlock()
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 	return true, nil
 }
@@ -261,8 +295,14 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 	// Connector fields are already validated at apply time with CEL validation
 	// on custom resource fields. The checks here are a backup in case the
 	// CEL validation breaks without us noticing.
-	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
-		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
+	if cn.Spec.SubnetRouter == nil && !cn.Spec.ExitNode && cn.Spec.AppConnector == nil {
+		return errors.New("invalid spec: a Connector must be configured as at least one of subnet router, exit node or app connector")
+	}
+	if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
+		return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
+	}
+	if cn.Spec.AppConnector != nil {
+		return validateAppConnector(cn.Spec.AppConnector)
 	}
 	if cn.Spec.SubnetRouter == nil {
 		return nil
@@ -271,19 +311,27 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 }
 
 func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
-	if len(sb.AdvertiseRoutes) < 1 {
+	if len(sb.AdvertiseRoutes) == 0 {
 		return errors.New("invalid subnet router spec: no routes defined")
 	}
-	var err error
-	for _, route := range sb.AdvertiseRoutes {
+	return validateRoutes(sb.AdvertiseRoutes)
+}
+
+func validateAppConnector(ac *tsapi.AppConnector) error {
+	return validateRoutes(ac.Routes)
+}
+
+func validateRoutes(routes tsapi.Routes) error {
+	var errs []error
+	for _, route := range routes {
 		pfx, e := netip.ParsePrefix(string(route))
 		if e != nil {
-			err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err))
+			errs = append(errs, fmt.Errorf("route %v is invalid: %v", route, e))
 			continue
 		}
 		if pfx.Masked() != pfx {
-			err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
+			errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
 		}
 	}
-	return err
+	return errors.Join(errs...)
 }

cmd/k8s-operator/connector_test.go

@@ -8,14 +8,17 @@ package main
 import (
 	"context"
 	"testing"
+	"time"
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/util/mak"
 )
@@ -74,9 +77,10 @@ func TestConnector(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Connector status should get updated with the IP/hostname info when available.
 	const hostname = "foo.tailnetxyz.ts.net"
@@ -102,7 +106,7 @@ func TestConnector(t *testing.T) {
 	opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20"
 	expectReconciled(t, cr, "", "test")
 
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Remove a route.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -110,7 +114,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.subnetRoutes = "10.44.0.0/20"
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Remove the subnet router.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -118,7 +122,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.subnetRoutes = ""
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Re-add the subnet router.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -128,7 +132,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.subnetRoutes = "10.44.0.0/20"
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Delete the Connector.
 	if err = fc.Delete(context.Background(), cn); err != nil {
@@ -169,9 +173,10 @@ func TestConnector(t *testing.T) {
 		parentType:   "connector",
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
+		app:          kubetypes.AppConnector,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Add an exit node.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -179,7 +184,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.isExitNode = true
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Delete the Connector.
 	if err = fc.Delete(context.Background(), cn); err != nil {
@@ -198,7 +203,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
 		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
+			Labels:      tsapi.Labels{"foo": "bar"},
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
@@ -254,9 +259,10 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet
 	// ready, so its configuration is NOT applied to the Connector
@@ -265,7 +271,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		conn.Spec.ProxyClass = "custom-metadata"
 	})
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 3. ProxyClass is set to Ready by proxy-class reconciler. Connector
 	// get reconciled and configuration from the ProxyClass is applied to
@@ -274,13 +280,13 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassready),
+				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
 	opts.proxyClass = pc.Name
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 4. Connector.spec.proxyClass field is unset, Connector gets
 	// reconciled and configuration from the ProxyClass is removed from the
@@ -290,5 +296,102 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	})
 	opts.proxyClass = ""
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
+}
+
+func TestConnectorWithAppConnector(t *testing.T) {
+	// Setup
+	cn := &tsapi.Connector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  types.UID("1234-UID"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       tsapi.ConnectorKind,
+			APIVersion: "tailscale.io/v1alpha1",
+		},
+		Spec: tsapi.ConnectorSpec{
+			AppConnector: &tsapi.AppConnector{},
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(cn).
+		WithStatusSubresource(cn).
+		Build()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	fr := record.NewFakeRecorder(1)
+	cr := &ConnectorReconciler{
+		Client: fc,
+		clock:  cl,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		recorder: fr,
+	}
+
+	// 1. Connector with app connnector is created and becomes ready
+	expectReconciled(t, cr, "", "test")
+	fullName, shortName := findGenName(t, fc, "", "test", "connector")
+	opts := configOpts{
+		stsName:        shortName,
+		secretName:     fullName,
+		parentType:     "connector",
+		hostname:       "test-connector",
+		app:            kubetypes.AppConnector,
+		isAppConnector: true,
+	}
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
+	// Connector's ready condition should be set to true
+
+	cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
+	cn.Status.IsAppConnector = true
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectEqual(t, fc, cn)
+
+	// 2. Connector with invalid app connector routes has status set to invalid
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	expectReconciled(t, cr, "", "test")
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionFalse,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorInvalid,
+		Message:            "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
+	}}
+	expectEqual(t, fc, cn)
+
+	// 3. Connector with valid app connnector routes becomes ready
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectReconciled(t, cr, "", "test")
 }

cmd/k8s-operator/depaware.txt

@@ -80,10 +80,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -98,7 +94,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/evanphx/json-patch/v5                             from sigs.k8s.io/controller-runtime/pkg/client
         github.com/evanphx/json-patch/v5/internal/json               from github.com/evanphx/json-patch/v5
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
-        github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
+        github.com/fxamacker/cbor/v2                                 from tailscale.com/tka+
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
         github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
@@ -114,11 +110,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/go-openapi/jsonpointer                            from github.com/go-openapi/jsonreference
         github.com/go-openapi/jsonreference                          from k8s.io/kube-openapi/pkg/internal+
         github.com/go-openapi/jsonreference/internal                 from github.com/go-openapi/jsonreference
-        github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
+     💣 github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
      💣 github.com/gogo/protobuf/proto                               from k8s.io/api/admission/v1+
         github.com/gogo/protobuf/sortkeys                            from k8s.io/api/admission/v1+
-        github.com/golang/groupcache/lru                             from k8s.io/client-go/tools/record+
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/golang/protobuf/proto                             from k8s.io/client-go/discovery+
         github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header+
         github.com/google/gnostic-models/compiler                    from github.com/google/gnostic-models/openapiv2+
@@ -143,15 +139,13 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
-        github.com/imdario/mergo                                     from k8s.io/client-go/tools/clientcmd
-   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
+   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
+   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/feature/tap
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/interfaces                      from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/rfc1035label                    from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/jmespath/go-jmespath                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/josharian/intern                                  from github.com/mailru/easyjson/jlexer
-   L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
      💣 github.com/json-iterator/go                                  from sigs.k8s.io/structured-merge-diff/v4/fieldpath+
@@ -162,7 +156,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
-        github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
+        github.com/kortschak/wol                                     from tailscale.com/feature/wakeonlan
         github.com/mailru/easyjson/buffer                            from github.com/mailru/easyjson/jwriter
      💣 github.com/mailru/easyjson/jlexer                            from github.com/go-openapi/swag
         github.com/mailru/easyjson/jwriter                           from github.com/go-openapi/swag
@@ -171,12 +165,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink+
         github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
         github.com/modern-go/concurrent                              from github.com/json-iterator/go
      💣 github.com/modern-go/reflect2                                from github.com/json-iterator/go
-        github.com/munnerz/goautoneg                                 from k8s.io/kube-openapi/pkg/handler3
+        github.com/munnerz/goautoneg                                 from k8s.io/kube-openapi/pkg/handler3+
         github.com/opencontainers/go-digest                          from github.com/distribution/reference
    L    github.com/pierrec/lz4/v4                                    from github.com/u-root/uio/uio
    L    github.com/pierrec/lz4/v4/internal/lz4block                  from github.com/pierrec/lz4/v4+
@@ -191,7 +185,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/prometheus/client_golang/prometheus/promhttp      from sigs.k8s.io/controller-runtime/pkg/metrics/server+
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
         github.com/prometheus/common/expfmt                          from github.com/prometheus/client_golang/prometheus+
-        github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
         github.com/prometheus/common/model                           from github.com/prometheus/client_golang/prometheus+
   LD    github.com/prometheus/procfs                                 from github.com/prometheus/client_golang/prometheus
   LD    github.com/prometheus/procfs/internal/fs                     from github.com/prometheus/procfs
@@ -216,6 +209,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tailscale/hujson                                  from tailscale.com/ipn/conffile
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/net/routetable+
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/peercred                                from tailscale.com/ipn/ipnauth
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
      💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
@@ -228,10 +222,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
         github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device
      💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
-        github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
         go.uber.org/multierr                                         from go.uber.org/zap+
@@ -256,6 +248,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         google.golang.org/protobuf/internal/descopts                 from google.golang.org/protobuf/internal/filedesc+
         google.golang.org/protobuf/internal/detrand                  from google.golang.org/protobuf/internal/descfmt+
         google.golang.org/protobuf/internal/editiondefaults          from google.golang.org/protobuf/internal/filedesc+
+        google.golang.org/protobuf/internal/editionssupport          from google.golang.org/protobuf/reflect/protodesc
         google.golang.org/protobuf/internal/encoding/defval          from google.golang.org/protobuf/internal/encoding/tag+
         google.golang.org/protobuf/internal/encoding/messageset      from google.golang.org/protobuf/encoding/prototext+
         google.golang.org/protobuf/internal/encoding/tag             from google.golang.org/protobuf/internal/impl
@@ -281,8 +274,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         google.golang.org/protobuf/types/gofeaturespb                from google.golang.org/protobuf/reflect/protodesc
         google.golang.org/protobuf/types/known/anypb                 from github.com/google/gnostic-models/compiler+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
+        gopkg.in/evanphx/json-patch.v4                               from k8s.io/client-go/testing
         gopkg.in/inf.v0                                              from k8s.io/apimachinery/pkg/api/resource
-        gopkg.in/yaml.v2                                             from k8s.io/kube-openapi/pkg/util/proto+
         gopkg.in/yaml.v3                                             from github.com/go-openapi/swag+
         gvisor.dev/gvisor/pkg/atomicbitops                           from gvisor.dev/gvisor/pkg/buffer+
         gvisor.dev/gvisor/pkg/bits                                   from gvisor.dev/gvisor/pkg/buffer
@@ -309,8 +302,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/feature/tap+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
@@ -351,6 +344,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/api/certificates/v1alpha1                             from k8s.io/client-go/applyconfigurations/certificates/v1alpha1+
         k8s.io/api/certificates/v1beta1                              from k8s.io/client-go/applyconfigurations/certificates/v1beta1+
         k8s.io/api/coordination/v1                                   from k8s.io/client-go/applyconfigurations/coordination/v1+
+        k8s.io/api/coordination/v1alpha2                             from k8s.io/client-go/applyconfigurations/coordination/v1alpha2+
         k8s.io/api/coordination/v1beta1                              from k8s.io/client-go/applyconfigurations/coordination/v1beta1+
         k8s.io/api/core/v1                                           from k8s.io/api/apps/v1+
         k8s.io/api/discovery/v1                                      from k8s.io/client-go/applyconfigurations/discovery/v1+
@@ -373,7 +367,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/api/rbac/v1                                           from k8s.io/client-go/applyconfigurations/rbac/v1+
         k8s.io/api/rbac/v1alpha1                                     from k8s.io/client-go/applyconfigurations/rbac/v1alpha1+
         k8s.io/api/rbac/v1beta1                                      from k8s.io/client-go/applyconfigurations/rbac/v1beta1+
-        k8s.io/api/resource/v1alpha2                                 from k8s.io/client-go/applyconfigurations/resource/v1alpha2+
+        k8s.io/api/resource/v1alpha3                                 from k8s.io/client-go/applyconfigurations/resource/v1alpha3+
+        k8s.io/api/resource/v1beta1                                  from k8s.io/client-go/applyconfigurations/resource/v1beta1+
         k8s.io/api/scheduling/v1                                     from k8s.io/client-go/applyconfigurations/scheduling/v1+
         k8s.io/api/scheduling/v1alpha1                               from k8s.io/client-go/applyconfigurations/scheduling/v1alpha1+
         k8s.io/api/scheduling/v1beta1                                from k8s.io/client-go/applyconfigurations/scheduling/v1beta1+
@@ -382,14 +377,16 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/api/storage/v1beta1                                   from k8s.io/client-go/applyconfigurations/storage/v1beta1+
         k8s.io/api/storagemigration/v1alpha1                         from k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1+
         k8s.io/apiextensions-apiserver/pkg/apis/apiextensions        from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
-     💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1     from sigs.k8s.io/controller-runtime/pkg/webhook/conversion
+     💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1     from sigs.k8s.io/controller-runtime/pkg/webhook/conversion+
         k8s.io/apimachinery/pkg/api/equality                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
         k8s.io/apimachinery/pkg/api/errors                           from k8s.io/apimachinery/pkg/util/managedfields/internal+
         k8s.io/apimachinery/pkg/api/meta                             from k8s.io/apimachinery/pkg/api/validation+
+        k8s.io/apimachinery/pkg/api/meta/testrestmapper              from k8s.io/client-go/testing
         k8s.io/apimachinery/pkg/api/resource                         from k8s.io/api/autoscaling/v1+
         k8s.io/apimachinery/pkg/api/validation                       from k8s.io/apimachinery/pkg/util/managedfields/internal+
      💣 k8s.io/apimachinery/pkg/apis/meta/internalversion            from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
         k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme     from k8s.io/client-go/metadata
+        k8s.io/apimachinery/pkg/apis/meta/internalversion/validation from k8s.io/client-go/util/watchlist
      💣 k8s.io/apimachinery/pkg/apis/meta/v1                         from k8s.io/api/admission/v1+
         k8s.io/apimachinery/pkg/apis/meta/v1/unstructured            from k8s.io/apimachinery/pkg/runtime/serializer/versioning+
         k8s.io/apimachinery/pkg/apis/meta/v1/validation              from k8s.io/apimachinery/pkg/api/validation+
@@ -401,6 +398,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/runtime                              from k8s.io/api/admission/v1+
         k8s.io/apimachinery/pkg/runtime/schema                       from k8s.io/api/admission/v1+
         k8s.io/apimachinery/pkg/runtime/serializer                   from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
+        k8s.io/apimachinery/pkg/runtime/serializer/cbor              from k8s.io/client-go/dynamic+
+        k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct       from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
+        k8s.io/apimachinery/pkg/runtime/serializer/cbor/internal/modes from k8s.io/apimachinery/pkg/runtime/serializer/cbor+
         k8s.io/apimachinery/pkg/runtime/serializer/json              from k8s.io/apimachinery/pkg/runtime/serializer+
         k8s.io/apimachinery/pkg/runtime/serializer/protobuf          from k8s.io/apimachinery/pkg/runtime/serializer
         k8s.io/apimachinery/pkg/runtime/serializer/recognizer        from k8s.io/apimachinery/pkg/runtime/serializer+
@@ -452,6 +452,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/applyconfigurations/certificates/v1alpha1   from k8s.io/client-go/kubernetes/typed/certificates/v1alpha1
         k8s.io/client-go/applyconfigurations/certificates/v1beta1    from k8s.io/client-go/kubernetes/typed/certificates/v1beta1
         k8s.io/client-go/applyconfigurations/coordination/v1         from k8s.io/client-go/kubernetes/typed/coordination/v1
+        k8s.io/client-go/applyconfigurations/coordination/v1alpha2   from k8s.io/client-go/kubernetes/typed/coordination/v1alpha2
         k8s.io/client-go/applyconfigurations/coordination/v1beta1    from k8s.io/client-go/kubernetes/typed/coordination/v1beta1
         k8s.io/client-go/applyconfigurations/core/v1                 from k8s.io/client-go/applyconfigurations/apps/v1+
         k8s.io/client-go/applyconfigurations/discovery/v1            from k8s.io/client-go/kubernetes/typed/discovery/v1
@@ -476,7 +477,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/applyconfigurations/rbac/v1                 from k8s.io/client-go/kubernetes/typed/rbac/v1
         k8s.io/client-go/applyconfigurations/rbac/v1alpha1           from k8s.io/client-go/kubernetes/typed/rbac/v1alpha1
         k8s.io/client-go/applyconfigurations/rbac/v1beta1            from k8s.io/client-go/kubernetes/typed/rbac/v1beta1
-        k8s.io/client-go/applyconfigurations/resource/v1alpha2       from k8s.io/client-go/kubernetes/typed/resource/v1alpha2
+        k8s.io/client-go/applyconfigurations/resource/v1alpha3       from k8s.io/client-go/kubernetes/typed/resource/v1alpha3
+        k8s.io/client-go/applyconfigurations/resource/v1beta1        from k8s.io/client-go/kubernetes/typed/resource/v1beta1
         k8s.io/client-go/applyconfigurations/scheduling/v1           from k8s.io/client-go/kubernetes/typed/scheduling/v1
         k8s.io/client-go/applyconfigurations/scheduling/v1alpha1     from k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1
         k8s.io/client-go/applyconfigurations/scheduling/v1beta1      from k8s.io/client-go/kubernetes/typed/scheduling/v1beta1
@@ -486,8 +488,80 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1
         k8s.io/client-go/discovery                                   from k8s.io/client-go/applyconfigurations/meta/v1+
         k8s.io/client-go/dynamic                                     from sigs.k8s.io/controller-runtime/pkg/cache/internal+
-        k8s.io/client-go/features                                    from k8s.io/client-go/tools/cache
-        k8s.io/client-go/kubernetes                                  from k8s.io/client-go/tools/leaderelection/resourcelock
+        k8s.io/client-go/features                                    from k8s.io/client-go/tools/cache+
+        k8s.io/client-go/gentype                                     from k8s.io/client-go/kubernetes/typed/admissionregistration/v1+
+        k8s.io/client-go/informers                                   from k8s.io/client-go/tools/leaderelection
+        k8s.io/client-go/informers/admissionregistration             from k8s.io/client-go/informers
+        k8s.io/client-go/informers/admissionregistration/v1          from k8s.io/client-go/informers/admissionregistration
+        k8s.io/client-go/informers/admissionregistration/v1alpha1    from k8s.io/client-go/informers/admissionregistration
+        k8s.io/client-go/informers/admissionregistration/v1beta1     from k8s.io/client-go/informers/admissionregistration
+        k8s.io/client-go/informers/apiserverinternal                 from k8s.io/client-go/informers
+        k8s.io/client-go/informers/apiserverinternal/v1alpha1        from k8s.io/client-go/informers/apiserverinternal
+        k8s.io/client-go/informers/apps                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/apps/v1                           from k8s.io/client-go/informers/apps
+        k8s.io/client-go/informers/apps/v1beta1                      from k8s.io/client-go/informers/apps
+        k8s.io/client-go/informers/apps/v1beta2                      from k8s.io/client-go/informers/apps
+        k8s.io/client-go/informers/autoscaling                       from k8s.io/client-go/informers
+        k8s.io/client-go/informers/autoscaling/v1                    from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/autoscaling/v2                    from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/autoscaling/v2beta1               from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/autoscaling/v2beta2               from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/batch                             from k8s.io/client-go/informers
+        k8s.io/client-go/informers/batch/v1                          from k8s.io/client-go/informers/batch
+        k8s.io/client-go/informers/batch/v1beta1                     from k8s.io/client-go/informers/batch
+        k8s.io/client-go/informers/certificates                      from k8s.io/client-go/informers
+        k8s.io/client-go/informers/certificates/v1                   from k8s.io/client-go/informers/certificates
+        k8s.io/client-go/informers/certificates/v1alpha1             from k8s.io/client-go/informers/certificates
+        k8s.io/client-go/informers/certificates/v1beta1              from k8s.io/client-go/informers/certificates
+        k8s.io/client-go/informers/coordination                      from k8s.io/client-go/informers
+        k8s.io/client-go/informers/coordination/v1                   from k8s.io/client-go/informers/coordination
+        k8s.io/client-go/informers/coordination/v1alpha2             from k8s.io/client-go/informers/coordination
+        k8s.io/client-go/informers/coordination/v1beta1              from k8s.io/client-go/informers/coordination
+        k8s.io/client-go/informers/core                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/core/v1                           from k8s.io/client-go/informers/core
+        k8s.io/client-go/informers/discovery                         from k8s.io/client-go/informers
+        k8s.io/client-go/informers/discovery/v1                      from k8s.io/client-go/informers/discovery
+        k8s.io/client-go/informers/discovery/v1beta1                 from k8s.io/client-go/informers/discovery
+        k8s.io/client-go/informers/events                            from k8s.io/client-go/informers
+        k8s.io/client-go/informers/events/v1                         from k8s.io/client-go/informers/events
+        k8s.io/client-go/informers/events/v1beta1                    from k8s.io/client-go/informers/events
+        k8s.io/client-go/informers/extensions                        from k8s.io/client-go/informers
+        k8s.io/client-go/informers/extensions/v1beta1                from k8s.io/client-go/informers/extensions
+        k8s.io/client-go/informers/flowcontrol                       from k8s.io/client-go/informers
+        k8s.io/client-go/informers/flowcontrol/v1                    from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/flowcontrol/v1beta1               from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/flowcontrol/v1beta2               from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/flowcontrol/v1beta3               from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/internalinterfaces                from k8s.io/client-go/informers+
+        k8s.io/client-go/informers/networking                        from k8s.io/client-go/informers
+        k8s.io/client-go/informers/networking/v1                     from k8s.io/client-go/informers/networking
+        k8s.io/client-go/informers/networking/v1alpha1               from k8s.io/client-go/informers/networking
+        k8s.io/client-go/informers/networking/v1beta1                from k8s.io/client-go/informers/networking
+        k8s.io/client-go/informers/node                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/node/v1                           from k8s.io/client-go/informers/node
+        k8s.io/client-go/informers/node/v1alpha1                     from k8s.io/client-go/informers/node
+        k8s.io/client-go/informers/node/v1beta1                      from k8s.io/client-go/informers/node
+        k8s.io/client-go/informers/policy                            from k8s.io/client-go/informers
+        k8s.io/client-go/informers/policy/v1                         from k8s.io/client-go/informers/policy
+        k8s.io/client-go/informers/policy/v1beta1                    from k8s.io/client-go/informers/policy
+        k8s.io/client-go/informers/rbac                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/rbac/v1                           from k8s.io/client-go/informers/rbac
+        k8s.io/client-go/informers/rbac/v1alpha1                     from k8s.io/client-go/informers/rbac
+        k8s.io/client-go/informers/rbac/v1beta1                      from k8s.io/client-go/informers/rbac
+        k8s.io/client-go/informers/resource                          from k8s.io/client-go/informers
+        k8s.io/client-go/informers/resource/v1alpha3                 from k8s.io/client-go/informers/resource
+        k8s.io/client-go/informers/resource/v1beta1                  from k8s.io/client-go/informers/resource
+        k8s.io/client-go/informers/scheduling                        from k8s.io/client-go/informers
+        k8s.io/client-go/informers/scheduling/v1                     from k8s.io/client-go/informers/scheduling
+        k8s.io/client-go/informers/scheduling/v1alpha1               from k8s.io/client-go/informers/scheduling
+        k8s.io/client-go/informers/scheduling/v1beta1                from k8s.io/client-go/informers/scheduling
+        k8s.io/client-go/informers/storage                           from k8s.io/client-go/informers
+        k8s.io/client-go/informers/storage/v1                        from k8s.io/client-go/informers/storage
+        k8s.io/client-go/informers/storage/v1alpha1                  from k8s.io/client-go/informers/storage
+        k8s.io/client-go/informers/storage/v1beta1                   from k8s.io/client-go/informers/storage
+        k8s.io/client-go/informers/storagemigration                  from k8s.io/client-go/informers
+        k8s.io/client-go/informers/storagemigration/v1alpha1         from k8s.io/client-go/informers/storagemigration
+        k8s.io/client-go/kubernetes                                  from k8s.io/client-go/tools/leaderelection/resourcelock+
         k8s.io/client-go/kubernetes/scheme                           from k8s.io/client-go/discovery+
         k8s.io/client-go/kubernetes/typed/admissionregistration/v1   from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes
@@ -511,6 +585,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/kubernetes/typed/certificates/v1alpha1      from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/certificates/v1beta1       from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/coordination/v1            from k8s.io/client-go/kubernetes+
+        k8s.io/client-go/kubernetes/typed/coordination/v1alpha2      from k8s.io/client-go/kubernetes+
         k8s.io/client-go/kubernetes/typed/coordination/v1beta1       from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/core/v1                    from k8s.io/client-go/kubernetes+
         k8s.io/client-go/kubernetes/typed/discovery/v1               from k8s.io/client-go/kubernetes
@@ -533,7 +608,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/kubernetes/typed/rbac/v1                    from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/rbac/v1alpha1              from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/rbac/v1beta1               from k8s.io/client-go/kubernetes
-        k8s.io/client-go/kubernetes/typed/resource/v1alpha2          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/resource/v1alpha3          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/resource/v1beta1           from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/scheduling/v1              from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1        from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/scheduling/v1beta1         from k8s.io/client-go/kubernetes
@@ -541,6 +617,56 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/kubernetes/typed/storage/v1alpha1           from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/storage/v1beta1            from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1  from k8s.io/client-go/kubernetes
+        k8s.io/client-go/listers                                     from k8s.io/client-go/listers/admissionregistration/v1+
+        k8s.io/client-go/listers/admissionregistration/v1            from k8s.io/client-go/informers/admissionregistration/v1
+        k8s.io/client-go/listers/admissionregistration/v1alpha1      from k8s.io/client-go/informers/admissionregistration/v1alpha1
+        k8s.io/client-go/listers/admissionregistration/v1beta1       from k8s.io/client-go/informers/admissionregistration/v1beta1
+        k8s.io/client-go/listers/apiserverinternal/v1alpha1          from k8s.io/client-go/informers/apiserverinternal/v1alpha1
+        k8s.io/client-go/listers/apps/v1                             from k8s.io/client-go/informers/apps/v1
+        k8s.io/client-go/listers/apps/v1beta1                        from k8s.io/client-go/informers/apps/v1beta1
+        k8s.io/client-go/listers/apps/v1beta2                        from k8s.io/client-go/informers/apps/v1beta2
+        k8s.io/client-go/listers/autoscaling/v1                      from k8s.io/client-go/informers/autoscaling/v1
+        k8s.io/client-go/listers/autoscaling/v2                      from k8s.io/client-go/informers/autoscaling/v2
+        k8s.io/client-go/listers/autoscaling/v2beta1                 from k8s.io/client-go/informers/autoscaling/v2beta1
+        k8s.io/client-go/listers/autoscaling/v2beta2                 from k8s.io/client-go/informers/autoscaling/v2beta2
+        k8s.io/client-go/listers/batch/v1                            from k8s.io/client-go/informers/batch/v1
+        k8s.io/client-go/listers/batch/v1beta1                       from k8s.io/client-go/informers/batch/v1beta1
+        k8s.io/client-go/listers/certificates/v1                     from k8s.io/client-go/informers/certificates/v1
+        k8s.io/client-go/listers/certificates/v1alpha1               from k8s.io/client-go/informers/certificates/v1alpha1
+        k8s.io/client-go/listers/certificates/v1beta1                from k8s.io/client-go/informers/certificates/v1beta1
+        k8s.io/client-go/listers/coordination/v1                     from k8s.io/client-go/informers/coordination/v1
+        k8s.io/client-go/listers/coordination/v1alpha2               from k8s.io/client-go/informers/coordination/v1alpha2
+        k8s.io/client-go/listers/coordination/v1beta1                from k8s.io/client-go/informers/coordination/v1beta1
+        k8s.io/client-go/listers/core/v1                             from k8s.io/client-go/informers/core/v1
+        k8s.io/client-go/listers/discovery/v1                        from k8s.io/client-go/informers/discovery/v1
+        k8s.io/client-go/listers/discovery/v1beta1                   from k8s.io/client-go/informers/discovery/v1beta1
+        k8s.io/client-go/listers/events/v1                           from k8s.io/client-go/informers/events/v1
+        k8s.io/client-go/listers/events/v1beta1                      from k8s.io/client-go/informers/events/v1beta1
+        k8s.io/client-go/listers/extensions/v1beta1                  from k8s.io/client-go/informers/extensions/v1beta1
+        k8s.io/client-go/listers/flowcontrol/v1                      from k8s.io/client-go/informers/flowcontrol/v1
+        k8s.io/client-go/listers/flowcontrol/v1beta1                 from k8s.io/client-go/informers/flowcontrol/v1beta1
+        k8s.io/client-go/listers/flowcontrol/v1beta2                 from k8s.io/client-go/informers/flowcontrol/v1beta2
+        k8s.io/client-go/listers/flowcontrol/v1beta3                 from k8s.io/client-go/informers/flowcontrol/v1beta3
+        k8s.io/client-go/listers/networking/v1                       from k8s.io/client-go/informers/networking/v1
+        k8s.io/client-go/listers/networking/v1alpha1                 from k8s.io/client-go/informers/networking/v1alpha1
+        k8s.io/client-go/listers/networking/v1beta1                  from k8s.io/client-go/informers/networking/v1beta1
+        k8s.io/client-go/listers/node/v1                             from k8s.io/client-go/informers/node/v1
+        k8s.io/client-go/listers/node/v1alpha1                       from k8s.io/client-go/informers/node/v1alpha1
+        k8s.io/client-go/listers/node/v1beta1                        from k8s.io/client-go/informers/node/v1beta1
+        k8s.io/client-go/listers/policy/v1                           from k8s.io/client-go/informers/policy/v1
+        k8s.io/client-go/listers/policy/v1beta1                      from k8s.io/client-go/informers/policy/v1beta1
+        k8s.io/client-go/listers/rbac/v1                             from k8s.io/client-go/informers/rbac/v1
+        k8s.io/client-go/listers/rbac/v1alpha1                       from k8s.io/client-go/informers/rbac/v1alpha1
+        k8s.io/client-go/listers/rbac/v1beta1                        from k8s.io/client-go/informers/rbac/v1beta1
+        k8s.io/client-go/listers/resource/v1alpha3                   from k8s.io/client-go/informers/resource/v1alpha3
+        k8s.io/client-go/listers/resource/v1beta1                    from k8s.io/client-go/informers/resource/v1beta1
+        k8s.io/client-go/listers/scheduling/v1                       from k8s.io/client-go/informers/scheduling/v1
+        k8s.io/client-go/listers/scheduling/v1alpha1                 from k8s.io/client-go/informers/scheduling/v1alpha1
+        k8s.io/client-go/listers/scheduling/v1beta1                  from k8s.io/client-go/informers/scheduling/v1beta1
+        k8s.io/client-go/listers/storage/v1                          from k8s.io/client-go/informers/storage/v1
+        k8s.io/client-go/listers/storage/v1alpha1                    from k8s.io/client-go/informers/storage/v1alpha1
+        k8s.io/client-go/listers/storage/v1beta1                     from k8s.io/client-go/informers/storage/v1beta1
+        k8s.io/client-go/listers/storagemigration/v1alpha1           from k8s.io/client-go/informers/storagemigration/v1alpha1
         k8s.io/client-go/metadata                                    from sigs.k8s.io/controller-runtime/pkg/cache/internal+
         k8s.io/client-go/openapi                                     from k8s.io/client-go/discovery
         k8s.io/client-go/pkg/apis/clientauthentication               from k8s.io/client-go/pkg/apis/clientauthentication/install+
@@ -552,6 +678,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/rest                                        from k8s.io/client-go/discovery+
         k8s.io/client-go/rest/watch                                  from k8s.io/client-go/rest
         k8s.io/client-go/restmapper                                  from sigs.k8s.io/controller-runtime/pkg/client/apiutil
+        k8s.io/client-go/testing                                     from k8s.io/client-go/gentype
         k8s.io/client-go/tools/auth                                  from k8s.io/client-go/tools/clientcmd
         k8s.io/client-go/tools/cache                                 from sigs.k8s.io/controller-runtime/pkg/cache+
         k8s.io/client-go/tools/cache/synctrack                       from k8s.io/client-go/tools/cache
@@ -568,11 +695,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/tools/record/util                           from k8s.io/client-go/tools/record
         k8s.io/client-go/tools/reference                             from k8s.io/client-go/kubernetes/typed/core/v1+
         k8s.io/client-go/transport                                   from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/util/apply                                  from k8s.io/client-go/dynamic+
         k8s.io/client-go/util/cert                                   from k8s.io/client-go/rest+
         k8s.io/client-go/util/connrotation                           from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/util/consistencydetector                    from k8s.io/client-go/dynamic+
         k8s.io/client-go/util/flowcontrol                            from k8s.io/client-go/kubernetes+
         k8s.io/client-go/util/homedir                                from k8s.io/client-go/tools/clientcmd
         k8s.io/client-go/util/keyutil                                from k8s.io/client-go/util/cert
+        k8s.io/client-go/util/watchlist                              from k8s.io/client-go/dynamic+
         k8s.io/client-go/util/workqueue                              from k8s.io/client-go/transport+
         k8s.io/klog/v2                                               from k8s.io/apimachinery/pkg/api/meta+
         k8s.io/klog/v2/internal/buffer                               from k8s.io/klog/v2
@@ -593,11 +723,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/utils/buffer                                          from k8s.io/client-go/tools/cache
         k8s.io/utils/clock                                           from k8s.io/apimachinery/pkg/util/cache+
         k8s.io/utils/clock/testing                                   from k8s.io/client-go/util/flowcontrol
+        k8s.io/utils/internal/third_party/forked/golang/golang-lru   from k8s.io/utils/lru
         k8s.io/utils/internal/third_party/forked/golang/net          from k8s.io/utils/net
+        k8s.io/utils/lru                                             from k8s.io/client-go/tools/record
         k8s.io/utils/net                                             from k8s.io/apimachinery/pkg/util/net+
         k8s.io/utils/pointer                                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
         k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
-        k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
         k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
         sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
         sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
@@ -630,12 +761,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         sigs.k8s.io/controller-runtime/pkg/metrics                   from sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics+
         sigs.k8s.io/controller-runtime/pkg/metrics/server            from sigs.k8s.io/controller-runtime/pkg/manager
         sigs.k8s.io/controller-runtime/pkg/predicate                 from sigs.k8s.io/controller-runtime/pkg/builder+
-        sigs.k8s.io/controller-runtime/pkg/ratelimiter               from sigs.k8s.io/controller-runtime/pkg/controller+
         sigs.k8s.io/controller-runtime/pkg/reconcile                 from sigs.k8s.io/controller-runtime/pkg/builder+
         sigs.k8s.io/controller-runtime/pkg/recorder                  from sigs.k8s.io/controller-runtime/pkg/leaderelection+
         sigs.k8s.io/controller-runtime/pkg/source                    from sigs.k8s.io/controller-runtime/pkg/builder+
         sigs.k8s.io/controller-runtime/pkg/webhook                   from sigs.k8s.io/controller-runtime/pkg/manager
         sigs.k8s.io/controller-runtime/pkg/webhook/admission         from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/webhook/admission/metrics from sigs.k8s.io/controller-runtime/pkg/webhook/admission
         sigs.k8s.io/controller-runtime/pkg/webhook/conversion        from sigs.k8s.io/controller-runtime/pkg/builder
         sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics  from sigs.k8s.io/controller-runtime/pkg/webhook+
         sigs.k8s.io/json                                             from k8s.io/apimachinery/pkg/runtime/serializer/json+
@@ -646,18 +777,19 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         sigs.k8s.io/structured-merge-diff/v4/typed                   from k8s.io/apimachinery/pkg/util/managedfields+
         sigs.k8s.io/structured-merge-diff/v4/value                   from k8s.io/apimachinery/pkg/runtime+
         sigs.k8s.io/yaml                                             from k8s.io/apimachinery/pkg/runtime/serializer/json+
-        sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml
+        sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
-        tailscale.com/atomicfile                                     from tailscale.com/ipn+
+     💣 tailscale.com/atomicfile                                     from tailscale.com/ipn+
         tailscale.com/client/tailscale                               from tailscale.com/client/web+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
-        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+  LW    tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/ipn/localapi+
@@ -668,6 +800,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
+        tailscale.com/feature                                        from tailscale.com/feature/wakeonlan+
+        tailscale.com/feature/condregister                           from tailscale.com/tsnet
+   L    tailscale.com/feature/tap                                    from tailscale.com/feature/condregister
+        tailscale.com/feature/wakeonlan                              from tailscale.com/feature/condregister
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -690,7 +827,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
         tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/k8s-operator/sessionrecording/ws               from tailscale.com/k8s-operator/sessionrecording
-        tailscale.com/kube                                           from tailscale.com/cmd/k8s-operator+
+        tailscale.com/kube/egressservices                            from tailscale.com/cmd/k8s-operator
+        tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
+        tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
+        tailscale.com/kube/kubetypes                                 from tailscale.com/cmd/k8s-operator+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -699,6 +839,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial+
         tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
@@ -731,11 +872,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/tsd+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -754,6 +895,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
@@ -769,6 +911,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/appc+
@@ -786,7 +929,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/appc+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlclient+
@@ -795,7 +938,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/osdiag                                    from tailscale.com/ipn/localapi
    W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
         tailscale.com/util/osshare                                   from tailscale.com/ipn/ipnlocal
-        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal
         tailscale.com/util/progresstracking                          from tailscale.com/ipn/localapi
         tailscale.com/util/race                                      from tailscale.com/net/dns/resolver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -805,17 +948,21 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
-        tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/usermetric                                from tailscale.com/health+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+
@@ -849,6 +996,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
@@ -870,7 +1018,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
         golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+
         golang.org/x/sync/errgroup                                   from github.com/mdlayher/socket+
-        golang.org/x/sys/cpu                                         from github.com/josharian/native+
+        golang.org/x/sys/cpu                                         from github.com/tailscale/certstore+
   LD    golang.org/x/sys/unix                                        from github.com/fsnotify/fsnotify+
    W    golang.org/x/sys/windows                                     from github.com/dblohm7/wingoes+
    W    golang.org/x/sys/windows/registry                            from github.com/dblohm7/wingoes+
@@ -950,6 +1098,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
+        iter                                                         from go/ast+
         log                                                          from expvar+
         log/internal                                                 from log+
         log/slog                                                     from github.com/go-logr/logr+
@@ -987,7 +1136,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from encoding/base32+
-        sort                                                         from archive/tar+
+        sort                                                         from compress/flate+
         strconv                                                      from archive/tar+
         strings                                                      from archive/tar+
         sync                                                         from archive/tar+
@@ -1000,3 +1149,4 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -35,9 +35,13 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      - name: oauth
-        secret:
-          secretName: operator-oauth
+        - name: oauth
+          {{- with .Values.oauthSecretVolume }}
+          {{- toYaml . | nindent 10 }}
+          {{- else }}
+          secret:
+            secretName: operator-oauth
+          {{- end }}
       containers:
         - name: operator
           {{- with .Values.operatorConfig.securityContext }}
@@ -81,6 +85,14 @@ spec:
             - name: PROXY_DEFAULT_CLASS
               value: {{ .Values.proxyConfig.defaultProxyClass }}
             {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.ingressClass.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: IngressClass
 metadata:
@@ -6,3 +7,4 @@ metadata:
 spec:
   controller: tailscale.com/ts-ingress # controller name currently can not be changed
   # parameters: {} # currently no parameters are supported
+{{- end }}

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -6,6 +6,10 @@ kind: ServiceAccount
 metadata:
   name: operator
   namespace: {{ .Release.Namespace }}
+  {{- with .Values.operatorConfig.serviceAccountAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -22,11 +26,18 @@ rules:
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["tailscale.com"]
-  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
+  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status", "proxygroups", "proxygroups/status"]
   verbs: ["get", "list", "watch", "update"]
 - apiGroups: ["tailscale.com"]
   resources: ["dnsconfigs", "dnsconfigs/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["recorders", "recorders/status"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["get", "list", "watch"]
+  resourceNames: ["servicemonitors.monitoring.coreos.com"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -50,12 +61,21 @@ rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get","list","watch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings"]
+  verbs: ["get", "create", "patch", "update", "list", "watch"]
+- apiGroups: ["monitoring.coreos.com"]
+  resources: ["servicemonitors"]
+  verbs: ["get", "list", "update", "create", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -16,6 +16,9 @@ rules:
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -3,11 +3,26 @@
 
 # Operator oauth credentials. If set a Kubernetes Secret with the provided
 # values will be created in the operator namespace. If unset a Secret named
-# operator-oauth must be precreated.
+# operator-oauth must be precreated or oauthSecretVolume needs to be adjusted.
+# This block will be overridden by oauthSecretVolume, if set.
 oauth: {}
   # clientId: ""
   # clientSecret: ""
 
+# Secret volume.
+# If set it defines the volume the oauth secrets will be mounted from.
+# The volume needs to contain two files named `client_id` and `client_secret`.
+# If unset the volume will reference the Secret named operator-oauth.
+# This block will override the oauth block.
+oauthSecretVolume: {}
+  # csi:
+  #   driver: secrets-store.csi.k8s.io
+  #   readOnly: true
+  #   volumeAttributes:
+  #     secretProviderClass: tailscale-oauth
+  #
+  ## NAME is pre-defined!
+
 # installCRDs determines whether tailscale.com CRDs should be installed as part
 # of chart installation. We do not use Helm's CRD installation mechanism as that
 # does not allow for upgrading CRDs.
@@ -40,6 +55,9 @@ operatorConfig:
   podAnnotations: {}
   podLabels: {}
 
+  serviceAccountAnnotations: {}
+  # eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/tailscale-operator-role
+
   tolerations: []
 
   affinity: {}
@@ -54,15 +72,18 @@ operatorConfig:
   # - name: EXTRA_VAR2
   #   value: "value2"
 
+# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
+ingressClass:
+  enabled: true
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.
-# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
-# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
+# https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
+# https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
 # Note that this section contains only a few global configuration options and
 # will not be updated with more configuration options in the future.
 # If you need more configuration options, take a look at ProxyClass:
-# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
+# https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
@@ -79,12 +100,13 @@ proxyConfig:
   defaultTags: "tag:k8s"
   firewallMode: auto
   # If defined, this proxy class will be used as the default proxy class for
-  # service and ingress resources that do not have a proxy class defined.
+  # service and ingress resources that do not have a proxy class defined. It
+  # does not apply to Connector resources.
   defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.
-# https://tailscale.com/kb/1236/kubernetes-operator/#accessing-the-kubernetes-control-plane-using-an-api-server-proxy
+# https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
 apiServerProxyConfig:
   mode: "false" # "true", "false", "noauth"
 

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: connectors.tailscale.com
 spec:
   group: tailscale.com
@@ -24,6 +24,10 @@ spec:
           jsonPath: .status.isExitNode
           name: IsExitNode
           type: string
+        - description: Whether this Connector instance is an app connector.
+          jsonPath: .status.isAppConnector
+          name: IsAppConnector
+          type: string
         - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
@@ -37,7 +41,7 @@ spec:
             exit node.
             Connector is a cluster-scoped resource.
             More info:
-            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
+            https://tailscale.com/kb/1441/kubernetes-operator-connector
           type: object
           required:
             - spec
@@ -66,10 +70,40 @@ spec:
                 https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
+                appConnector:
+                  description: |-
+                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                    Connector does not act as an app connector.
+                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                    Admin panel.
+                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                    tested or optimised for.
+                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                    device with a static IP address.
+                    https://tailscale.com/kb/1281/app-connectors
+                  type: object
+                  properties:
+                    routes:
+                      description: |-
+                        Routes are optional preconfigured routes for the domains routed via the app connector.
+                        If not set, routes for the domains will be discovered dynamically.
+                        If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                        also dynamically discover other routes.
+                        https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                      type: array
+                      minItems: 1
+                      items:
+                        type: string
+                        format: cidr
                 exitNode:
                   description: |-
-                    ExitNode defines whether the Connector node should act as a
-                    Tailscale exit node. Defaults to false.
+                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                    This field is mutually exclusive with the appConnector field.
                     https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
@@ -90,9 +124,11 @@ spec:
                   type: string
                 subnetRouter:
                   description: |-
-                    SubnetRouter defines subnet routes that the Connector node should
-                    expose to tailnet. If unset, none are exposed.
+                    SubnetRouter defines subnet routes that the Connector device should
+                    expose to tailnet as a Tailscale subnet router.
                     https://tailscale.com/kb/1019/subnets/
+                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                    This field is mutually exclusive with the appConnector field.
                   type: object
                   required:
                     - advertiseRoutes
@@ -115,7 +151,7 @@ spec:
                     To autoapprove the subnet routes or exit node defined by a Connector,
                     you can configure Tailscale ACLs to give these tags the necessary
                     permissions.
-                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
+                    See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
                     If you specify custom tags here, you must also make the operator an owner of these tags.
                     See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
                     Tags cannot be changed once a Connector node has been created.
@@ -125,8 +161,10 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
               x-kubernetes-validations:
-                - rule: has(self.subnetRouter) || self.exitNode == true
-                  message: A Connector needs to be either an exit node or a subnet router, or both.
+                - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                  message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                - rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
+                  message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
             status:
               description: |-
                 ConnectorStatus describes the status of the Connector. This is set
@@ -200,6 +238,9 @@ spec:
                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                     node.
                   type: string
+                isAppConnector:
+                  description: IsAppConnector is set to true if the Connector acts as an app connector.
+                  type: boolean
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: dnsconfigs.tailscale.com
 spec:
   group: tailscale.com
@@ -89,14 +89,14 @@ spec:
                   type: object
                   properties:
                     image:
-                      description: Nameserver image.
+                      description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
                       type: object
                       properties:
                         repo:
                           description: Repo defaults to tailscale/k8s-nameserver.
                           type: string
                         tag:
-                          description: Tag defaults to operator's own tag.
+                          description: Tag defaults to unstable.
                           type: string
             status:
               description: |-

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: proxyclasses.tailscale.com
 spec:
   group: tailscale.com
@@ -30,7 +30,7 @@ spec:
             connector.spec.proxyClass field.
             ProxyClass is a cluster scoped resource.
             More info:
-            https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource.
+            https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
           type: object
           required:
             - spec
@@ -73,9 +73,45 @@ spec:
                     enable:
                       description: |-
                         Setting enable to true will make the proxy serve Tailscale metrics
-                        at <pod-ip>:9001/debug/metrics.
+                        at <pod-ip>:9002/metrics.
+                        A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
+                        serve the metrics at <service-ip>:9002/metrics.
+
+                        In 1.78.x and 1.80.x, this field also serves as the default value for
+                        .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                        fields will independently default to false.
+
                         Defaults to false.
                       type: boolean
+                    serviceMonitor:
+                      description: |-
+                        Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
+                        The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
+                        The ingested metrics for each Service monitor will have labels to identify the proxy:
+                        ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
+                        ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
+                        ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
+                        job: ts_<proxy type>_[<parent namespace>]_<parent_name>
+                      type: object
+                      required:
+                        - enable
+                      properties:
+                        enable:
+                          description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
+                          type: boolean
+                        labels:
+                          description: |-
+                            Labels to add to the ServiceMonitor.
+                            Labels must be valid Kubernetes labels.
+                            https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+                          type: object
+                          additionalProperties:
+                            type: string
+                            maxLength: 63
+                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
+                  x-kubernetes-validations:
+                    - rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable  && !self.enable)'
+                      message: ServiceMonitor can only be enabled if metrics are enabled
                 statefulSet:
                   description: |-
                     Configuration parameters for the proxy's StatefulSet. Tailscale
@@ -107,6 +143,8 @@ spec:
                       type: object
                       additionalProperties:
                         type: string
+                        maxLength: 63
+                        pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                     pod:
                       description: Configuration for the proxy Pod.
                       type: object
@@ -390,7 +428,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                               Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -405,7 +443,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                               Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -562,7 +600,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                           Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -577,7 +615,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                           Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -735,7 +773,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                               Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -750,7 +788,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                               Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -907,7 +945,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                           Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -922,7 +960,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                           Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -1036,6 +1074,8 @@ spec:
                           type: object
                           additionalProperties:
                             type: string
+                            maxLength: 63
+                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                         nodeName:
                           description: |-
                             Proxy Pod's node name.
@@ -1134,6 +1174,32 @@ spec:
                                 Note that this field cannot be set when spec.os.name is windows.
                               type: integer
                               format: int64
+                            seLinuxChangePolicy:
+                              description: |-
+                                seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                                It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                                Valid values are "MountOption" and "Recursive".
+
+                                "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                                This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                                "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                                This requires all Pods that share the same volume to use the same SELinux label.
+                                It is not possible to share the same volume among privileged and unprivileged Pods.
+                                Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                                whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                                CSIDriver instance. Other volumes are always re-labelled recursively.
+                                "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                                If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                                If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                                and "Recursive" for all other volumes.
+
+                                This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                                All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                                Note that this field cannot be set when spec.os.name is windows.
+                              type: string
                             seLinuxOptions:
                               description: |-
                                 The SELinux context to be applied to all containers.
@@ -1182,18 +1248,28 @@ spec:
                                   type: string
                             supplementalGroups:
                               description: |-
-                                A list of groups applied to the first process run in each container, in addition
-                                to the container's primary GID, the fsGroup (if specified), and group memberships
-                                defined in the container image for the uid of the container process. If unspecified,
-                                no additional groups are added to any container. Note that group memberships
-                                defined in the container image for the uid of the container process are still effective,
-                                even if they are not included in this list.
+                                A list of groups applied to the first process run in each container, in
+                                addition to the container's primary GID and fsGroup (if specified).  If
+                                the SupplementalGroupsPolicy feature is enabled, the
+                                supplementalGroupsPolicy field determines whether these are in addition
+                                to or instead of any group memberships defined in the container image.
+                                If unspecified, no additional groups are added, though group memberships
+                                defined in the container image may still be used, depending on the
+                                supplementalGroupsPolicy field.
                                 Note that this field cannot be set when spec.os.name is windows.
                               type: array
                               items:
                                 type: integer
                                 format: int64
                               x-kubernetes-list-type: atomic
+                            supplementalGroupsPolicy:
+                              description: |-
+                                Defines how supplemental groups of the first container processes are calculated.
+                                Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                                (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                                and the container runtime must implement support for this feature.
+                                Note that this field cannot be set when spec.os.name is windows.
+                              type: string
                             sysctls:
                               description: |-
                                 Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -1249,6 +1325,25 @@ spec:
                           description: Configuration for the proxy container running tailscale.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1330,6 +1425,12 @@ spec:
                                           the Pod where this field is used. It makes that resource available
                                           inside a container.
                                         type: string
+                                      request:
+                                        description: |-
+                                          Request is the name chosen for a request in the referenced claim.
+                                          If empty, everything from the claim is made available, otherwise
+                                          only the result of this request.
+                                        type: string
                                   x-kubernetes-list-map-keys:
                                     - name
                                   x-kubernetes-list-type: map
@@ -1360,11 +1461,12 @@ spec:
                             securityContext:
                               description: |-
                                 Container security context.
-                                Security context specified here will override the security context by the operator.
-                                By default the operator:
-                                - sets 'privileged: true' for the init container
-                                - set NET_ADMIN capability for tailscale container for proxies that
-                                are created for Services or Connector.
+                                Security context specified here will override the security context set by the operator.
+                                By default the operator sets the Tailscale container and the Tailscale init container to privileged
+                                for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
+                                You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
+                                installing device plugin in your cluster and configuring the proxies tun device to be created
+                                by the device plugin, see  https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
                                 https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
                               type: object
                               properties:
@@ -1433,7 +1535,7 @@ spec:
                                 procMount:
                                   description: |-
                                     procMount denotes the type of proc mount to use for the containers.
-                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                    The default value is Default which uses the container runtime defaults for
                                     readonly paths and masked paths.
                                     This requires the ProcMountType feature flag to be enabled.
                                     Note that this field cannot be set when spec.os.name is windows.
@@ -1553,6 +1655,25 @@ spec:
                           description: Configuration for the proxy init container that enables forwarding.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1634,6 +1755,12 @@ spec:
                                           the Pod where this field is used. It makes that resource available
                                           inside a container.
                                         type: string
+                                      request:
+                                        description: |-
+                                          Request is the name chosen for a request in the referenced claim.
+                                          If empty, everything from the claim is made available, otherwise
+                                          only the result of this request.
+                                        type: string
                                   x-kubernetes-list-map-keys:
                                     - name
                                   x-kubernetes-list-type: map
@@ -1664,11 +1791,12 @@ spec:
                             securityContext:
                               description: |-
                                 Container security context.
-                                Security context specified here will override the security context by the operator.
-                                By default the operator:
-                                - sets 'privileged: true' for the init container
-                                - set NET_ADMIN capability for tailscale container for proxies that
-                                are created for Services or Connector.
+                                Security context specified here will override the security context set by the operator.
+                                By default the operator sets the Tailscale container and the Tailscale init container to privileged
+                                for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
+                                You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
+                                installing device plugin in your cluster and configuring the proxies tun device to be created
+                                by the device plugin, see  https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
                                 https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
                               type: object
                               properties:
@@ -1737,7 +1865,7 @@ spec:
                                 procMount:
                                   description: |-
                                     procMount denotes the type of proc mount to use for the containers.
-                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                    The default value is Default which uses the container runtime defaults for
                                     readonly paths and masked paths.
                                     This requires the ProcMountType feature flag to be enabled.
                                     Note that this field cannot be set when spec.os.name is windows.
@@ -1896,6 +2024,182 @@ spec:
                                   Value is the taint value the toleration matches to.
                                   If the operator is Exists, the value should be empty, otherwise just a regular string.
                                 type: string
+                        topologySpreadConstraints:
+                          description: |-
+                            Proxy Pod's topology spread constraints.
+                            By default Tailscale Kubernetes operator does not apply any topology spread constraints.
+                            https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
+                          type: array
+                          items:
+                            description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                            type: object
+                            required:
+                              - maxSkew
+                              - topologyKey
+                              - whenUnsatisfiable
+                            properties:
+                              labelSelector:
+                                description: |-
+                                  LabelSelector is used to find matching pods.
+                                  Pods that match this label selector are counted to determine the number of pods
+                                  in their corresponding topology domain.
+                                type: object
+                                properties:
+                                  matchExpressions:
+                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                    type: array
+                                    items:
+                                      description: |-
+                                        A label selector requirement is a selector that contains values, a key, and an operator that
+                                        relates the key and values.
+                                      type: object
+                                      required:
+                                        - key
+                                        - operator
+                                      properties:
+                                        key:
+                                          description: key is the label key that the selector applies to.
+                                          type: string
+                                        operator:
+                                          description: |-
+                                            operator represents a key's relationship to a set of values.
+                                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                                          type: string
+                                        values:
+                                          description: |-
+                                            values is an array of string values. If the operator is In or NotIn,
+                                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                            the values array must be empty. This array is replaced during a strategic
+                                            merge patch.
+                                          type: array
+                                          items:
+                                            type: string
+                                          x-kubernetes-list-type: atomic
+                                    x-kubernetes-list-type: atomic
+                                  matchLabels:
+                                    description: |-
+                                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                    type: object
+                                    additionalProperties:
+                                      type: string
+                                x-kubernetes-map-type: atomic
+                              matchLabelKeys:
+                                description: |-
+                                  MatchLabelKeys is a set of pod label keys to select the pods over which
+                                  spreading will be calculated. The keys are used to lookup values from the
+                                  incoming pod labels, those key-value labels are ANDed with labelSelector
+                                  to select the group of existing pods over which spreading will be calculated
+                                  for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
+                                  MatchLabelKeys cannot be set when LabelSelector isn't set.
+                                  Keys that don't exist in the incoming pod labels will
+                                  be ignored. A null or empty list means only match against labelSelector.
+
+                                  This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+                                type: array
+                                items:
+                                  type: string
+                                x-kubernetes-list-type: atomic
+                              maxSkew:
+                                description: |-
+                                  MaxSkew describes the degree to which pods may be unevenly distributed.
+                                  When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
+                                  between the number of matching pods in the target topology and the global minimum.
+                                  The global minimum is the minimum number of matching pods in an eligible domain
+                                  or zero if the number of eligible domains is less than MinDomains.
+                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                  labelSelector spread as 2/2/1:
+                                  In this case, the global minimum is 1.
+                                  | zone1 | zone2 | zone3 |
+                                  |  P P  |  P P  |   P   |
+                                  - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
+                                  scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
+                                  violate MaxSkew(1).
+                                  - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
+                                  When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
+                                  to topologies that satisfy it.
+                                  It's a required field. Default value is 1 and 0 is not allowed.
+                                type: integer
+                                format: int32
+                              minDomains:
+                                description: |-
+                                  MinDomains indicates a minimum number of eligible domains.
+                                  When the number of eligible domains with matching topology keys is less than minDomains,
+                                  Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
+                                  And when the number of eligible domains with matching topology keys equals or greater than minDomains,
+                                  this value has no effect on scheduling.
+                                  As a result, when the number of eligible domains is less than minDomains,
+                                  scheduler won't schedule more than maxSkew Pods to those domains.
+                                  If value is nil, the constraint behaves as if MinDomains is equal to 1.
+                                  Valid values are integers greater than 0.
+                                  When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
+
+                                  For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
+                                  labelSelector spread as 2/2/2:
+                                  | zone1 | zone2 | zone3 |
+                                  |  P P  |  P P  |  P P  |
+                                  The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
+                                  In this situation, new pod with the same labelSelector cannot be scheduled,
+                                  because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
+                                  it will violate MaxSkew.
+                                type: integer
+                                format: int32
+                              nodeAffinityPolicy:
+                                description: |-
+                                  NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
+                                  when calculating pod topology spread skew. Options are:
+                                  - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
+                                  - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
+
+                                  If this value is nil, the behavior is equivalent to the Honor policy.
+                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                type: string
+                              nodeTaintsPolicy:
+                                description: |-
+                                  NodeTaintsPolicy indicates how we will treat node taints when calculating
+                                  pod topology spread skew. Options are:
+                                  - Honor: nodes without taints, along with tainted nodes for which the incoming pod
+                                  has a toleration, are included.
+                                  - Ignore: node taints are ignored. All nodes are included.
+
+                                  If this value is nil, the behavior is equivalent to the Ignore policy.
+                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                type: string
+                              topologyKey:
+                                description: |-
+                                  TopologyKey is the key of node labels. Nodes that have a label with this key
+                                  and identical values are considered to be in the same topology.
+                                  We consider each <key, value> as a "bucket", and try to put balanced number
+                                  of pods into each bucket.
+                                  We define a domain as a particular instance of a topology.
+                                  Also, we define an eligible domain as a domain whose nodes meet the requirements of
+                                  nodeAffinityPolicy and nodeTaintsPolicy.
+                                  e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
+                                  And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
+                                  It's a required field.
+                                type: string
+                              whenUnsatisfiable:
+                                description: |-
+                                  WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
+                                  the spread constraint.
+                                  - DoNotSchedule (default) tells the scheduler not to schedule it.
+                                  - ScheduleAnyway tells the scheduler to schedule the pod in any location,
+                                    but giving higher precedence to topologies that would help reduce the
+                                    skew.
+                                  A constraint is considered "Unsatisfiable" for an incoming pod
+                                  if and only if every possible node assignment for that pod would violate
+                                  "MaxSkew" on some topology.
+                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                  labelSelector spread as 3/1/1:
+                                  | zone1 | zone2 | zone3 |
+                                  | P P P |   P   |   P   |
+                                  If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
+                                  to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
+                                  MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
+                                  won't make it *more* imbalanced.
+                                  It's a required field.
+                                type: string
                 tailscale:
                   description: |-
                     TailscaleConfig contains options to configure the tailscale-specific
@@ -1908,7 +2212,7 @@ spec:
                         routes advertized by other nodes on the tailnet, such as subnet
                         routes.
                         This is equivalent of passing --accept-routes flag to a tailscale Linux client.
-                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
+                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
             status:

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -0,0 +1,209 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.17.0
+  name: proxygroups.tailscale.com
+spec:
+  group: tailscale.com
+  names:
+    kind: ProxyGroup
+    listKind: ProxyGroupList
+    plural: proxygroups
+    shortNames:
+      - pg
+    singular: proxygroup
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Status of the deployed ProxyGroup resources.
+          jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
+          name: Status
+          type: string
+        - description: ProxyGroup type.
+          jsonPath: .spec.type
+          name: Type
+          type: string
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: |-
+            ProxyGroup defines a set of Tailscale devices that will act as proxies.
+            Currently only egress ProxyGroups are supported.
+
+            Use the tailscale.com/proxy-group annotation on a Service to specify that
+            the egress proxy should be implemented by a ProxyGroup instead of a single
+            dedicated proxy. In addition to running a highly available set of proxies,
+            ProxyGroup also allows for serving many annotated Services from a single
+            set of proxies to minimise resource consumption.
+
+            More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
+          type: object
+          required:
+            - spec
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: Spec describes the desired ProxyGroup instances.
+              type: object
+              required:
+                - type
+              properties:
+                hostnamePrefix:
+                  description: |-
+                    HostnamePrefix is the hostname prefix to use for tailnet devices created
+                    by the ProxyGroup. Each device will have the integer number from its
+                    StatefulSet pod appended to this prefix to form the full hostname.
+                    HostnamePrefix can contain lower case letters, numbers and dashes, it
+                    must not start with a dash and must be between 1 and 62 characters long.
+                  type: string
+                  pattern: ^[a-z0-9][a-z0-9-]{0,61}$
+                proxyClass:
+                  description: |-
+                    ProxyClass is the name of the ProxyClass custom resource that contains
+                    configuration options that should be applied to the resources created
+                    for this ProxyGroup. If unset, and there is no default ProxyClass
+                    configured, the operator will create resources with the default
+                    configuration.
+                  type: string
+                replicas:
+                  description: |-
+                    Replicas specifies how many replicas to create the StatefulSet with.
+                    Defaults to 2.
+                  type: integer
+                  format: int32
+                  minimum: 0
+                tags:
+                  description: |-
+                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
+                    If you specify custom tags here, make sure you also make the operator
+                    an owner of these tags.
+                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+                    Tags cannot be changed once a ProxyGroup device has been created.
+                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                  type: array
+                  items:
+                    type: string
+                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+                type:
+                  description: |-
+                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                    Type is immutable once a ProxyGroup is created.
+                  type: string
+                  enum:
+                    - egress
+                    - ingress
+                  x-kubernetes-validations:
+                    - rule: self == oldSelf
+                      message: ProxyGroup type is immutable
+            status:
+              description: |-
+                ProxyGroupStatus describes the status of the ProxyGroup resources. This is
+                set and managed by the Tailscale operator.
+              type: object
+              properties:
+                conditions:
+                  description: |-
+                    List of status conditions to indicate the status of the ProxyGroup
+                    resources. Known condition types are `ProxyGroupReady`.
+                  type: array
+                  items:
+                    description: Condition contains details for one aspect of the current state of this API Resource.
+                    type: object
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        type: string
+                        format: date-time
+                      message:
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
+                        type: string
+                        maxLength: 32768
+                      observedGeneration:
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
+                        type: integer
+                        format: int64
+                        minimum: 0
+                      reason:
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
+                        type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                  x-kubernetes-list-map-keys:
+                    - type
+                  x-kubernetes-list-type: map
+                devices:
+                  description: List of tailnet devices associated with the ProxyGroup StatefulSet.
+                  type: array
+                  items:
+                    type: object
+                    required:
+                      - hostname
+                    properties:
+                      hostname:
+                        description: |-
+                          Hostname is the fully qualified domain name of the device.
+                          If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+                          node.
+                        type: string
+                      tailnetIPs:
+                        description: |-
+                          TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+                          assigned to the device.
+                        type: array
+                        items:
+                          type: string
+                  x-kubernetes-list-map-keys:
+                    - hostname
+                  x-kubernetes-list-type: map
+      served: true
+      storage: true
+      subresources:
+        status: {}

cmd/k8s-operator/deploy/examples/proxygroup.yaml

@@ -0,0 +1,7 @@
+apiVersion: tailscale.com/v1alpha1
+kind: ProxyGroup
+metadata:
+  name: egress-proxies
+spec:
+  type: egress
+  replicas: 3

cmd/k8s-operator/deploy/examples/recorder.yaml

@@ -0,0 +1,6 @@
+apiVersion: tailscale.com/v1alpha1
+kind: Recorder
+metadata:
+  name: recorder
+spec:
+  enableUI: true

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -30,7 +30,13 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
           securityContext:
-            capabilities:
-              add:
-                - NET_ADMIN
+            privileged: true

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -24,3 +24,11 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid

cmd/k8s-operator/dnsrecords.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"slices"
+	"strings"
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -98,7 +99,15 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.
 		return reconcile.Result{}, nil
 	}
 
-	return reconcile.Result{}, dnsRR.maybeProvision(ctx, headlessSvc, logger)
+	if err := dnsRR.maybeProvision(ctx, headlessSvc, logger); err != nil {
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			logger.Infof("optimistic lock error, retrying: %s", err)
+		} else {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return reconcile.Result{}, nil
 }
 
 // maybeProvision ensures that dnsrecords ConfigMap contains a record for the

cmd/k8s-operator/e2e/ingress_test.go

@@ -0,0 +1,108 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	kube "tailscale.com/k8s-operator"
+	"tailscale.com/tstest"
+)
+
+// See [TestMain] for test requirements.
+func TestIngress(t *testing.T) {
+	if tsClient == nil {
+		t.Skip("TestIngress requires credentials for a tailscale client")
+	}
+
+	ctx := context.Background()
+	cfg := config.GetConfigOrDie()
+	cl, err := client.New(cfg, client.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Apply nginx
+	createAndCleanup(t, ctx, cl, &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nginx",
+			Namespace: "default",
+			Labels: map[string]string{
+				"app.kubernetes.io/name": "nginx",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "nginx",
+					Image: "nginx",
+				},
+			},
+		},
+	})
+	// Apply service to expose it as ingress
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			Annotations: map[string]string{
+				"tailscale.com/expose": "true",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"app.kubernetes.io/name": "nginx",
+			},
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+			},
+		},
+	}
+	createAndCleanup(t, ctx, cl, svc)
+
+	// TODO: instead of timing out only when test times out, cancel context after 60s or so.
+	if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) {
+		maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")}
+		if err := get(ctx, cl, maybeReadySvc); err != nil {
+			return false, err
+		}
+		isReady := kube.SvcIsReady(maybeReadySvc)
+		if isReady {
+			t.Log("Service is ready")
+		}
+		return isReady, nil
+	}); err != nil {
+		t.Fatalf("error waiting for the Service to become Ready: %v", err)
+	}
+
+	var resp *http.Response
+	if err := tstest.WaitFor(time.Second*60, func() error {
+		// TODO(tomhjp): Get the tailnet DNS name from the associated secret instead.
+		// If we are not the first tailnet node with the requested name, we'll get
+		// a -N suffix.
+		resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name))
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("error trying to reach service: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v; response body s", resp.StatusCode)
+	}
+}

cmd/k8s-operator/e2e/main_test.go

@@ -0,0 +1,194 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/go-logr/zapr"
+	"github.com/tailscale/hujson"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/oauth2/clientcredentials"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"tailscale.com/client/tailscale"
+)
+
+const (
+	e2eManagedComment = "// This is managed by the k8s-operator e2e tests"
+)
+
+var (
+	tsClient   *tailscale.Client
+	testGrants = map[string]string{
+		"test-proxy": `{
+			"src": ["tag:e2e-test-proxy"],
+			"dst": ["tag:k8s-operator"],
+			"app": {
+				"tailscale.com/cap/kubernetes": [{
+					"impersonate": {
+						"groups": ["ts:e2e-test-proxy"],
+					},
+				}],
+			},
+		}`,
+	}
+)
+
+// This test suite is currently not run in CI.
+// It requires some setup not handled by this code:
+// - Kubernetes cluster with tailscale operator installed
+// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy)
+// - Operator installed with --set apiServerProxyConfig.mode="true"
+// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key
+// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env
+// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag
+func TestMain(m *testing.M) {
+	code, err := runTests(m)
+	if err != nil {
+		log.Fatal(err)
+	}
+	os.Exit(code)
+}
+
+func runTests(m *testing.M) (int, error) {
+	zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar()
+	logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
+	tailscale.I_Acknowledge_This_API_Is_Unstable = true
+
+	if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" {
+		cleanup, err := setupClientAndACLs()
+		if err != nil {
+			return 0, err
+		}
+		defer func() {
+			err = errors.Join(err, cleanup())
+		}()
+	}
+
+	return m.Run(), nil
+}
+
+func setupClientAndACLs() (cleanup func() error, _ error) {
+	ctx := context.Background()
+	credentials := clientcredentials.Config{
+		ClientID:     os.Getenv("TS_API_CLIENT_ID"),
+		ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"),
+		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
+		Scopes:       []string{"auth_keys", "policy_file"},
+	}
+	tsClient = tailscale.NewClient("-", nil)
+	tsClient.HTTPClient = credentials.Client(ctx)
+
+	if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) {
+		for test, grant := range testGrants {
+			deleteTestGrants(test, acls)
+			addTestGrant(test, grant, acls)
+		}
+	}); err != nil {
+		return nil, err
+	}
+
+	return func() error {
+		return patchACLs(ctx, tsClient, func(acls *hujson.Value) {
+			for test := range testGrants {
+				deleteTestGrants(test, acls)
+			}
+		})
+	}, nil
+}
+
+func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error {
+	acls, err := tsClient.ACLHuJSON(ctx)
+	if err != nil {
+		return err
+	}
+	hj, err := hujson.Parse([]byte(acls.ACL))
+	if err != nil {
+		return err
+	}
+
+	patchFn(&hj)
+
+	hj.Format()
+	acls.ACL = hj.String()
+	if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func addTestGrant(test, grant string, acls *hujson.Value) error {
+	v, err := hujson.Parse([]byte(grant))
+	if err != nil {
+		return err
+	}
+
+	// Add the managed comment to the first line of the grant object contents.
+	v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test))
+
+	if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deleteTestGrants(test string, acls *hujson.Value) error {
+	grants := acls.Find("/grants")
+
+	var patches []string
+	for i, g := range grants.Value.(*hujson.Array).Elements {
+		members := g.Value.(*hujson.Object).Members
+		if len(members) == 0 {
+			continue
+		}
+		comment := strings.TrimSpace(string(members[0].Name.BeforeExtra))
+		if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test {
+			patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i))
+		}
+	}
+
+	// Remove in reverse order so we don't affect the found indices as we mutate.
+	slices.Reverse(patches)
+
+	if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func objectMeta(namespace, name string) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Namespace: namespace,
+		Name:      name,
+	}
+}
+
+func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) {
+	t.Helper()
+	if err := cl.Create(ctx, obj); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := cl.Delete(ctx, obj); err != nil {
+			t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
+		}
+	})
+}
+
+func get(ctx context.Context, cl client.Client, obj client.Object) error {
+	return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)
+}

cmd/k8s-operator/e2e/proxy_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/tsnet"
+	"tailscale.com/tstest"
+)
+
+// See [TestMain] for test requirements.
+func TestProxy(t *testing.T) {
+	if tsClient == nil {
+		t.Skip("TestProxy requires credentials for a tailscale client")
+	}
+
+	ctx := context.Background()
+	cfg := config.GetConfigOrDie()
+	cl, err := client.New(cfg, client.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create role and role binding to allow a group we'll impersonate to do stuff.
+	createAndCleanup(t, ctx, cl, &rbacv1.Role{
+		ObjectMeta: objectMeta("tailscale", "read-secrets"),
+		Rules: []rbacv1.PolicyRule{{
+			APIGroups: []string{""},
+			Verbs:     []string{"get"},
+			Resources: []string{"secrets"},
+		}},
+	})
+	createAndCleanup(t, ctx, cl, &rbacv1.RoleBinding{
+		ObjectMeta: objectMeta("tailscale", "read-secrets"),
+		Subjects: []rbacv1.Subject{{
+			Kind: "Group",
+			Name: "ts:e2e-test-proxy",
+		}},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: "read-secrets",
+		},
+	})
+
+	// Get operator host name from kube secret.
+	operatorSecret := corev1.Secret{
+		ObjectMeta: objectMeta("tailscale", "operator"),
+	}
+	if err := get(ctx, cl, &operatorSecret); err != nil {
+		t.Fatal(err)
+	}
+
+	// Connect to tailnet with test-specific tag so we can use the
+	// [testGrants] ACLs when connecting to the API server proxy
+	ts := tsnetServerWithTag(t, ctx, "tag:e2e-test-proxy")
+	proxyCfg := &rest.Config{
+		Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)),
+		Dial: ts.Dial,
+	}
+	proxyCl, err := client.New(proxyCfg, client.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Expect success.
+	allowedSecret := corev1.Secret{
+		ObjectMeta: objectMeta("tailscale", "operator"),
+	}
+	// Wait for up to a minute the first time we use the proxy, to give it time
+	// to provision the TLS certs.
+	if err := tstest.WaitFor(time.Second*60, func() error {
+		return get(ctx, proxyCl, &allowedSecret)
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Expect forbidden.
+	forbiddenSecret := corev1.Secret{
+		ObjectMeta: objectMeta("default", "operator"),
+	}
+	if err := get(ctx, proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) {
+		t.Fatalf("expected forbidden error fetching secret from default namespace: %s", err)
+	}
+}
+
+func tsnetServerWithTag(t *testing.T, ctx context.Context, tag string) *tsnet.Server {
+	caps := tailscale.KeyCapabilities{
+		Devices: tailscale.KeyDeviceCapabilities{
+			Create: tailscale.KeyDeviceCreateCapabilities{
+				Reusable:      false,
+				Preauthorized: true,
+				Ephemeral:     true,
+				Tags:          []string{tag},
+			},
+		},
+	}
+
+	authKey, authKeyMeta, err := tsClient.CreateKey(ctx, caps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := tsClient.DeleteKey(ctx, authKeyMeta.ID); err != nil {
+			t.Errorf("error deleting auth key: %s", err)
+		}
+	})
+
+	ts := &tsnet.Server{
+		Hostname:  "test-proxy",
+		Ephemeral: true,
+		Dir:       t.TempDir(),
+		AuthKey:   authKey,
+	}
+	_, err = ts.Up(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := ts.Close(); err != nil {
+			t.Errorf("error shutting down tsnet.Server: %s", err)
+		}
+	})
+
+	return ts
+}
+
+func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string {
+	profiles := map[string]any{}
+	if err := json.Unmarshal(s.Data["_profiles"], &profiles); err != nil {
+		t.Fatal(err)
+	}
+	key, ok := strings.CutPrefix(string(s.Data["_current-profile"]), "profile-")
+	if !ok {
+		t.Fatal(string(s.Data["_current-profile"]))
+	}
+	profile, ok := profiles[key]
+	if !ok {
+		t.Fatal(profiles)
+	}
+
+	return ((profile.(map[string]any))["Name"]).(string)
+}

cmd/k8s-operator/egress-eps.go

@@ -0,0 +1,213 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/netip"
+	"reflect"
+	"strings"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsoperator "tailscale.com/k8s-operator"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/types/ptr"
+)
+
+// egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies.
+type egressEpsReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	tsNamespace string
+}
+
+// Reconcile reconciles an EndpointSlice for a tailnet service. It updates the EndpointSlice with the endpoints of
+// those ProxyGroup Pods that are ready to route traffic to the tailnet service.
+// It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired
+// configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready.
+func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := er.logger.With("Service", req.NamespacedName)
+	l.Debugf("starting reconcile")
+	defer l.Debugf("reconcile finished")
+
+	eps := new(discoveryv1.EndpointSlice)
+	err = er.Get(ctx, req.NamespacedName, eps)
+	if apierrors.IsNotFound(err) {
+		l.Debugf("EndpointSlice not found")
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get EndpointSlice: %w", err)
+	}
+	if !eps.DeletionTimestamp.IsZero() {
+		l.Debugf("EnpointSlice is being deleted")
+		return res, nil
+	}
+
+	// Get the user-created ExternalName Service and use its status conditions to determine whether cluster
+	// resources are set up for this tailnet service.
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      eps.Labels[LabelParentName],
+			Namespace: eps.Labels[LabelParentNamespace],
+		},
+	}
+	err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
+	if apierrors.IsNotFound(err) {
+		l.Infof("ExternalName Service %s/%s not found, perhaps it was deleted", svc.Namespace, svc.Name)
+		return res, nil
+	}
+	if err != nil {
+		return res, fmt.Errorf("error retrieving ExternalName Service: %w", err)
+	}
+	if !tsoperator.EgressServiceIsValidAndConfigured(svc) {
+		l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name)
+		return res, nil
+	}
+
+	// TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is
+	// wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to
+	// determine if a reconcile is needed.
+
+	oldEps := eps.DeepCopy()
+	proxyGroupName := eps.Labels[labelProxyGroup]
+	tailnetSvc := tailnetSvcName(svc)
+	l = l.With("tailnet-service-name", tailnetSvc)
+
+	// Retrieve the desired tailnet service configuration from the ConfigMap.
+	_, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace)
+	if err != nil {
+		return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err)
+	}
+	cfg, ok := (*cfgs)[tailnetSvc]
+	if !ok {
+		l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc)
+		return res, nil
+	}
+
+	// Check which Pods in ProxyGroup are ready to route traffic to this
+	// egress service.
+	podList := &corev1.PodList{}
+	if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil {
+		return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
+	}
+	newEndpoints := make([]discoveryv1.Endpoint, 0)
+	for _, pod := range podList.Items {
+		ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, l)
+		if err != nil {
+			return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err)
+		}
+		if !ready {
+			continue // maybe next time
+		}
+		podIP, err := podIPv4(&pod) // we currently only support IPv4
+		if err != nil {
+			return res, fmt.Errorf("error determining IPv4 address for Pod: %w", err)
+		}
+		newEndpoints = append(newEndpoints, discoveryv1.Endpoint{
+			Hostname:  (*string)(&pod.UID),
+			Addresses: []string{podIP},
+			Conditions: discoveryv1.EndpointConditions{
+				Ready:       ptr.To(true),
+				Serving:     ptr.To(true),
+				Terminating: ptr.To(false),
+			},
+		})
+	}
+	// Note that Endpoints are being overwritten with the currently valid endpoints so we don't need to explicitly
+	// run a cleanup for deleted Pods etc.
+	eps.Endpoints = newEndpoints
+	if !reflect.DeepEqual(eps, oldEps) {
+		l.Infof("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods")
+		if err := er.Update(ctx, eps); err != nil {
+			return res, fmt.Errorf("error updating EndpointSlice: %w", err)
+		}
+	}
+	return res, nil
+}
+
+func podIPv4(pod *corev1.Pod) (string, error) {
+	for _, ip := range pod.Status.PodIPs {
+		parsed, err := netip.ParseAddr(ip.IP)
+		if err != nil {
+			return "", fmt.Errorf("error parsing IP address %s: %w", ip, err)
+		}
+		if parsed.Is4() {
+			return parsed.String(), nil
+		}
+	}
+	return "", nil
+}
+
+// podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to
+// route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service
+// status written there to the desired service configuration.
+func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, l *zap.SugaredLogger) (bool, error) {
+	l = l.With("proxy_pod", pod.Name)
+	l.Debugf("checking whether proxy is ready to route to egress service")
+	if !pod.DeletionTimestamp.IsZero() {
+		l.Debugf("proxy Pod is being deleted, ignore")
+		return false, nil
+	}
+	podIP, err := podIPv4(&pod)
+	if err != nil {
+		return false, fmt.Errorf("error determining Pod IP address: %v", err)
+	}
+	if podIP == "" {
+		l.Infof("[unexpected] Pod does not have an IPv4 address, and IPv6 is not currently supported")
+		return false, nil
+	}
+	stateS := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pod.Name,
+			Namespace: pod.Namespace,
+		},
+	}
+	err = er.Get(ctx, client.ObjectKeyFromObject(stateS), stateS)
+	if apierrors.IsNotFound(err) {
+		l.Debugf("proxy does not have a state Secret, waiting...")
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("error getting state Secret: %w", err)
+	}
+	svcStatusBS := stateS.Data[egressservices.KeyEgressServices]
+	if len(svcStatusBS) == 0 {
+		l.Debugf("proxy's state Secret does not contain egress services status, waiting...")
+		return false, nil
+	}
+	svcStatus := &egressservices.Status{}
+	if err := json.Unmarshal(svcStatusBS, svcStatus); err != nil {
+		return false, fmt.Errorf("error unmarshalling egress service status: %w", err)
+	}
+	if !strings.EqualFold(podIP, svcStatus.PodIPv4) {
+		l.Infof("proxy's egress service status is for Pod IP %s, current proxy's Pod IP %s, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP)
+		return false, nil
+	}
+	st, ok := (*svcStatus).Services[tailnetSvcName]
+	if !ok {
+		l.Infof("proxy's state Secret does not have egress service status, waiting...")
+		return false, nil
+	}
+	if !reflect.DeepEqual(cfg.TailnetTarget, st.TailnetTarget) {
+		l.Infof("proxy has configured egress service for tailnet target %v, current target is %v, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget)
+		return false, nil
+	}
+	if !reflect.DeepEqual(cfg.Ports, st.Ports) {
+		l.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports)
+		return false, nil
+	}
+	l.Debugf("proxy is ready to route traffic to egress service")
+	return true, nil
+}

cmd/k8s-operator/egress-eps_test.go

@@ -0,0 +1,211 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand/v2"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/tstest"
+	"tailscale.com/util/mak"
+)
+
+func TestTailscaleEgressEndpointSlices(t *testing.T) {
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: "foo.bar.ts.net",
+				AnnotationProxyGroup:        "foo",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+			},
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{
+				condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, "", "", clock),
+				condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "", "", clock),
+			},
+		},
+	}
+	port := randomPort()
+	cm := configMapForSvc(t, svc, port)
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(svc, cm).
+		WithStatusSubresource(svc).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	er := &egressEpsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		tsNamespace: "operator-ns",
+	}
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: "operator-ns",
+			Labels: map[string]string{
+				LabelParentName:      "test",
+				LabelParentNamespace: "default",
+				labelSvcType:         typeEgress,
+				labelProxyGroup:      "foo"},
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+	mustCreate(t, fc, eps)
+
+	t.Run("no_proxy_group_resources", func(t *testing.T) {
+		expectReconciled(t, er, "operator-ns", "foo") // should not error
+	})
+
+	t.Run("no_pods_ready_to_route_traffic", func(t *testing.T) {
+		pod, stateS := podAndSecretForProxyGroup("foo")
+		mustCreate(t, fc, pod)
+		mustCreate(t, fc, stateS)
+		expectReconciled(t, er, "operator-ns", "foo") // should not error
+	})
+
+	t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) {
+		pod, stateS := podAndSecretForProxyGroup("foo")
+		stBs := serviceStatusForPodIP(t, svc, pod.Status.PodIPs[0].IP, port)
+		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
+			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
+		})
+		expectReconciled(t, er, "operator-ns", "foo")
+		eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
+			Addresses: []string{"10.0.0.1"},
+			Hostname:  pointer.To("foo"),
+			Conditions: discoveryv1.EndpointConditions{
+				Serving:     pointer.ToBool(true),
+				Ready:       pointer.ToBool(true),
+				Terminating: pointer.ToBool(false),
+			},
+		})
+		expectEqual(t, fc, eps)
+	})
+	t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
+		_, stateS := podAndSecretForProxyGroup("foo")           // replica Pod has IP 10.0.0.1
+		stBs := serviceStatusForPodIP(t, svc, "10.0.0.2", port) // status is for a Pod with IP 10.0.0.2
+		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
+			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
+		})
+		expectReconciled(t, er, "operator-ns", "foo")
+		eps.Endpoints = []discoveryv1.Endpoint{}
+		expectEqual(t, fc, eps)
+	})
+}
+
+func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.ConfigMap {
+	t.Helper()
+	ports := make(map[egressservices.PortMap]struct{})
+	for _, port := range svc.Spec.Ports {
+		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
+	}
+	cfg := egressservices.Config{
+		Ports: ports,
+	}
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		cfg.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
+	}
+	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
+		cfg.TailnetTarget = egressservices.TailnetTarget{IP: ip}
+	}
+	name := tailnetSvcName(svc)
+	cfgs := egressservices.Configs{name: cfg}
+	bs, err := json.Marshal(&cfgs)
+	if err != nil {
+		t.Fatalf("error marshalling config: %v", err)
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgEgressCMName(svc.Annotations[AnnotationProxyGroup]),
+			Namespace: "operator-ns",
+		},
+		BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs},
+	}
+	return cm
+}
+
+func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint16) []byte {
+	t.Helper()
+	ports := make(map[egressservices.PortMap]struct{})
+	for _, port := range svc.Spec.Ports {
+		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
+	}
+	svcSt := egressservices.ServiceStatus{Ports: ports}
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		svcSt.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
+	}
+	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
+		svcSt.TailnetTarget = egressservices.TailnetTarget{IP: ip}
+	}
+	svcName := tailnetSvcName(svc)
+	st := egressservices.Status{
+		PodIPv4:  ip,
+		Services: map[string]*egressservices.ServiceStatus{svcName: &svcSt},
+	}
+	bs, err := json.Marshal(st)
+	if err != nil {
+		t.Fatalf("error marshalling service status: %v", err)
+	}
+	return bs
+}
+
+func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
+	p := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-0", pg),
+			Namespace: "operator-ns",
+			Labels:    pgLabels(pg, nil),
+			UID:       "foo",
+		},
+		Status: corev1.PodStatus{
+			PodIPs: []corev1.PodIP{
+				{IP: "10.0.0.1"},
+			},
+		},
+	}
+	s := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-0", pg),
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels(pg, "state"),
+		},
+	}
+	return p, s
+}
+
+func randomPort() uint16 {
+	return uint16(rand.Int32N(1000) + 1000)
+}

cmd/k8s-operator/egress-services-readiness.go

@@ -0,0 +1,179 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstime"
+)
+
+const (
+	reasonReadinessCheckFailed     = "ReadinessCheckFailed"
+	reasonClusterResourcesNotReady = "ClusterResourcesNotReady"
+	reasonNoProxies                = "NoProxiesConfigured"
+	reasonNotReady                 = "NotReadyToRouteTraffic"
+	reasonReady                    = "ReadyToRouteTraffic"
+	reasonPartiallyReady           = "PartiallyReadyToRouteTraffic"
+	msgReadyToRouteTemplate        = "%d out of %d replicas are ready to route traffic"
+)
+
+type egressSvcsReadinessReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	clock       tstime.Clock
+	tsNamespace string
+}
+
+// Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the
+// EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to
+// route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress
+// service to determine how many replicas are currently able to route traffic.
+func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := esrr.logger.With("Service", req.NamespacedName)
+	defer l.Info("reconcile finished")
+
+	svc := new(corev1.Service)
+	if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
+		l.Info("Service not found")
+		return res, nil
+	} else if err != nil {
+		return res, fmt.Errorf("failed to get Service: %w", err)
+	}
+	var (
+		reason, msg string
+		st          metav1.ConditionStatus = metav1.ConditionUnknown
+	)
+	oldStatus := svc.Status.DeepCopy()
+	defer func() {
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
+		if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
+			err = errors.Join(err, esrr.Status().Update(ctx, svc))
+		}
+	}()
+
+	crl := egressSvcChildResourceLabels(svc)
+	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl)
+	if err != nil {
+		err = fmt.Errorf("error getting EndpointSlice: %w", err)
+		reason = reasonReadinessCheckFailed
+		msg = err.Error()
+		return res, err
+	}
+	if eps == nil {
+		l.Infof("EndpointSlice for Service does not yet exist, waiting...")
+		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: svc.Annotations[AnnotationProxyGroup],
+		},
+	}
+	err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg)
+	if apierrors.IsNotFound(err) {
+		l.Infof("ProxyGroup for Service does not exist, waiting...")
+		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	if err != nil {
+		err = fmt.Errorf("error retrieving ProxyGroup: %w", err)
+		reason = reasonReadinessCheckFailed
+		msg = err.Error()
+		return res, err
+	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		l.Infof("ProxyGroup for Service is not ready, waiting...")
+		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+
+	replicas := pgReplicas(pg)
+	if replicas == 0 {
+		l.Infof("ProxyGroup replicas set to 0")
+		reason, msg = reasonNoProxies, reasonNoProxies
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	podLabels := pgLabels(pg.Name, nil)
+	var readyReplicas int32
+	for i := range replicas {
+		podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
+		pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
+		if err != nil {
+			err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err)
+			reason = reasonReadinessCheckFailed
+			msg = err.Error()
+			return res, err
+		}
+		if pod == nil {
+			l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
+			reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+			return res, nil
+		}
+		l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
+		ready := false
+		for _, ep := range eps.Endpoints {
+			l.Infof("looking at endpoint with addresses %v", ep.Addresses)
+			if endpointReadyForPod(&ep, pod, l) {
+				l.Infof("endpoint is ready for Pod")
+				ready = true
+				break
+			}
+		}
+		if ready {
+			readyReplicas++
+		}
+	}
+	msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
+	if readyReplicas == 0 {
+		reason = reasonNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	st = metav1.ConditionTrue
+	if readyReplicas < replicas {
+		reason = reasonPartiallyReady
+	} else {
+		reason = reasonReady
+	}
+	return res, nil
+}
+
+// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic.
+// Endpoint must not be nil.
+func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
+	podIP, err := podIPv4(pod)
+	if err != nil {
+		l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
+		return false
+	}
+	// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.
+	if len(ep.Addresses) != 1 {
+		return false
+	}
+	return strings.EqualFold(ep.Addresses[0], podIP) &&
+		*ep.Conditions.Ready &&
+		*ep.Conditions.Serving &&
+		!*ep.Conditions.Terminating
+}

cmd/k8s-operator/egress-services-readiness_test.go

@@ -0,0 +1,169 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+)
+
+func TestEgressServiceReadiness(t *testing.T) {
+	// We need to pass a ProxyGroup object to WithStatusSubresource because of some quirks in how the fake client
+	// works. Without this code further down would not be able to update ProxyGroup status.
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithStatusSubresource(&tsapi.ProxyGroup{}).
+		Build()
+	zl, _ := zap.NewDevelopment()
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	rec := &egressSvcsReadinessReconciler{
+		tsNamespace: "operator-ns",
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       cl,
+	}
+	tailnetFQDN := "my-app.tailnetxyz.ts.net"
+	egressSvc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-app",
+			Namespace: "dev",
+			Annotations: map[string]string{
+				AnnotationProxyGroup:        "dev",
+				AnnotationTailnetTargetFQDN: tailnetFQDN,
+			},
+		},
+	}
+	fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}}
+	l := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-app",
+			Namespace: "operator-ns",
+			Labels:    l,
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "dev",
+		},
+	}
+	mustCreate(t, fc, egressSvc)
+	setClusterNotReady(egressSvc, cl, zl.Sugar())
+	t.Run("endpointslice_does_not_exist", func(t *testing.T) {
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc) // not ready
+	})
+	t.Run("proxy_group_does_not_exist", func(t *testing.T) {
+		mustCreate(t, fc, eps)
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc) // still not ready
+	})
+	t.Run("proxy_group_not_ready", func(t *testing.T) {
+		mustCreate(t, fc, pg)
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc) // still not ready
+	})
+	t.Run("no_ready_replicas", func(t *testing.T) {
+		setPGReady(pg, cl, zl.Sugar())
+		mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
+			p.Status = pg.Status
+		})
+		expectEqual(t, fc, pg)
+		for i := range pgReplicas(pg) {
+			p := pod(pg, i)
+			mustCreate(t, fc, p)
+			mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
+				existing.Status.PodIPs = p.Status.PodIPs
+			})
+		}
+		expectReconciled(t, rec, "dev", "my-app")
+		setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
+		expectEqual(t, fc, egressSvc) // still not ready
+	})
+	t.Run("one_ready_replica", func(t *testing.T) {
+		setEndpointForReplica(pg, 0, eps)
+		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
+			e.Endpoints = eps.Endpoints
+		})
+		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc) // partially ready
+	})
+	t.Run("all_replicas_ready", func(t *testing.T) {
+		for i := range pgReplicas(pg) {
+			setEndpointForReplica(pg, i, eps)
+		}
+		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
+			e.Endpoints = eps.Endpoints
+		})
+		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc) // ready
+	})
+}
+
+func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger) {
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, l)
+}
+
+func setNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas int32) {
+	msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas)
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, l)
+}
+
+func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas, readyReplicas int32) {
+	reason := reasonPartiallyReady
+	if readyReplicas == replicas {
+		reason = reasonReady
+	}
+	msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, l)
+}
+
+func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) {
+	tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l)
+}
+
+func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {
+	p := pod(pg, ordinal)
+	eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
+		Addresses: []string{p.Status.PodIPs[0].IP},
+		Conditions: discoveryv1.EndpointConditions{
+			Ready:       pointer.ToBool(true),
+			Serving:     pointer.ToBool(true),
+			Terminating: pointer.ToBool(false),
+		},
+	})
+}
+
+func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
+	l := pgLabels(pg.Name, nil)
+	l[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
+	ip := fmt.Sprintf("10.0.0.%d", ordinal)
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%d", pg.Name, ordinal),
+			Namespace: "operator-ns",
+			Labels:    l,
+		},
+		Status: corev1.PodStatus{
+			PodIPs: []corev1.PodIP{{IP: ip}},
+		},
+	}
+}

cmd/k8s-operator/egress-services.go

@@ -0,0 +1,742 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"reflect"
+	"slices"
+	"strings"
+	"sync"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/storage/names"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonEgressSvcInvalid        = "EgressSvcInvalid"
+	reasonEgressSvcValid          = "EgressSvcValid"
+	reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
+	reasonProxyGroupNotReady      = "ProxyGroupNotReady"
+
+	labelProxyGroup = "tailscale.com/proxy-group"
+
+	labelSvcType = "tailscale.com/svc-type" // ingress or egress
+	typeEgress   = "egress"
+	// maxPorts is the maximum number of ports that can be exposed on a
+	// container. In practice this will be ports in range [10000 - 11000). The
+	// high range should make it easier to distinguish container ports from
+	// the tailnet target ports for debugging purposes (i.e when reading
+	// netfilter rules). The limit of 1000 is somewhat arbitrary, the
+	// assumption is that this would not be hit in practice.
+	maxPorts = 1000
+
+	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
+)
+
+var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
+
+// egressSvcsReconciler reconciles user created ExternalName Services that specify a tailnet
+// endpoint that should be exposed to cluster workloads and an egress ProxyGroup
+// on whose proxies it should be exposed.
+type egressSvcsReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+
+	mu   sync.Mutex           // protects following
+	svcs set.Slice[types.UID] // UIDs of all currently managed egress Services for ProxyGroup
+}
+
+// Reconcile reconciles an ExternalName Service that specifies a tailnet target and a ProxyGroup on whose proxies should
+// forward cluster traffic to the target.
+// For an ExternalName Service the reconciler:
+//
+// - for each port N defined on the ExternalName Service, allocates a port X in range [3000- 4000), unique for the
+// ProxyGroup proxies. Proxies will forward cluster traffic received on port N to port M on the tailnet target
+//
+// - creates a ClusterIP Service in the operator's namespace with portmappings for all M->N port pairs. This will allow
+// cluster workloads to send traffic on the user-defined tailnet target port and get it transparently mapped to the
+// randomly selected port on proxy Pods.
+//
+// - creates an EndpointSlice in the operator's namespace with kubernetes.io/service-name label pointing to the
+// ClusterIP Service. The endpoints will get dynamically updates to proxy Pod IPs as the Pods become ready to route
+// traffic to the tailnet target. kubernetes.io/service-name label ensures that kube-proxy sets up routing rules to
+// forward cluster traffic received on ClusterIP Service's IP address to the endpoints (Pod IPs).
+//
+// - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the
+// portmappings.
+func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := esr.logger.With("Service", req.NamespacedName)
+	defer l.Info("reconcile finished")
+
+	svc := new(corev1.Service)
+	if err = esr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
+		l.Info("Service not found")
+		return res, nil
+	} else if err != nil {
+		return res, fmt.Errorf("failed to get Service: %w", err)
+	}
+
+	// Name of the 'egress service', meaning the tailnet target.
+	tailnetSvc := tailnetSvcName(svc)
+	l = l.With("tailnet-service", tailnetSvc)
+
+	// Note that resources for egress Services are only cleaned up when the
+	// Service is actually deleted (and not if, for example, user decides to
+	// remove the Tailscale annotation from it). This should be fine- we
+	// assume that the egress ExternalName Services are always created for
+	// Tailscale operator specifically.
+	if !svc.DeletionTimestamp.IsZero() {
+		l.Info("Service is being deleted, ensuring resource cleanup")
+		return res, esr.maybeCleanup(ctx, svc, l)
+	}
+
+	oldStatus := svc.Status.DeepCopy()
+	defer func() {
+		if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
+			err = errors.Join(err, esr.Status().Update(ctx, svc))
+		}
+	}()
+
+	// Validate the user-created ExternalName Service and the associated ProxyGroup.
+	if ok, err := esr.validateClusterResources(ctx, svc, l); err != nil {
+		return res, fmt.Errorf("error validating cluster resources: %w", err)
+	} else if !ok {
+		return res, nil
+	}
+
+	if !slices.Contains(svc.Finalizers, FinalizerName) {
+		svc.Finalizers = append(svc.Finalizers, FinalizerName)
+		if err := esr.updateSvcSpec(ctx, svc); err != nil {
+			err := fmt.Errorf("failed to add finalizer: %w", err)
+			r := svcConfiguredReason(svc, false, l)
+			tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
+			return res, err
+		}
+		esr.mu.Lock()
+		esr.svcs.Add(svc.UID)
+		gaugeEgressServices.Set(int64(esr.svcs.Len()))
+		esr.mu.Unlock()
+	}
+
+	if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, l); err != nil {
+		err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err)
+		r := svcConfiguredReason(svc, false, l)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
+		return res, err
+	}
+
+	if err := esr.maybeProvision(ctx, svc, l); err != nil {
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			l.Infof("optimistic lock error, retrying: %s", err)
+		} else {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return res, nil
+}
+
+func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) {
+	r := svcConfiguredReason(svc, false, l)
+	st := metav1.ConditionFalse
+	defer func() {
+		msg := r
+		if st != metav1.ConditionTrue && err != nil {
+			msg = err.Error()
+		}
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, l)
+	}()
+
+	crl := egressSvcChildResourceLabels(svc)
+	clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl)
+	if err != nil {
+		err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
+		return err
+	}
+	if clusterIPSvc == nil {
+		clusterIPSvc = esr.clusterIPSvcForEgress(crl)
+	}
+	upToDate := svcConfigurationUpToDate(svc, l)
+	provisioned := true
+	if !upToDate {
+		if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, l); err != nil {
+			return err
+		}
+	}
+	if !provisioned {
+		l.Infof("unable to provision cluster resources")
+		return nil
+	}
+
+	// Update ExternalName Service to point at the ClusterIP Service.
+	clusterDomain := retrieveClusterDomain(esr.tsNamespace, l)
+	clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain)
+	if svc.Spec.ExternalName != clusterIPSvcFQDN {
+		l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
+		svc.Spec.ExternalName = clusterIPSvcFQDN
+		if err = esr.updateSvcSpec(ctx, svc); err != nil {
+			err = fmt.Errorf("error updating ExternalName Service: %w", err)
+			return err
+		}
+	}
+	r = svcConfiguredReason(svc, true, l)
+	st = metav1.ConditionTrue
+	return nil
+}
+
+func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, l *zap.SugaredLogger) (*corev1.Service, bool, error) {
+	l.Infof("updating configuration...")
+	usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName)
+	if err != nil {
+		return nil, false, fmt.Errorf("error calculating used ports for ProxyGroup %s: %w", proxyGroupName, err)
+	}
+
+	oldClusterIPSvc := clusterIPSvc.DeepCopy()
+	// loop over ClusterIP Service ports, remove any that are not needed.
+	for i := len(clusterIPSvc.Spec.Ports) - 1; i >= 0; i-- {
+		pm := clusterIPSvc.Spec.Ports[i]
+		found := false
+		for _, wantsPM := range svc.Spec.Ports {
+			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
+				// We don't use the port name to distinguish this port internally, but Kubernetes
+				// require that, for Service ports with more than one name each port is uniquely named.
+				// So we can always pick the port name from the ExternalName Service as at this point we
+				// know that those are valid names because Kuberentes already validated it once. Note
+				// that users could have changed an unnamed port to a named port and might have changed
+				// port names- this should still work.
+				// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
+				// See also https://github.com/tailscale/tailscale/issues/13406#issuecomment-2507230388
+				clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
+				found = true
+				break
+			}
+		}
+		if !found {
+			l.Debugf("portmapping %s:%d -> %s:%d is no longer required, removing", pm.Protocol, pm.TargetPort.IntVal, pm.Protocol, pm.Port)
+			clusterIPSvc.Spec.Ports = slices.Delete(clusterIPSvc.Spec.Ports, i, i+1)
+		}
+	}
+
+	// loop over ExternalName Service ports, for each one not found on
+	// ClusterIP Service produce new target port and add a portmapping to
+	// the ClusterIP Service.
+	for _, wantsPM := range svc.Spec.Ports {
+		found := false
+		for _, gotPM := range clusterIPSvc.Spec.Ports {
+			if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			// Calculate a free port to expose on container and add
+			// a new PortMap to the ClusterIP Service.
+			if usedPorts.Len() >= maxPorts {
+				// TODO(irbekrm): refactor to avoid extra reconciles here. Low priority as in practice,
+				// the limit should not be hit.
+				return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts)
+			}
+			p := unusedPort(usedPorts)
+			l.Debugf("mapping tailnet target port %d to container port %d", wantsPM.Port, p)
+			usedPorts.Insert(p)
+			clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
+				Name:       wantsPM.Name,
+				Protocol:   wantsPM.Protocol,
+				Port:       wantsPM.Port,
+				TargetPort: intstr.FromInt32(p),
+			})
+		}
+	}
+	if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
+		if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
+			svc.Labels = clusterIPSvc.Labels
+			svc.Spec = clusterIPSvc.Spec
+		}); err != nil {
+			return nil, false, fmt.Errorf("error ensuring ClusterIP Service: %v", err)
+		}
+	}
+
+	crl := egressSvcEpsLabels(svc, clusterIPSvc)
+	// TODO(irbekrm): support IPv6, but need to investigate how kube proxy
+	// sets up Service -> Pod routing when IPv6 is involved.
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
+			Namespace: esr.tsNamespace,
+			Labels:    crl,
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Ports:       epsPortsFromSvc(clusterIPSvc),
+	}
+	if eps, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
+		e.Labels = eps.Labels
+		e.AddressType = eps.AddressType
+		e.Ports = eps.Ports
+		for _, p := range e.Endpoints {
+			p.Conditions.Ready = nil
+		}
+	}); err != nil {
+		return nil, false, fmt.Errorf("error ensuring EndpointSlice: %w", err)
+	}
+
+	cm, cfgs, err := egressSvcsConfigs(ctx, esr.Client, proxyGroupName, esr.tsNamespace)
+	if err != nil {
+		return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err)
+	}
+	if cm == nil {
+		l.Info("ConfigMap not yet created, waiting..")
+		return nil, false, nil
+	}
+	tailnetSvc := tailnetSvcName(svc)
+	gotCfg := (*cfgs)[tailnetSvc]
+	wantsCfg := egressSvcCfg(svc, clusterIPSvc)
+	if !reflect.DeepEqual(gotCfg, wantsCfg) {
+		l.Debugf("updating egress services ConfigMap %s", cm.Name)
+		mak.Set(cfgs, tailnetSvc, wantsCfg)
+		bs, err := json.Marshal(cfgs)
+		if err != nil {
+			return nil, false, fmt.Errorf("error marshalling egress services configs: %w", err)
+		}
+		mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
+		if err := esr.Update(ctx, cm); err != nil {
+			return nil, false, fmt.Errorf("error updating egress services ConfigMap: %w", err)
+		}
+	}
+	l.Infof("egress service configuration has been updated")
+	return clusterIPSvc, true, nil
+}
+
+func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
+	logger.Info("ensuring that resources created for egress service are deleted")
+
+	// Delete egress service config from the ConfigMap mounted by the proxies.
+	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, logger); err != nil {
+		return fmt.Errorf("error deleting egress service config: %w", err)
+	}
+
+	// Delete the ClusterIP Service and EndpointSlice for the egress
+	// service.
+	types := []client.Object{
+		&corev1.Service{},
+		&discoveryv1.EndpointSlice{},
+	}
+	crl := egressSvcChildResourceLabels(svc)
+	for _, typ := range types {
+		if err := esr.DeleteAllOf(ctx, typ, client.InNamespace(esr.tsNamespace), client.MatchingLabels(crl)); err != nil {
+			return fmt.Errorf("error deleting %s: %w", typ, err)
+		}
+	}
+
+	ix := slices.Index(svc.Finalizers, FinalizerName)
+	if ix != -1 {
+		logger.Debug("Removing Tailscale finalizer from Service")
+		svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
+		if err := esr.Update(ctx, svc); err != nil {
+			return fmt.Errorf("failed to remove finalizer: %w", err)
+		}
+	}
+	esr.mu.Lock()
+	esr.svcs.Remove(svc.UID)
+	gaugeEgressServices.Set(int64(esr.svcs.Len()))
+	esr.mu.Unlock()
+	logger.Info("successfully cleaned up resources for egress Service")
+	return nil
+}
+
+func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) error {
+	wantsProxyGroup := svc.Annotations[AnnotationProxyGroup]
+	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
+	if cond == nil {
+		return nil
+	}
+	ss := strings.Split(cond.Reason, ":")
+	if len(ss) < 3 {
+		return nil
+	}
+	if strings.EqualFold(wantsProxyGroup, ss[2]) {
+		return nil
+	}
+	esr.logger.Infof("egress Service configured on ProxyGroup %s, wants ProxyGroup %s, cleaning up...", ss[2], wantsProxyGroup)
+	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, l); err != nil {
+		return fmt.Errorf("error deleting egress service config: %w", err)
+	}
+	return nil
+}
+
+// usedPortsForPG calculates the currently used match ports for ProxyGroup
+// containers. It does that by looking by retrieving all target ports of all
+// ClusterIP Services created for egress services exposed on this ProxyGroup's
+// proxies.
+// TODO(irbekrm): this is currently good enough because we only have a single worker and
+// because these Services are created by us, so we can always expect to get the
+// latest ClusterIP Services via the controller cache. It will not work as well
+// once we split into multiple workers- at that point we probably want to set
+// used ports on ProxyGroup's status.
+func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) {
+	svcList := &corev1.ServiceList{}
+	if err := esr.List(ctx, svcList, client.InNamespace(esr.tsNamespace), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
+		return nil, fmt.Errorf("error listing Services: %w", err)
+	}
+	usedPorts := sets.New[int32]()
+	for _, s := range svcList.Items {
+		for _, p := range s.Spec.Ports {
+			usedPorts.Insert(p.TargetPort.IntVal)
+		}
+	}
+	return usedPorts, nil
+}
+
+// clusterIPSvcForEgress returns a template for the ClusterIP Service created
+// for an egress service exposed on ProxyGroup proxies. The ClusterIP Service
+// has no selector. Traffic sent to it will be routed to the endpoints defined
+// by an EndpointSlice created for this egress service.
+func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: svcNameBase(crl[LabelParentName]),
+			Namespace:    esr.tsNamespace,
+			Labels:       crl,
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeClusterIP,
+		},
+	}
+}
+
+func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
+	crl := egressSvcChildResourceLabels(svc)
+	cmName := pgEgressCMName(crl[labelProxyGroup])
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cmName,
+			Namespace: esr.tsNamespace,
+		},
+	}
+	l := logger.With("ConfigMap", client.ObjectKeyFromObject(cm))
+	l.Debug("ensuring that egress service configuration is removed from proxy config")
+	if err := esr.Get(ctx, client.ObjectKeyFromObject(cm), cm); apierrors.IsNotFound(err) {
+		l.Debugf("ConfigMap not found")
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("error retrieving ConfigMap: %w", err)
+	}
+	bs := cm.BinaryData[egressservices.KeyEgressServices]
+	if len(bs) == 0 {
+		l.Debugf("ConfigMap does not contain egress service configs")
+		return nil
+	}
+	cfgs := &egressservices.Configs{}
+	if err := json.Unmarshal(bs, cfgs); err != nil {
+		return fmt.Errorf("error unmarshalling egress services configs")
+	}
+	tailnetSvc := tailnetSvcName(svc)
+	_, ok := (*cfgs)[tailnetSvc]
+	if !ok {
+		l.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted")
+		return nil
+	}
+	l.Infof("before deleting config %+#v", *cfgs)
+	delete(*cfgs, tailnetSvc)
+	l.Infof("after deleting config %+#v", *cfgs)
+	bs, err := json.Marshal(cfgs)
+	if err != nil {
+		return fmt.Errorf("error marshalling egress services configs: %w", err)
+	}
+	mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
+	return esr.Update(ctx, cm)
+}
+
+func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (bool, error) {
+	proxyGroupName := svc.Annotations[AnnotationProxyGroup]
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: proxyGroupName,
+		},
+	}
+	if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
+		l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	} else if err != nil {
+		err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, err
+	}
+	if violations := validateEgressService(svc, pg); len(violations) > 0 {
+		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
+		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
+		l.Info(msg)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	}
+
+	l.Debugf("egress service is valid")
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
+	return true, nil
+}
+
+func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
+	violations := validateService(svc)
+
+	// We check that only one of these two is set in the earlier validateService function.
+	if svc.Annotations[AnnotationTailnetTargetFQDN] == "" && svc.Annotations[AnnotationTailnetTargetIP] == "" {
+		violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", AnnotationTailnetTargetFQDN, AnnotationTailnetTargetIP))
+	}
+	if len(svc.Spec.Ports) == 0 {
+		violations = append(violations, "egress Service for ProxyGroup must have at least one target Port specified")
+	}
+	if svc.Spec.Type != corev1.ServiceTypeExternalName {
+		violations = append(violations, fmt.Sprintf("unexpected egress Service type %s. The only supported type is ExternalName.", svc.Spec.Type))
+	}
+	if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
+		violations = append(violations, fmt.Sprintf("egress Service references ProxyGroup of type %s, must be type %s", pg.Spec.Type, tsapi.ProxyGroupTypeEgress))
+	}
+	return violations
+}
+
+// egressSvcNameBase returns a name base that can be passed to
+// ObjectMeta.GenerateName to generate a name for the ClusterIP Service.
+// The generated name needs to be short enough so that it can later be used to
+// generate a valid Kubernetes resource name for the EndpointSlice in form
+// 'ipv4-|ipv6-<ClusterIP Service name>.
+// A valid Kubernetes resource name must not be longer than 253 chars.
+func svcNameBase(s string) string {
+	// -ipv4 - ipv6
+	const maxClusterIPSvcNameLength = 253 - 5
+	base := fmt.Sprintf("ts-%s-", s)
+	generator := names.SimpleNameGenerator
+	for {
+		generatedName := generator.GenerateName(base)
+		excess := len(generatedName) - maxClusterIPSvcNameLength
+		if excess <= 0 {
+			return base
+		}
+		base = base[:len(base)-1-excess] // cut off the excess chars
+		base = base + "-"                // re-instate the dash
+	}
+}
+
+// unusedPort returns a port in range [10000 - 11000). The caller must ensure that
+// usedPorts does not contain all ports in range [10000 - 11000).
+func unusedPort(usedPorts sets.Set[int32]) int32 {
+	foundFreePort := false
+	var suggestPort int32
+	for !foundFreePort {
+		suggestPort = rand.Int32N(maxPorts) + 10000
+		if !usedPorts.Has(suggestPort) {
+			foundFreePort = true
+		}
+	}
+	return suggestPort
+}
+
+// tailnetTargetFromSvc returns a tailnet target for the given egress Service.
+// Service must contain exactly one of tailscale.com/tailnet-ip,
+// tailscale.com/tailnet-fqdn annotations.
+func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		return egressservices.TailnetTarget{
+			FQDN: fqdn,
+		}
+	}
+	return egressservices.TailnetTarget{
+		IP: svc.Annotations[AnnotationTailnetTargetIP],
+	}
+}
+
+func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config {
+	tt := tailnetTargetFromSvc(externalNameSvc)
+	cfg := egressservices.Config{TailnetTarget: tt}
+	for _, svcPort := range clusterIPSvc.Spec.Ports {
+		pm := portMap(svcPort)
+		mak.Set(&cfg.Ports, pm, struct{}{})
+	}
+	return cfg
+}
+
+func portMap(p corev1.ServicePort) egressservices.PortMap {
+	// TODO (irbekrm): out of bounds check?
+	return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)}
+}
+
+func isEgressSvcForProxyGroup(obj client.Object) bool {
+	s, ok := obj.(*corev1.Service)
+	if !ok {
+		return false
+	}
+	annots := s.ObjectMeta.Annotations
+	return annots[AnnotationProxyGroup] != "" && (annots[AnnotationTailnetTargetFQDN] != "" || annots[AnnotationTailnetTargetIP] != "")
+}
+
+// egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well
+// as unmarshalled configuration from the ConfigMap.
+func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) {
+	name := pgEgressCMName(proxyGroupName)
+	cm = &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: tsNamespace,
+		},
+	}
+	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
+		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
+	}
+	cfgs = &egressservices.Configs{}
+	if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 {
+		if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], cfgs); err != nil {
+			return nil, nil, fmt.Errorf("error unmarshaling egress services config %v: %w", cm.BinaryData[egressservices.KeyEgressServices], err)
+		}
+	}
+	return cm, cfgs, nil
+}
+
+// egressSvcChildResourceLabels returns labels that should be applied to the
+// ClusterIP Service and the EndpointSlice created for the egress service.
+// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes
+// resource names (ProxyGroup, Service). Maximum allowed label length is 63
+// chars whilst the maximum allowed resource name length is 253 chars, so we
+// should probably validate and truncate (?) the names is they are too long.
+func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
+	return map[string]string{
+		LabelManaged:         "true",
+		LabelParentType:      "svc",
+		LabelParentName:      svc.Name,
+		LabelParentNamespace: svc.Namespace,
+		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:         typeEgress,
+	}
+}
+
+// egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service.
+func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string {
+	l := egressSvcChildResourceLabels(extNSvc)
+	// Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the
+	// endpoints defined on this EndpointSlice.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
+	l[discoveryv1.LabelServiceName] = clusterIPSvc.Name
+	// Kubernetes recommends setting this label.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management
+	l[discoveryv1.LabelManagedBy] = "tailscale.com"
+	return l
+}
+
+func svcConfigurationUpToDate(svc *corev1.Service, l *zap.SugaredLogger) bool {
+	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
+	if cond == nil {
+		return false
+	}
+	if cond.Status != metav1.ConditionTrue {
+		return false
+	}
+	wantsReadyReason := svcConfiguredReason(svc, true, l)
+	return strings.EqualFold(wantsReadyReason, cond.Reason)
+}
+
+func cfgHash(c cfg, l *zap.SugaredLogger) string {
+	bs, err := json.Marshal(c)
+	if err != nil {
+		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
+		l.Infof("error marhsalling Config: %v", err)
+		return ""
+	}
+	h := sha256.New()
+	if _, err := h.Write(bs); err != nil {
+		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
+		l.Infof("error producing Config hash: %v", err)
+		return ""
+	}
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+type cfg struct {
+	Ports         []corev1.ServicePort         `json:"ports"`
+	TailnetTarget egressservices.TailnetTarget `json:"tailnetTarget"`
+	ProxyGroup    string                       `json:"proxyGroup"`
+}
+
+func svcConfiguredReason(svc *corev1.Service, configured bool, l *zap.SugaredLogger) string {
+	var r string
+	if configured {
+		r = "ConfiguredFor:"
+	} else {
+		r = fmt.Sprintf("ConfigurationFailed:%s", r)
+	}
+	r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[AnnotationProxyGroup])
+	tt := tailnetTargetFromSvc(svc)
+	s := cfg{
+		Ports:         svc.Spec.Ports,
+		TailnetTarget: tt,
+		ProxyGroup:    svc.Annotations[AnnotationProxyGroup],
+	}
+	r += fmt.Sprintf(":Config:%s", cfgHash(s, l))
+	return r
+}
+
+// tailnetSvc accepts and ExternalName Service name and returns a name that will be used to distinguish this tailnet
+// service from other tailnet services exposed to cluster workloads.
+func tailnetSvcName(extNSvc *corev1.Service) string {
+	return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name)
+}
+
+// epsPortsFromSvc takes the ClusterIP Service created for an egress service and
+// returns its Port array in a form that can be used for an EndpointSlice.
+func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
+	for _, p := range svc.Spec.Ports {
+		ep = append(ep, discoveryv1.EndpointPort{
+			Protocol: &p.Protocol,
+			Port:     &p.TargetPort.IntVal,
+			Name:     &p.Name,
+		})
+	}
+	return ep
+}
+
+// updateSvcSpec ensures that the given Service's spec is updated in cluster, but the local Service object still retains
+// the not-yet-applied status.
+// TODO(irbekrm): once we do SSA for these patch updates, this will no longer be needed.
+func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error {
+	st := svc.Status.DeepCopy()
+	err := esr.Update(ctx, svc)
+	svc.Status = *st
+	return err
+}

cmd/k8s-operator/egress-services_test.go

@@ -0,0 +1,303 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+)
+
+func TestTailscaleEgressServices(t *testing.T) {
+	pg := &tsapi.ProxyGroup{
+		TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foo",
+			UID:  types.UID("1234-UID"),
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Replicas: pointer.To[int32](3),
+			Type:     tsapi.ProxyGroupTypeEgress,
+		},
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgEgressCMName("foo"),
+			Namespace: "operator-ns",
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pg, cm).
+		WithStatusSubresource(pg).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+
+	esr := &egressSvcsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       clock,
+		tsNamespace: "operator-ns",
+	}
+	tailnetTargetFQDN := "foo.bar.ts.net."
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
+				AnnotationProxyGroup:        "foo",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+				{
+					Name:     "https",
+					Protocol: "TCP",
+					Port:     443,
+				},
+			},
+		},
+	}
+
+	t.Run("proxy_group_not_ready", func(t *testing.T) {
+		mustCreate(t, fc, svc)
+		expectReconciled(t, esr, "default", "test")
+		// Service should have EgressSvcValid condition set to Unknown.
+		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
+		expectEqual(t, fc, svc)
+	})
+
+	t.Run("proxy_group_ready", func(t *testing.T) {
+		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
+			pg.Status.Conditions = []metav1.Condition{
+				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
+			}
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_retain_one_unnamed_port", func(t *testing.T) {
+		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80}}
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_add_two_named_ports", func(t *testing.T) {
+		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}}
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_add_udp_port", func(t *testing.T) {
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Port: 53, Protocol: "UDP", Name: "dns"})
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_change_protocol", func(t *testing.T) {
+		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}, {Port: 53, Protocol: "TCP", Name: "tcp_dns"}}
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+
+	t.Run("delete_external_name_service", func(t *testing.T) {
+		name := findGenNameForEgressSvcResources(t, fc, svc)
+		if err := fc.Delete(context.Background(), svc); err != nil {
+			t.Fatalf("error deleting ExternalName Service: %v", err)
+		}
+		expectReconciled(t, esr, "default", "test")
+		// Verify that ClusterIP Service and EndpointSlice have been deleted.
+		expectMissing[corev1.Service](t, fc, "operator-ns", name)
+		expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
+		// Verify that service config has been deleted from the ConfigMap.
+		mustNotHaveConfigForSvc(t, fc, svc, cm)
+	})
+}
+
+func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) {
+	expectReconciled(t, esr, "default", "test")
+	// Verify that a ClusterIP Service has been created.
+	name := findGenNameForEgressSvcResources(t, fc, svc)
+	expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
+	clusterSvc := mustGetClusterIPSvc(t, fc, name)
+	// Verify that an EndpointSlice has been created.
+	expectEqual(t, fc, endpointSlice(name, svc, clusterSvc))
+	// Verify that ConfigMap contains configuration for the new egress service.
+	mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
+	r := svcConfiguredReason(svc, true, zl.Sugar())
+	// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
+	// CluterIP Service.
+	svc.Status.Conditions = []metav1.Condition{
+		condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "EgressSvcValid", "EgressSvcValid", clock),
+		condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
+	}
+	svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
+	svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
+	expectEqual(t, fc, svc)
+
+}
+
+func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
+	return metav1.Condition{
+		Type:               string(typ),
+		Status:             st,
+		LastTransitionTime: conditionTime(clock),
+		Reason:             r,
+		Message:            msg,
+	}
+}
+
+func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
+	t.Helper()
+	labels := egressSvcChildResourceLabels(svc)
+	s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
+	if err != nil {
+		t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
+	}
+	if s == nil {
+		t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name)
+	}
+	return s.GetName()
+}
+
+func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
+	labels := egressSvcChildResourceLabels(extNSvc)
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:         name,
+			Namespace:    "operator-ns",
+			GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name),
+			Labels:       labels,
+		},
+		Spec: corev1.ServiceSpec{
+			Type:  corev1.ServiceTypeClusterIP,
+			Ports: extNSvc.Spec.Ports,
+		},
+	}
+}
+
+func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service {
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: "operator-ns",
+		},
+	}
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil {
+		t.Fatalf("error retrieving Service")
+	}
+	return svc
+}
+
+func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
+	labels := egressSvcChildResourceLabels(extNSvc)
+	labels[discoveryv1.LabelManagedBy] = "tailscale.com"
+	labels[discoveryv1.LabelServiceName] = name
+	return &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-ipv4", name),
+			Namespace: "operator-ns",
+			Labels:    labels,
+		},
+		Ports:       portsForEndpointSlice(clusterIPSvc),
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+}
+
+func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
+	ports := make([]discoveryv1.EndpointPort, 0)
+	for _, p := range svc.Spec.Ports {
+		ports = append(ports, discoveryv1.EndpointPort{
+			Name:     &p.Name,
+			Protocol: &p.Protocol,
+			Port:     pointer.ToInt32(p.TargetPort.IntVal),
+		})
+	}
+	return ports
+}
+
+func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
+	t.Helper()
+	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
+		t.Fatalf("Error retrieving ConfigMap: %v", err)
+	}
+	name := tailnetSvcName(extNSvc)
+	gotCfg := configFromCM(t, cm, name)
+	if gotCfg == nil {
+		t.Fatalf("No config found for service %q", name)
+	}
+	if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" {
+		t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff)
+	}
+}
+
+func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) {
+	t.Helper()
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
+		t.Fatalf("Error retrieving ConfigMap: %v", err)
+	}
+	name := tailnetSvcName(extNSvc)
+	gotCfg := configFromCM(t, cm, name)
+	if gotCfg != nil {
+		t.Fatalf("Config  %#+v for service %q found when it should not be present", gotCfg, name)
+	}
+}
+
+func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config {
+	t.Helper()
+	cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices]
+	if !ok {
+		return nil
+	}
+	cfgs := &egressservices.Configs{}
+	if err := json.Unmarshal(cfgBs, cfgs); err != nil {
+		t.Fatalf("error unmarshalling config: %v", err)
+	}
+	cfg, ok := (*cfgs)[svcName]
+	if ok {
+		return &cfg
+	}
+	return nil
+}

cmd/k8s-operator/generate/main.go

@@ -24,10 +24,14 @@ const (
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
+	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
+	proxyGroupCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
+	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
+	proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -111,7 +115,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass, DNSConfig, Recorder) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -137,28 +141,34 @@ func generate(baseDir string) error {
 		}
 		return nil
 	}
-	if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
+	for _, crd := range []struct {
+		crdPath, templatePath string
+	}{
+		{connectorCRDPath, connectorCRDHelmTemplatePath},
+		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
+		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
+		{recorderCRDPath, recorderCRDHelmTemplatePath},
+		{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
+	} {
+		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
+			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
+		}
 	}
 	return nil
 }
 
 func cleanup(baseDir string) error {
 	log.Print("Cleaning up CRD from Helm templates")
-	if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up Connector CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
+	for _, path := range []string{
+		connectorCRDHelmTemplatePath,
+		proxyClassCRDHelmTemplatePath,
+		dnsConfigCRDHelmTemplatePath,
+		recorderCRDHelmTemplatePath,
+		proxyGroupCRDHelmTemplatePath,
+	} {
+		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("error cleaning up %s: %w", path, err)
+		}
 	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -59,6 +59,12 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD not found in default chart install")
+	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: proxygroups.tailscale.com") {
+		t.Errorf("ProxyGroup CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -77,4 +83,10 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
+	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: proxygroups.tailscale.com") {
+		t.Errorf("ProxyGroup CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/ingress-for-pg.go

@@ -0,0 +1,569 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"reflect"
+	"slices"
+	"strings"
+	"sync"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
+)
+
+const (
+	serveConfigKey = "serve-config.json"
+	VIPSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s"
+	// FinalizerNamePG is the finalizer used by the IngressPGReconciler
+	FinalizerNamePG = "tailscale.com/ingress-pg-finalizer"
+)
+
+var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
+
+// IngressPGReconciler is a controller that reconciles Tailscale Ingresses should be exposed on an ingress ProxyGroup
+// (in HA mode).
+type IngressPGReconciler struct {
+	client.Client
+
+	recorder    record.EventRecorder
+	logger      *zap.SugaredLogger
+	tsClient    tsClient
+	tsnetServer tsnetServer
+	tsNamespace string
+	lc          localClient
+	defaultTags []string
+
+	mu sync.Mutex // protects following
+	// managedIngresses is a set of all ingress resources that we're currently
+	// managing. This is only used for metrics.
+	managedIngresses set.Slice[types.UID]
+}
+
+// Reconcile reconciles Ingresses that should be exposed over Tailscale in HA mode (on a ProxyGroup). It looks at all
+// Ingresses with tailscale.com/proxy-group annotation. For each such Ingress, it ensures that a VIPService named after
+// the hostname of the Ingress exists and is up to date. It also ensures that the serve config for the ingress
+// ProxyGroup is updated to route traffic for the VIPService to the Ingress's backend Services.
+// When an Ingress is deleted or unexposed, the VIPService and the associated serve config are cleaned up.
+// Ingress hostname change also results in the VIPService for the previous hostname being cleaned up and a new VIPService
+// being created for the new hostname.
+func (a *IngressPGReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	logger := a.logger.With("Ingress", req.NamespacedName)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	ing := new(networkingv1.Ingress)
+	err = a.Get(ctx, req.NamespacedName, ing)
+	if apierrors.IsNotFound(err) {
+		// Request object not found, could have been deleted after reconcile request.
+		logger.Debugf("Ingress not found, assuming it was deleted")
+		return res, nil
+	} else if err != nil {
+		return res, fmt.Errorf("failed to get Ingress: %w", err)
+	}
+
+	// hostname is the name of the VIPService that will be created for this Ingress as well as the first label in
+	// the MagicDNS name of the Ingress.
+	hostname := hostnameForIngress(ing)
+	logger = logger.With("hostname", hostname)
+
+	if !ing.DeletionTimestamp.IsZero() || !a.shouldExpose(ing) {
+		return res, a.maybeCleanup(ctx, hostname, ing, logger)
+	}
+
+	if err := a.maybeProvision(ctx, hostname, ing, logger); err != nil {
+		return res, fmt.Errorf("failed to provision: %w", err)
+	}
+	return res, nil
+}
+
+// maybeProvision ensures that the VIPService and serve config for the Ingress are created or updated.
+func (a *IngressPGReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
+	if err := validateIngressClass(ctx, a.Client); err != nil {
+		logger.Infof("error validating tailscale IngressClass: %v.", err)
+		return nil
+	}
+
+	// Get and validate ProxyGroup readiness
+	pgName := ing.Annotations[AnnotationProxyGroup]
+	if pgName == "" {
+		logger.Infof("[unexpected] no ProxyGroup annotation, skipping VIPService provisioning")
+		return nil
+	}
+	pg := &tsapi.ProxyGroup{}
+	if err := a.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.Infof("ProxyGroup %q does not exist", pgName)
+			return nil
+		}
+		return fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
+	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		// TODO(irbekrm): we need to reconcile ProxyGroup Ingresses on ProxyGroup changes to not miss the status update
+		// in this case.
+		logger.Infof("ProxyGroup %q is not ready", pgName)
+		return nil
+	}
+
+	// Validate Ingress configuration
+	if err := a.validateIngress(ing, pg); err != nil {
+		logger.Infof("invalid Ingress configuration: %v", err)
+		a.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error())
+		return nil
+	}
+
+	if !IsHTTPSEnabledOnTailnet(a.tsnetServer) {
+		a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
+	}
+
+	logger = logger.With("proxy-group", pg)
+
+	if !slices.Contains(ing.Finalizers, FinalizerNamePG) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to tell the operator that the high level,
+		// multi-reconcile operation is underway.
+		logger.Infof("exposing Ingress over tailscale")
+		ing.Finalizers = append(ing.Finalizers, FinalizerNamePG)
+		if err := a.Update(ctx, ing); err != nil {
+			return fmt.Errorf("failed to add finalizer: %w", err)
+		}
+		a.mu.Lock()
+		a.managedIngresses.Add(ing.UID)
+		gaugePGIngressResources.Set(int64(a.managedIngresses.Len()))
+		a.mu.Unlock()
+	}
+
+	// 1. Ensure that if Ingress' hostname has changed, any VIPService resources corresponding to the old hostname
+	// are cleaned up.
+	// In practice, this function will ensure that any VIPServices that are associated with the provided ProxyGroup
+	// and no longer owned by an Ingress are cleaned up. This is fine- it is not expensive and ensures that in edge
+	// cases (a single update changed both hostname and removed ProxyGroup annotation) the VIPService is more likely
+	// to be (eventually) removed.
+	if err := a.maybeCleanupProxyGroup(ctx, pgName, logger); err != nil {
+		return fmt.Errorf("failed to cleanup VIPService resources for ProxyGroup: %w", err)
+	}
+
+	// 2. Ensure that there isn't a VIPService with the same hostname already created and not owned by this Ingress.
+	// TODO(irbekrm): perhaps in future we could have record names being stored on VIPServices. I am not certain if
+	// there might not be edge cases (custom domains, etc?) where attempting to determine the DNS name of the
+	// VIPService in this way won't be incorrect.
+	tcd, err := a.tailnetCertDomain(ctx)
+	if err != nil {
+		return fmt.Errorf("error determining DNS name base: %w", err)
+	}
+	dnsName := hostname + "." + tcd
+	existingVIPSvc, err := a.tsClient.getVIPServiceByName(ctx, hostname)
+	// TODO(irbekrm): here and when creating the VIPService, verify if the error is not terminal (and therefore
+	// should not be reconciled). For example, if the hostname is already a hostname of a Tailscale node, the GET
+	// here will fail.
+	if err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound {
+			return fmt.Errorf("error getting VIPService %q: %w", hostname, err)
+		}
+	}
+	if existingVIPSvc != nil && !isVIPServiceForIngress(existingVIPSvc, ing) {
+		logger.Infof("VIPService %q for MagicDNS name %q  already exists, but is not owned by this Ingress. Please delete it manually and recreate this Ingress to proceed or create an Ingress for a different MagicDNS name", hostname, dnsName)
+		a.recorder.Event(ing, corev1.EventTypeWarning, "ConflictingVIPServiceExists", fmt.Sprintf("VIPService %q for MagicDNS name %q already exists, but is not owned by this Ingress. Please delete it manually to proceed or create an Ingress for a different MagicDNS name", hostname, dnsName))
+		return nil
+	}
+
+	// 3. Ensure that the serve config for the ProxyGroup contains the VIPService
+	cm, cfg, err := a.proxyGroupServeConfig(ctx, pgName)
+	if err != nil {
+		return fmt.Errorf("error getting ingress serve config: %w", err)
+	}
+	if cm == nil {
+		logger.Infof("no ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.")
+		return nil
+	}
+	ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName))
+	handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, dnsName, logger)
+	if err != nil {
+		return fmt.Errorf("failed to get handlers for ingress: %w", err)
+	}
+	ingCfg := &ipn.ServiceConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{
+			443: {
+				HTTPS: true,
+			},
+		},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{
+			ep: {
+				Handlers: handlers,
+			},
+		},
+	}
+	serviceName := tailcfg.ServiceName("svc:" + hostname)
+	var gotCfg *ipn.ServiceConfig
+	if cfg != nil && cfg.Services != nil {
+		gotCfg = cfg.Services[serviceName]
+	}
+	if !reflect.DeepEqual(gotCfg, ingCfg) {
+		logger.Infof("Updating serve config")
+		mak.Set(&cfg.Services, serviceName, ingCfg)
+		cfgBytes, err := json.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("error marshaling serve config: %w", err)
+		}
+		mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
+		if err := a.Update(ctx, cm); err != nil {
+			return fmt.Errorf("error updating serve config: %w", err)
+		}
+	}
+
+	// 4. Ensure that the VIPService exists and is up to date.
+	tags := a.defaultTags
+	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
+		tags = strings.Split(tstr, ",")
+	}
+
+	vipSvc := &VIPService{
+		Name:    hostname,
+		Tags:    tags,
+		Ports:   []string{"443"}, // always 443 for Ingress
+		Comment: fmt.Sprintf(VIPSvcOwnerRef, ing.UID),
+	}
+	if existingVIPSvc != nil {
+		vipSvc.Addrs = existingVIPSvc.Addrs
+	}
+	if existingVIPSvc == nil || !reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) {
+		logger.Infof("Ensuring VIPService %q exists and is up to date", hostname)
+		if err := a.tsClient.createOrUpdateVIPServiceByName(ctx, vipSvc); err != nil {
+			logger.Infof("error creating VIPService: %v", err)
+			return fmt.Errorf("error creating VIPService: %w", err)
+		}
+	}
+
+	// 5. Update Ingress status
+	oldStatus := ing.Status.DeepCopy()
+	// TODO(irbekrm): once we have ingress ProxyGroup, we can determine if instances are ready to route traffic to the VIPService
+	ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
+		{
+			Hostname: dnsName,
+			Ports: []networkingv1.IngressPortStatus{
+				{
+					Protocol: "TCP",
+					Port:     443,
+				},
+			},
+		},
+	}
+	if apiequality.Semantic.DeepEqual(oldStatus, ing.Status) {
+		return nil
+	}
+	if err := a.Status().Update(ctx, ing); err != nil {
+		return fmt.Errorf("failed to update Ingress status: %w", err)
+	}
+	return nil
+}
+
+// maybeCleanupProxyGroup ensures that if an Ingress hostname has changed, any VIPService resources created for the
+// Ingress' ProxyGroup corresponding to the old hostname are cleaned up. A run of this function will ensure that any
+// VIPServices that are associated with the provided ProxyGroup and no longer owned by an Ingress are cleaned up.
+func (a *IngressPGReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) error {
+	// Get serve config for the ProxyGroup
+	cm, cfg, err := a.proxyGroupServeConfig(ctx, proxyGroupName)
+	if err != nil {
+		return fmt.Errorf("getting serve config: %w", err)
+	}
+	if cfg == nil {
+		return nil // ProxyGroup does not have any VIPServices
+	}
+
+	ingList := &networkingv1.IngressList{}
+	if err := a.List(ctx, ingList); err != nil {
+		return fmt.Errorf("listing Ingresses: %w", err)
+	}
+	serveConfigChanged := false
+	// For each VIPService in serve config...
+	for vipHostname := range cfg.Services {
+		// ...check if there is currently an Ingress with this hostname
+		found := false
+		for _, i := range ingList.Items {
+			ingressHostname := hostnameForIngress(&i)
+			if ingressHostname == vipHostname.WithoutPrefix() {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipHostname)
+			svc, err := a.getVIPService(ctx, vipHostname.WithoutPrefix(), logger)
+			if err != nil {
+				errResp := &tailscale.ErrResponse{}
+				if errors.As(err, &errResp) && errResp.Status == http.StatusNotFound {
+					delete(cfg.Services, vipHostname)
+					serveConfigChanged = true
+					continue
+				}
+				return err
+			}
+			if isVIPServiceForAnyIngress(svc) {
+				logger.Infof("cleaning up orphaned VIPService %q", vipHostname)
+				if err := a.tsClient.deleteVIPServiceByName(ctx, vipHostname.WithoutPrefix()); err != nil {
+					errResp := &tailscale.ErrResponse{}
+					if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
+						return fmt.Errorf("deleting VIPService %q: %w", vipHostname, err)
+					}
+				}
+			}
+			delete(cfg.Services, vipHostname)
+			serveConfigChanged = true
+		}
+	}
+
+	if serveConfigChanged {
+		cfgBytes, err := json.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("marshaling serve config: %w", err)
+		}
+		mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
+		if err := a.Update(ctx, cm); err != nil {
+			return fmt.Errorf("updating serve config: %w", err)
+		}
+	}
+	return nil
+}
+
+// maybeCleanup ensures that any resources, such as a VIPService created for this Ingress, are cleaned up when the
+// Ingress is being deleted or is unexposed.
+func (a *IngressPGReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
+	logger.Debugf("Ensuring any resources for Ingress are cleaned up")
+	ix := slices.Index(ing.Finalizers, FinalizerNamePG)
+	if ix < 0 {
+		logger.Debugf("no finalizer, nothing to do")
+		a.mu.Lock()
+		defer a.mu.Unlock()
+		a.managedIngresses.Remove(ing.UID)
+		gaugePGIngressResources.Set(int64(a.managedIngresses.Len()))
+		return nil
+	}
+
+	// 1. Check if there is a VIPService created for this Ingress.
+	pg := ing.Annotations[AnnotationProxyGroup]
+	cm, cfg, err := a.proxyGroupServeConfig(ctx, pg)
+	if err != nil {
+		return fmt.Errorf("error getting ProxyGroup serve config: %w", err)
+	}
+	serviceName := tailcfg.ServiceName("svc:" + hostname)
+	// VIPService is always first added to serve config and only then created in the Tailscale API, so if it is not
+	// found in the serve config, we can assume that there is no VIPService. TODO(irbekrm): once we have ingress
+	// ProxyGroup, we will probably add currently exposed VIPServices to its status. At that point, we can use the
+	// status rather than checking the serve config each time.
+	if cfg == nil || cfg.Services == nil || cfg.Services[serviceName] == nil {
+		return nil
+	}
+	logger.Infof("Ensuring that VIPService %q configuration is cleaned up", hostname)
+
+	// 2. Delete the VIPService.
+	if err := a.deleteVIPServiceIfExists(ctx, hostname, ing, logger); err != nil {
+		return fmt.Errorf("error deleting VIPService: %w", err)
+	}
+
+	// 3. Remove the VIPService from the serve config for the ProxyGroup.
+	logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
+	delete(cfg.Services, serviceName)
+	cfgBytes, err := json.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("error marshaling serve config: %w", err)
+	}
+	mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
+	if err := a.Update(ctx, cm); err != nil {
+		return fmt.Errorf("error updating ConfigMap %q: %w", cm.Name, err)
+	}
+
+	if err := a.deleteFinalizer(ctx, ing, logger); err != nil {
+		return fmt.Errorf("failed to remove finalizer: %w", err)
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.managedIngresses.Remove(ing.UID)
+	gaugePGIngressResources.Set(int64(a.managedIngresses.Len()))
+	return nil
+}
+
+func (a *IngressPGReconciler) deleteFinalizer(ctx context.Context, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
+	found := false
+	ing.Finalizers = slices.DeleteFunc(ing.Finalizers, func(f string) bool {
+		found = true
+		return f == FinalizerNamePG
+	})
+	if !found {
+		return nil
+	}
+	logger.Debug("ensure %q finalizer is removed", FinalizerNamePG)
+
+	if err := a.Update(ctx, ing); err != nil {
+		return fmt.Errorf("failed to remove finalizer %q: %w", FinalizerNamePG, err)
+	}
+	return nil
+}
+
+func pgIngressCMName(pg string) string {
+	return fmt.Sprintf("%s-ingress-config", pg)
+}
+
+func (a *IngressPGReconciler) proxyGroupServeConfig(ctx context.Context, pg string) (cm *corev1.ConfigMap, cfg *ipn.ServeConfig, err error) {
+	name := pgIngressCMName(pg)
+	cm = &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: a.tsNamespace,
+		},
+	}
+	if err := a.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil && !apierrors.IsNotFound(err) {
+		return nil, nil, fmt.Errorf("error retrieving ingress serve config ConfigMap %s: %v", name, err)
+	}
+	if apierrors.IsNotFound(err) {
+		return nil, nil, nil
+	}
+	cfg = &ipn.ServeConfig{}
+	if len(cm.BinaryData[serveConfigKey]) != 0 {
+		if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
+			return nil, nil, fmt.Errorf("error unmarshaling ingress serve config %v: %w", cm.BinaryData[serveConfigKey], err)
+		}
+	}
+	return cm, cfg, nil
+}
+
+type localClient interface {
+	StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error)
+}
+
+// tailnetCertDomain returns the base domain (TCD) of the current tailnet.
+func (a *IngressPGReconciler) tailnetCertDomain(ctx context.Context) (string, error) {
+	st, err := a.lc.StatusWithoutPeers(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error getting tailscale status: %w", err)
+	}
+	return st.CurrentTailnet.MagicDNSSuffix, nil
+}
+
+// shouldExpose returns true if the Ingress should be exposed over Tailscale in HA mode (on a ProxyGroup)
+func (a *IngressPGReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
+	isTSIngress := ing != nil &&
+		ing.Spec.IngressClassName != nil &&
+		*ing.Spec.IngressClassName == tailscaleIngressClassName
+	pgAnnot := ing.Annotations[AnnotationProxyGroup]
+	return isTSIngress && pgAnnot != ""
+}
+
+func (a *IngressPGReconciler) getVIPService(ctx context.Context, hostname string, logger *zap.SugaredLogger) (*VIPService, error) {
+	svc, err := a.tsClient.getVIPServiceByName(ctx, hostname)
+	if err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound {
+			logger.Infof("error getting VIPService %q: %v", hostname, err)
+			return nil, fmt.Errorf("error getting VIPService %q: %w", hostname, err)
+		}
+	}
+	return svc, nil
+}
+
+func isVIPServiceForIngress(svc *VIPService, ing *networkingv1.Ingress) bool {
+	if svc == nil || ing == nil {
+		return false
+	}
+	return strings.EqualFold(svc.Comment, fmt.Sprintf(VIPSvcOwnerRef, ing.UID))
+}
+
+func isVIPServiceForAnyIngress(svc *VIPService) bool {
+	if svc == nil {
+		return false
+	}
+	return strings.HasPrefix(svc.Comment, "tailscale.com/k8s-operator:owned-by:")
+}
+
+// validateIngress validates that the Ingress is properly configured.
+// Currently validates:
+// - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags
+// - The derived hostname is a valid DNS label
+// - The referenced ProxyGroup exists and is of type 'ingress'
+// - Ingress' TLS block is invalid
+func (a *IngressPGReconciler) validateIngress(ing *networkingv1.Ingress, pg *tsapi.ProxyGroup) error {
+	var errs []error
+
+	// Validate tags if present
+	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
+		tags := strings.Split(tstr, ",")
+		for _, tag := range tags {
+			tag = strings.TrimSpace(tag)
+			if err := tailcfg.CheckTag(tag); err != nil {
+				errs = append(errs, fmt.Errorf("tailscale.com/tags annotation contains invalid tag %q: %w", tag, err))
+			}
+		}
+	}
+
+	// Validate TLS configuration
+	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) {
+		errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS))
+	}
+
+	// Validate that the hostname will be a valid DNS label
+	hostname := hostnameForIngress(ing)
+	if err := dnsname.ValidLabel(hostname); err != nil {
+		errs = append(errs, fmt.Errorf("invalid hostname %q: %w. Ensure that the hostname is a valid DNS label", hostname, err))
+	}
+
+	// Validate ProxyGroup type
+	if pg.Spec.Type != tsapi.ProxyGroupTypeIngress {
+		errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q",
+			pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress))
+	}
+
+	// Validate ProxyGroup readiness
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name))
+	}
+
+	return errors.Join(errs...)
+}
+
+// deleteVIPServiceIfExists attempts to delete the VIPService if it exists and is owned by the given Ingress.
+func (a *IngressPGReconciler) deleteVIPServiceIfExists(ctx context.Context, name string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
+	svc, err := a.getVIPService(ctx, name, logger)
+	if err != nil {
+		return fmt.Errorf("error getting VIPService: %w", err)
+	}
+
+	// isVIPServiceForIngress handles nil svc, so we don't need to check it here
+	if !isVIPServiceForIngress(svc, ing) {
+		return nil
+	}
+
+	logger.Infof("Deleting VIPService %q", name)
+	if err = a.tsClient.deleteVIPServiceByName(ctx, name); err != nil {
+		return fmt.Errorf("error deleting VIPService: %w", err)
+	}
+	return nil
+}