tstest/integration: add integration test for capmap packet filters

This adds a new test that checks that connections are allowed after a
cap is added to the node, and no longer allowed after a cap has been
removed.

Updates #12542

Signed-off-by: Anton Tolchanov <anton@tailscale.com>

.github/workflows/checklocks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
+      uses: github/codeql-action/init@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
+      uses: github/codeql-action/autobuild@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
+      uses: github/codeql-action/analyze@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,18 +23,17 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: go.mod
           cache: false
 
       - name: golangci-lint
-        # Note: this is the 'v6.1.0' tag as of 2024-08-21
-        uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86
+        uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0
         with:
-          version: v1.60
+          version: v1.64
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -24,13 +24,13 @@ jobs:
 
       - name: Post to slack
         if: failure() && github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
-          channel-id: 'C05PXRM304B'
+          method: chat.postMessage
+          token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
           payload: |
             {
+              "channel": "C08FGKZCQTW",
               "blocks": [
                 {
                   "type": "section",

.github/workflows/installer.yml

@@ -6,11 +6,13 @@ on:
       - "main"
     paths:
       - scripts/installer.sh
+      - .github/workflows/installer.yml
   pull_request:
     branches:
       - "*"
     paths:
       - scripts/installer.sh
+      - .github/workflows/installer.yml
 
 jobs:
   test:
@@ -29,13 +31,11 @@ jobs:
           - "debian:stable-slim"
           - "debian:testing-slim"
           - "debian:sid-slim"
-          - "ubuntu:18.04"
           - "ubuntu:20.04"
           - "ubuntu:22.04"
-          - "ubuntu:23.04"
+          - "ubuntu:24.04"
           - "elementary/docker:stable"
           - "elementary/docker:unstable"
-          - "parrotsec/core:lts-amd64"
           - "parrotsec/core:latest"
           - "kalilinux/kali-rolling"
           - "kalilinux/kali-dev"
@@ -48,7 +48,7 @@ jobs:
           - "opensuse/leap:latest"
           - "opensuse/tumbleweed:latest"
           - "archlinux:latest"
-          - "alpine:3.14"
+          - "alpine:3.21"
           - "alpine:latest"
           - "alpine:edge"
         deps:
@@ -58,10 +58,6 @@ jobs:
           # Check a few images with wget rather than curl.
           - { image: "debian:oldstable-slim", deps: "wget" }
           - { image: "debian:sid-slim", deps: "wget" }
-          - { image: "ubuntu:23.04", deps: "wget" }
-          # Ubuntu 16.04 also needs apt-transport-https installed.
-          - { image: "ubuntu:16.04", deps: "curl apt-transport-https" }
-          - { image: "ubuntu:16.04", deps: "wget apt-transport-https" }
     runs-on: ubuntu-latest
     container:
       image: ${{ matrix.image }}
@@ -95,10 +91,7 @@ jobs:
         || contains(matrix.image, 'parrotsec')
         || contains(matrix.image, 'kalilinux')
     - name: checkout
-      # We cannot use v4, as it requires a newer glibc version than some of the
-      # tested images provide. See
-      # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/natlab-integrationtest.yml

@@ -0,0 +1,27 @@
+# Run some natlab integration tests.
+# See https://github.com/tailscale/tailscale/issues/13038
+name: "natlab-integrationtest"
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - "tstest/integration/nat/nat_test.go"
+jobs:
+  natlab-integrationtest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Install qemu
+        run: |
+          sudo rm /var/lib/man-db/auto-update
+          sudo apt-get -y update
+          sudo apt-get -y remove man-db
+          sudo apt-get install -y qemu-system-x86 qemu-utils
+      - name: Run natlab integration tests
+        run: |
+          ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat  --run-vm-tests

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -64,7 +64,6 @@ jobs:
       matrix:
         include:
           - goarch: amd64
-            coverflags: "-coverprofile=/tmp/coverage.out"
           - goarch: amd64
             buildflags: "-race"
             shard: '1/3'
@@ -78,9 +77,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -119,15 +118,10 @@ jobs:
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: test all
-      run: NOBASHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ${{matrix.coverflags}} ./... ${{matrix.buildflags}}
+      run: NOBASHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}}
       env:
         GOARCH: ${{ matrix.goarch }}
         TS_TEST_SHARD: ${{ matrix.shard }}
-    - name: Publish to coveralls.io
-      if: matrix.coverflags != '' # only publish results if we've tracked coverage
-      uses: shogo82148/actions-goveralls@v1
-      with:
-        path-to-profile: /tmp/coverage.out
     - name: bench all
       run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done)
       env:
@@ -145,21 +139,25 @@ jobs:
           echo "Build/test created untracked files in the repo (file names above)."
           exit 1
         fi
-
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
   windows:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Install Go
-      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -182,6 +180,11 @@ jobs:
       # Somewhere in the layers (powershell?)
       # the equals signs cause great confusion.
       run: go test ./... -bench . -benchtime 1x -run "^$"
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   privileged:
     runs-on: ubuntu-22.04
@@ -190,7 +193,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +205,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +217,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +261,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -289,13 +292,18 @@ jobs:
         GOOS: ${{ matrix.goos }}
         GOARCH: ${{ matrix.goarch }}
         CGO_ENABLED: "0"
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   ios: # similar to cross above, but iOS can't build most of the repo. So, just
        #make it build a few smoke packages.
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -313,13 +321,19 @@ jobs:
           # AIX
           - goos: aix
             goarch: ppc64
+          # Solaris
+          - goos: solaris
+            goarch: amd64
+          # illumos
+          - goos: illumos
+            goarch: amd64
 
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -342,6 +356,11 @@ jobs:
         GOARCH: ${{ matrix.goarch }}
         GOARM: ${{ matrix.goarm }}
         CGO_ENABLED: "0"
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   android:
     # similar to cross above, but android fails to build a few pieces of the
@@ -350,7 +369,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -365,9 +384,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -394,12 +413,17 @@ jobs:
       run: |
         ./tool/go run ./cmd/tsconnect --fast-compression build
         ./tool/go run ./cmd/tsconnect --fast-compression build-pkg
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   tailscale_go: # Subset of tests that depend on our custom Go toolchain.
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -461,7 +485,7 @@ jobs:
       run: |
         echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
@@ -471,17 +495,17 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
-        find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check
+        find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check --internal
 
   go_generate:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -494,7 +518,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -506,7 +530,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -522,7 +546,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck
@@ -563,8 +587,10 @@ jobs:
       # By having the job always run, but skipping its only step as needed, we
       # let the CI output collapse nicely in PRs.
       if: failure() && github.event_name == 'push'
-      uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+      uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
       with:
+        webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+        webhook-type: incoming-webhook
         payload: |
           {
             "attachments": [{
@@ -576,9 +602,6 @@ jobs:
               "color": "danger"
             }]
           }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   check_mergeability:
     if: always()

.github/workflows/update-flake.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Run update-flakes
         run: ./update-flake.sh
@@ -36,7 +36,7 @@ jobs:
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Run go get
         run: |
@@ -35,7 +35,7 @@ jobs:
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

.golangci.yml

@@ -26,16 +26,11 @@ issues:
 
 # Per-linter settings are contained in this top-level key
 linters-settings:
-  # Enable all rules by default; we don't use invisible unicode runes.
-  bidichk:
-
   gofmt:
     rewrite-rules:
       - pattern: 'interface{}'
         replacement: 'any'
 
-  goimports:
-
   govet:
     # Matches what we use in corp as of 2023-12-07
     enable:
@@ -78,8 +73,6 @@ linters-settings:
           # analyzer doesn't support type declarations
           #- github.com/tailscale/tailscale/types/logger.Logf
 
-  misspell:
-
   revive:
     enable-all-rules: false
     ignore-generated-header: true

Dockerfile

@@ -27,7 +27,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.23-alpine AS build-env
+FROM golang:1.24-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 

Makefile

@@ -17,7 +17,7 @@ lint: ## Run golangci-lint
 updatedeps: ## Update depaware deps
 	# depaware (via x/tools/go/packages) shells back to "go", so make sure the "go"
 	# it finds in its $$PATH is the right one.
-	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update \
+	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --internal \
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \
@@ -27,7 +27,7 @@ updatedeps: ## Update depaware deps
 depaware: ## Run depaware checks
 	# depaware (via x/tools/go/packages) shells back to "go", so make sure the "go"
 	# it finds in its $$PATH is the right one.
-	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check \
+	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --internal \
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \

README.md

@@ -72,7 +72,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
 `Signed-off-by` lines in commits.
 
 See `git log` for our commit message style. It's basically the same as
-[Go's style](https://github.com/golang/go/wiki/CommitMessage).
+[Go's style](https://go.dev/wiki/CommitMessage).
 
 ## About Us
 

VERSION.txt

@@ -1 +1 @@
-1.79.0
+1.81.0

appc/appconnector.go

@@ -18,7 +18,6 @@ import (
 	"sync"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
@@ -290,12 +289,14 @@ func (e *AppConnector) updateDomains(domains []string) {
 				toRemove = append(toRemove, netip.PrefixFrom(a, a.BitLen()))
 			}
 		}
-		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
-		}
+		e.queue.Add(func() {
+			if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+				e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err)
+			}
+		})
 	}
 
-	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
+	e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards)
 }
 
 // updateRoutes merges the supplied routes into the currently configured routes. The routes supplied
@@ -311,11 +312,6 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 		return
 	}
 
-	if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
-		e.logf("failed to advertise routes: %v: %v", routes, err)
-		return
-	}
-
 	var toRemove []netip.Prefix
 
 	// If we're storing routes and know e.controlRoutes is a good
@@ -339,9 +335,14 @@ nextRoute:
 		}
 	}
 
-	if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-		e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
-	}
+	e.queue.Add(func() {
+		if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
+			e.logf("failed to advertise routes: %v: %v", routes, err)
+		}
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
+		}
+	})
 
 	e.controlRoutes = routes
 	if err := e.storeRoutesLocked(); err != nil {
@@ -354,7 +355,7 @@ func (e *AppConnector) Domains() views.Slice[string] {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	return views.SliceOf(xmaps.Keys(e.domains))
+	return views.SliceOf(slicesx.MapKeys(e.domains))
 }
 
 // DomainRoutes returns a map of domains to resolved IP
@@ -375,13 +376,13 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr {
 // response is being returned over the PeerAPI. The response is parsed and
 // matched against the configured domains, if matched the routeAdvertiser is
 // advised to advertise the discovered route.
-func (e *AppConnector) ObserveDNSResponse(res []byte) {
+func (e *AppConnector) ObserveDNSResponse(res []byte) error {
 	var p dnsmessage.Parser
 	if _, err := p.Start(res); err != nil {
-		return
+		return err
 	}
 	if err := p.SkipAllQuestions(); err != nil {
-		return
+		return err
 	}
 
 	// cnameChain tracks a chain of CNAMEs for a given query in order to reverse
@@ -400,12 +401,12 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			break
 		}
 		if err != nil {
-			return
+			return err
 		}
 
 		if h.Class != dnsmessage.ClassINET {
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 		}
@@ -414,7 +415,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA:
 		default:
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 
@@ -428,7 +429,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		if h.Type == dnsmessage.TypeCNAME {
 			res, err := p.CNAMEResource()
 			if err != nil {
-				return
+				return err
 			}
 			cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".")
 			if len(cname) == 0 {
@@ -442,20 +443,20 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		case dnsmessage.TypeA:
 			r, err := p.AResource()
 			if err != nil {
-				return
+				return err
 			}
 			addr := netip.AddrFrom4(r.A)
 			mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
 		case dnsmessage.TypeAAAA:
 			r, err := p.AAAAResource()
 			if err != nil {
-				return
+				return err
 			}
 			addr := netip.AddrFrom16(r.AAAA)
 			mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
 		default:
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 		}
@@ -486,6 +487,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			e.scheduleAdvertisement(domain, toAdvertise...)
 		}
 	}
+	return nil
 }
 
 // starting from the given domain that resolved to an address, find it, or any

appc/appconnector_test.go

@@ -8,16 +8,17 @@ import (
 	"net/netip"
 	"reflect"
 	"slices"
+	"sync/atomic"
 	"testing"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
 	"tailscale.com/tstest"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
+	"tailscale.com/util/slicesx"
 )
 
 func fakeStoreRoutes(*RouteInfo) error { return nil }
@@ -50,7 +51,7 @@ func TestUpdateDomains(t *testing.T) {
 		// domains are explicitly downcased on set.
 		a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
 		a.Wait(ctx)
-		if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
+		if got, want := slicesx.MapKeys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
 	}
@@ -69,7 +70,9 @@ func TestUpdateRoutes(t *testing.T) {
 		a.updateDomains([]string{"*.example.com"})
 
 		// This route should be collapsed into the range
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 
 		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
@@ -77,11 +80,14 @@ func TestUpdateRoutes(t *testing.T) {
 		}
 
 		// This route should not be collapsed or removed
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 
 		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
 		a.updateRoutes(routes)
+		a.Wait(ctx)
 
 		slices.SortFunc(rc.Routes(), prefixCompare)
 		rc.SetRoutes(slices.Compact(rc.Routes()))
@@ -101,6 +107,7 @@ func TestUpdateRoutes(t *testing.T) {
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
+	ctx := context.Background()
 	for _, shouldStore := range []bool{false, true} {
 		rc := &appctest.RouteCollector{}
 		var a *AppConnector
@@ -113,6 +120,7 @@ func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
 		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
 		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
 		a.updateRoutes(routes)
+		a.Wait(ctx)
 
 		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
 			t.Fatalf("got %v, want %v", rc.Routes(), routes)
@@ -130,7 +138,9 @@ func TestDomainRoutes(t *testing.T) {
 			a = NewAppConnector(t.Logf, rc, nil, nil)
 		}
 		a.updateDomains([]string{"example.com"})
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(context.Background())
 
 		want := map[string][]netip.Addr{
@@ -155,7 +165,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		}
 
 		// a has no domains configured, so it should not advertise any routes
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
@@ -163,7 +175,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
 		a.updateDomains([]string{"example.com"})
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
@@ -172,7 +186,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		// a CNAME record chain should result in a route being added if the chain
 		// matches a routed domain.
 		a.updateDomains([]string{"www.example.com", "example.com"})
-		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com.")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -181,7 +197,9 @@ func TestObserveDNSResponse(t *testing.T) {
 
 		// a CNAME record chain should result in a route being added if the chain
 		// even if only found in the middle of the chain
-		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org.")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -190,14 +208,18 @@ func TestObserveDNSResponse(t *testing.T) {
 
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
 
 		// don't re-advertise routes that have already been advertised
-		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if !slices.Equal(rc.Routes(), wantRoutes) {
 			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -207,7 +229,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		pfx := netip.MustParsePrefix("192.0.2.0/24")
 		a.updateRoutes([]netip.Prefix{pfx})
 		wantRoutes = append(wantRoutes, pfx)
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if !slices.Equal(rc.Routes(), wantRoutes) {
 			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -230,7 +254,9 @@ func TestWildcardDomains(t *testing.T) {
 		}
 
 		a.updateDomains([]string{"*.example.com"})
-		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
 			t.Errorf("routes: got %v; want %v", got, want)
@@ -438,10 +464,16 @@ func TestUpdateDomainRouteRemoval(t *testing.T) {
 		// adding domains doesn't immediately cause any routes to be advertised
 		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
 
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
+		for _, res := range [][]byte{
+			dnsResponse("a.example.com.", "1.2.3.1"),
+			dnsResponse("a.example.com.", "1.2.3.2"),
+			dnsResponse("b.example.com.", "1.2.3.3"),
+			dnsResponse("b.example.com.", "1.2.3.4"),
+		} {
+			if err := a.ObserveDNSResponse(res); err != nil {
+				t.Errorf("ObserveDNSResponse: %v", err)
+			}
+		}
 		a.Wait(ctx)
 		// observing dns responses causes routes to be advertised
 		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
@@ -487,10 +519,16 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) {
 		// adding domains doesn't immediately cause any routes to be advertised
 		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
 
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
-		a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
-		a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
+		for _, res := range [][]byte{
+			dnsResponse("a.example.com.", "1.2.3.1"),
+			dnsResponse("a.example.com.", "1.2.3.2"),
+			dnsResponse("1.b.example.com.", "1.2.3.3"),
+			dnsResponse("2.b.example.com.", "1.2.3.4"),
+		} {
+			if err := a.ObserveDNSResponse(res); err != nil {
+				t.Errorf("ObserveDNSResponse: %v", err)
+			}
+		}
 		a.Wait(ctx)
 		// observing dns responses causes routes to be advertised
 		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
@@ -602,3 +640,57 @@ func TestMetricBucketsAreSorted(t *testing.T) {
 		t.Errorf("metricStoreRoutesNBuckets must be in order")
 	}
 }
+
+// TestUpdateRoutesDeadlock is a regression test for a deadlock in
+// LocalBackend<->AppConnector interaction. When using real LocalBackend as the
+// routeAdvertiser, calls to Advertise/UnadvertiseRoutes can end up calling
+// back into AppConnector via authReconfig. If everything is called
+// synchronously, this results in a deadlock on AppConnector.mu.
+func TestUpdateRoutesDeadlock(t *testing.T) {
+	ctx := context.Background()
+	rc := &appctest.RouteCollector{}
+	a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+
+	advertiseCalled := new(atomic.Bool)
+	unadvertiseCalled := new(atomic.Bool)
+	rc.AdvertiseCallback = func() {
+		// Call something that requires a.mu to be held.
+		a.DomainRoutes()
+		advertiseCalled.Store(true)
+	}
+	rc.UnadvertiseCallback = func() {
+		// Call something that requires a.mu to be held.
+		a.DomainRoutes()
+		unadvertiseCalled.Store(true)
+	}
+
+	a.updateDomains([]string{"example.com"})
+	a.Wait(ctx)
+
+	// Trigger rc.AdveriseRoute.
+	a.updateRoutes(
+		[]netip.Prefix{
+			netip.MustParsePrefix("127.0.0.1/32"),
+			netip.MustParsePrefix("127.0.0.2/32"),
+		},
+	)
+	a.Wait(ctx)
+	// Trigger rc.UnadveriseRoute.
+	a.updateRoutes(
+		[]netip.Prefix{
+			netip.MustParsePrefix("127.0.0.1/32"),
+		},
+	)
+	a.Wait(ctx)
+
+	if !advertiseCalled.Load() {
+		t.Error("AdvertiseRoute was not called")
+	}
+	if !unadvertiseCalled.Load() {
+		t.Error("UnadvertiseRoute was not called")
+	}
+
+	if want := []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}; !slices.Equal(slices.Compact(rc.Routes()), want) {
+		t.Fatalf("got %v, want %v", rc.Routes(), want)
+	}
+}

appc/appctest/appctest.go

@@ -11,12 +11,22 @@ import (
 
 // RouteCollector is a test helper that collects the list of routes advertised
 type RouteCollector struct {
+	// AdvertiseCallback (optional) is called synchronously from
+	// AdvertiseRoute.
+	AdvertiseCallback func()
+	// UnadvertiseCallback (optional) is called synchronously from
+	// UnadvertiseRoute.
+	UnadvertiseCallback func()
+
 	routes        []netip.Prefix
 	removedRoutes []netip.Prefix
 }
 
 func (rc *RouteCollector) AdvertiseRoute(pfx ...netip.Prefix) error {
 	rc.routes = append(rc.routes, pfx...)
+	if rc.AdvertiseCallback != nil {
+		rc.AdvertiseCallback()
+	}
 	return nil
 }
 
@@ -30,6 +40,9 @@ func (rc *RouteCollector) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 			rc.removedRoutes = append(rc.removedRoutes, r)
 		}
 	}
+	if rc.UnadvertiseCallback != nil {
+		rc.UnadvertiseCallback()
+	}
 	return nil
 }
 

atomicfile/atomicfile.go

@@ -15,8 +15,9 @@ import (
 )
 
 // WriteFile writes data to filename+some suffix, then renames it into filename.
-// The perm argument is ignored on Windows. If the target filename already
-// exists but is not a regular file, WriteFile returns an error.
+// The perm argument is ignored on Windows, but if the target filename already
+// exists then the target file's attributes and ACLs are preserved. If the target
+// filename already exists but is not a regular file, WriteFile returns an error.
 func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
 	fi, err := os.Stat(filename)
 	if err == nil && !fi.Mode().IsRegular() {
@@ -47,5 +48,5 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
 	if err := f.Close(); err != nil {
 		return err
 	}
-	return os.Rename(tmpName, filename)
+	return rename(tmpName, filename)
 }

atomicfile/atomicfile_notwindows.go

@@ -0,0 +1,14 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !windows
+
+package atomicfile
+
+import (
+	"os"
+)
+
+func rename(srcFile, destFile string) error {
+	return os.Rename(srcFile, destFile)
+}

atomicfile/atomicfile_windows.go

@@ -0,0 +1,33 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package atomicfile
+
+import (
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+func rename(srcFile, destFile string) error {
+	// Use replaceFile when possible to preserve the original file's attributes and ACLs.
+	if err := replaceFile(destFile, srcFile); err == nil || err != windows.ERROR_FILE_NOT_FOUND {
+		return err
+	}
+	// destFile doesn't exist. Just do a normal rename.
+	return os.Rename(srcFile, destFile)
+}
+
+func replaceFile(destFile, srcFile string) error {
+	destFile16, err := windows.UTF16PtrFromString(destFile)
+	if err != nil {
+		return err
+	}
+
+	srcFile16, err := windows.UTF16PtrFromString(srcFile)
+	if err != nil {
+		return err
+	}
+
+	return replaceFileW(destFile16, srcFile16, nil, 0, nil, nil)
+}

atomicfile/atomicfile_windows_test.go

@@ -0,0 +1,146 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package atomicfile
+
+import (
+	"os"
+	"testing"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _SECURITY_RESOURCE_MANAGER_AUTHORITY = windows.SidIdentifierAuthority{[6]byte{0, 0, 0, 0, 0, 9}}
+
+// makeRandomSID generates a SID derived from a v4 GUID.
+// This is basically the same algorithm used by browser sandboxes for generating
+// random SIDs.
+func makeRandomSID() (*windows.SID, error) {
+	guid, err := windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+
+	rids := *((*[4]uint32)(unsafe.Pointer(&guid)))
+
+	var pSID *windows.SID
+	if err := windows.AllocateAndInitializeSid(&_SECURITY_RESOURCE_MANAGER_AUTHORITY, 4, rids[0], rids[1], rids[2], rids[3], 0, 0, 0, 0, &pSID); err != nil {
+		return nil, err
+	}
+	defer windows.FreeSid(pSID)
+
+	// Make a copy that lives on the Go heap
+	return pSID.Copy()
+}
+
+func getExistingFileSD(name string) (*windows.SECURITY_DESCRIPTOR, error) {
+	const infoFlags = windows.DACL_SECURITY_INFORMATION
+	return windows.GetNamedSecurityInfo(name, windows.SE_FILE_OBJECT, infoFlags)
+}
+
+func getExistingFileDACL(name string) (*windows.ACL, error) {
+	sd, err := getExistingFileSD(name)
+	if err != nil {
+		return nil, err
+	}
+
+	dacl, _, err := sd.DACL()
+	return dacl, err
+}
+
+func addDenyACEForRandomSID(dacl *windows.ACL) (*windows.ACL, error) {
+	randomSID, err := makeRandomSID()
+	if err != nil {
+		return nil, err
+	}
+
+	randomSIDTrustee := windows.TRUSTEE{nil, windows.NO_MULTIPLE_TRUSTEE,
+		windows.TRUSTEE_IS_SID, windows.TRUSTEE_IS_UNKNOWN,
+		windows.TrusteeValueFromSID(randomSID)}
+
+	entries := []windows.EXPLICIT_ACCESS{
+		{
+			windows.GENERIC_ALL,
+			windows.DENY_ACCESS,
+			windows.NO_INHERITANCE,
+			randomSIDTrustee,
+		},
+	}
+
+	return windows.ACLFromEntries(entries, dacl)
+}
+
+func setExistingFileDACL(name string, dacl *windows.ACL) error {
+	return windows.SetNamedSecurityInfo(name, windows.SE_FILE_OBJECT,
+		windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)
+}
+
+// makeOrigFileWithCustomDACL creates a new, temporary file with a custom
+// DACL that we can check for later. It returns the name of the temporary
+// file and the security descriptor for the file in SDDL format.
+func makeOrigFileWithCustomDACL() (name, sddl string, err error) {
+	f, err := os.CreateTemp("", "foo*.tmp")
+	if err != nil {
+		return "", "", err
+	}
+	name = f.Name()
+	if err := f.Close(); err != nil {
+		return "", "", err
+	}
+	f = nil
+	defer func() {
+		if err != nil {
+			os.Remove(name)
+		}
+	}()
+
+	dacl, err := getExistingFileDACL(name)
+	if err != nil {
+		return "", "", err
+	}
+
+	// Add a harmless, deny-only ACE for a random SID that isn't used for anything
+	// (but that we can check for later).
+	dacl, err = addDenyACEForRandomSID(dacl)
+	if err != nil {
+		return "", "", err
+	}
+
+	if err := setExistingFileDACL(name, dacl); err != nil {
+		return "", "", err
+	}
+
+	sd, err := getExistingFileSD(name)
+	if err != nil {
+		return "", "", err
+	}
+
+	return name, sd.String(), nil
+}
+
+func TestPreserveSecurityInfo(t *testing.T) {
+	// Make a test file with a custom ACL.
+	origFileName, want, err := makeOrigFileWithCustomDACL()
+	if err != nil {
+		t.Fatalf("makeOrigFileWithCustomDACL returned %v", err)
+	}
+	t.Cleanup(func() {
+		os.Remove(origFileName)
+	})
+
+	if err := WriteFile(origFileName, []byte{}, 0); err != nil {
+		t.Fatalf("WriteFile returned %v", err)
+	}
+
+	// We expect origFileName's security descriptor to be unchanged despite
+	// the WriteFile call.
+	sd, err := getExistingFileSD(origFileName)
+	if err != nil {
+		t.Fatalf("getExistingFileSD(%q) returned %v", origFileName, err)
+	}
+
+	if got := sd.String(); got != want {
+		t.Errorf("security descriptor comparison failed: got %q, want %q", got, want)
+	}
+}

atomicfile/mksyscall.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package atomicfile
+
+//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go mksyscall.go
+
+//sys replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) [int32(failretval)==0] = kernel32.ReplaceFileW

atomicfile/zsyscall_windows.go

@@ -0,0 +1,52 @@
+// Code generated by 'go generate'; DO NOT EDIT.
+
+package atomicfile
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return errERROR_EINVAL
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
+
+	procReplaceFileW = modkernel32.NewProc("ReplaceFileW")
+)
+
+func replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) {
+	r1, _, e1 := syscall.Syscall6(procReplaceFileW.Addr(), 6, uintptr(unsafe.Pointer(replaced)), uintptr(unsafe.Pointer(replacement)), uintptr(unsafe.Pointer(backup)), uintptr(flags), uintptr(exclude), uintptr(reserved))
+	if int32(r1) == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}

build_dist.sh

@@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do
 	--extra-small)
 		shift
 		ldflags="$ldflags -w -s"
-		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion"
+		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan,ts_omit_capture"
 		;;
 	--box)
 		shift

client/local/local.go

@@ -1,15 +1,17 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build go1.19
+//go:build go1.22
 
-package tailscale
+// Package local contains a Go client for the Tailscale LocalAPI.
+package local
 
 import (
 	"bytes"
 	"cmp"
 	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -43,11 +45,11 @@ import (
 	"tailscale.com/util/syspolicy/setting"
 )
 
-// defaultLocalClient is the default LocalClient when using the legacy
+// defaultClient is the default Client when using the legacy
 // package-level functions.
-var defaultLocalClient LocalClient
+var defaultClient Client
 
-// LocalClient is a client to Tailscale's "LocalAPI", communicating with the
+// Client is a client to Tailscale's "LocalAPI", communicating with the
 // Tailscale daemon on the local machine. Its API is not necessarily stable and
 // subject to changes between releases. Some API calls have stricter
 // compatibility guarantees, once they've been widely adopted. See method docs
@@ -57,11 +59,17 @@ var defaultLocalClient LocalClient
 //
 // Any exported fields should be set before using methods on the type
 // and not changed thereafter.
-type LocalClient struct {
+type Client struct {
 	// Dial optionally specifies an alternate func that connects to the local
 	// machine's tailscaled or equivalent. If nil, a default is used.
 	Dial func(ctx context.Context, network, addr string) (net.Conn, error)
 
+	// Transport optionally specifies an alternate [http.RoundTripper]
+	// used to execute HTTP requests. If nil, a default [http.Transport] is used,
+	// potentially with custom dialing logic from [Dial].
+	// It is primarily used for testing.
+	Transport http.RoundTripper
+
 	// Socket specifies an alternate path to the local Tailscale socket.
 	// If empty, a platform-specific default is used.
 	Socket string
@@ -85,21 +93,21 @@ type LocalClient struct {
 	tsClientOnce sync.Once
 }
 
-func (lc *LocalClient) socket() string {
+func (lc *Client) socket() string {
 	if lc.Socket != "" {
 		return lc.Socket
 	}
 	return paths.DefaultTailscaledSocket()
 }
 
-func (lc *LocalClient) dialer() func(ctx context.Context, network, addr string) (net.Conn, error) {
+func (lc *Client) dialer() func(ctx context.Context, network, addr string) (net.Conn, error) {
 	if lc.Dial != nil {
 		return lc.Dial
 	}
 	return lc.defaultDialer
 }
 
-func (lc *LocalClient) defaultDialer(ctx context.Context, network, addr string) (net.Conn, error) {
+func (lc *Client) defaultDialer(ctx context.Context, network, addr string) (net.Conn, error) {
 	if addr != "local-tailscaled.sock:80" {
 		return nil, fmt.Errorf("unexpected URL address %q", addr)
 	}
@@ -125,13 +133,13 @@ func (lc *LocalClient) defaultDialer(ctx context.Context, network, addr string)
 // authenticating to the local Tailscale daemon vary by platform.
 //
 // DoLocalRequest may mutate the request to add Authorization headers.
-func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error) {
+func (lc *Client) DoLocalRequest(req *http.Request) (*http.Response, error) {
 	req.Header.Set("Tailscale-Cap", strconv.Itoa(int(tailcfg.CurrentCapabilityVersion)))
 	lc.tsClientOnce.Do(func() {
 		lc.tsClient = &http.Client{
-			Transport: &http.Transport{
-				DialContext: lc.dialer(),
-			},
+			Transport: cmp.Or(lc.Transport, http.RoundTripper(
+				&http.Transport{DialContext: lc.dialer()}),
+			),
 		}
 	})
 	if !lc.OmitAuth {
@@ -142,7 +150,7 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 	return lc.tsClient.Do(req)
 }
 
-func (lc *LocalClient) doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
+func (lc *Client) doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
 	res, err := lc.DoLocalRequest(req)
 	if err == nil {
 		if server := res.Header.Get("Tailscale-Version"); server != "" && server != envknob.IPCVersion() && onVersionMismatch != nil {
@@ -231,12 +239,17 @@ func SetVersionMismatchHandler(f func(clientVer, serverVer string)) {
 	onVersionMismatch = f
 }
 
-func (lc *LocalClient) send(ctx context.Context, method, path string, wantStatus int, body io.Reader) ([]byte, error) {
-	slurp, _, err := lc.sendWithHeaders(ctx, method, path, wantStatus, body, nil)
+func (lc *Client) send(ctx context.Context, method, path string, wantStatus int, body io.Reader) ([]byte, error) {
+	var headers http.Header
+	if reason := apitype.RequestReasonKey.Value(ctx); reason != "" {
+		reasonBase64 := base64.StdEncoding.EncodeToString([]byte(reason))
+		headers = http.Header{apitype.RequestReasonHeader: {reasonBase64}}
+	}
+	slurp, _, err := lc.sendWithHeaders(ctx, method, path, wantStatus, body, headers)
 	return slurp, err
 }
 
-func (lc *LocalClient) sendWithHeaders(
+func (lc *Client) sendWithHeaders(
 	ctx context.Context,
 	method,
 	path string,
@@ -275,15 +288,15 @@ type httpStatusError struct {
 	HTTPStatus int
 }
 
-func (lc *LocalClient) get200(ctx context.Context, path string) ([]byte, error) {
+func (lc *Client) get200(ctx context.Context, path string) ([]byte, error) {
 	return lc.send(ctx, "GET", path, 200, nil)
 }
 
 // WhoIs returns the owner of the remoteAddr, which must be an IP or IP:port.
 //
-// Deprecated: use LocalClient.WhoIs.
+// Deprecated: use Client.WhoIs.
 func WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
-	return defaultLocalClient.WhoIs(ctx, remoteAddr)
+	return defaultClient.WhoIs(ctx, remoteAddr)
 }
 
 func decodeJSON[T any](b []byte) (ret T, err error) {
@@ -301,7 +314,7 @@ func decodeJSON[T any](b []byte) (ret T, err error) {
 // For connections proxied by tailscaled, this looks up the owner of the given
 // address as TCP first, falling back to UDP; if you want to only check a
 // specific address family, use WhoIsProto.
-func (lc *LocalClient) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
+func (lc *Client) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr))
 	if err != nil {
 		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
@@ -318,7 +331,7 @@ var ErrPeerNotFound = errors.New("peer not found")
 // WhoIsNodeKey returns the owner of the given wireguard public key.
 //
 // If not found, the error is ErrPeerNotFound.
-func (lc *LocalClient) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) {
+func (lc *Client) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(key.String()))
 	if err != nil {
 		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
@@ -333,7 +346,7 @@ func (lc *LocalClient) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*a
 // IP:port, for the given protocol (tcp or udp).
 //
 // If not found, the error is ErrPeerNotFound.
-func (lc *LocalClient) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) {
+func (lc *Client) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?proto="+url.QueryEscape(proto)+"&addr="+url.QueryEscape(remoteAddr))
 	if err != nil {
 		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
@@ -345,19 +358,19 @@ func (lc *LocalClient) WhoIsProto(ctx context.Context, proto, remoteAddr string)
 }
 
 // Goroutines returns a dump of the Tailscale daemon's current goroutines.
-func (lc *LocalClient) Goroutines(ctx context.Context) ([]byte, error) {
+func (lc *Client) Goroutines(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/goroutines")
 }
 
 // DaemonMetrics returns the Tailscale daemon's metrics in
 // the Prometheus text exposition format.
-func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
+func (lc *Client) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
 // UserMetrics returns the user metrics in
 // the Prometheus text exposition format.
-func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
+func (lc *Client) UserMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/usermetrics")
 }
 
@@ -366,7 +379,7 @@ func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
 // metric is created and initialized to delta.
 //
 // IncrementCounter does not support gauge metrics or negative delta values.
-func (lc *LocalClient) IncrementCounter(ctx context.Context, name string, delta int) error {
+func (lc *Client) IncrementCounter(ctx context.Context, name string, delta int) error {
 	type metricUpdate struct {
 		Name  string `json:"name"`
 		Type  string `json:"type"`
@@ -385,7 +398,7 @@ func (lc *LocalClient) IncrementCounter(ctx context.Context, name string, delta
 
 // TailDaemonLogs returns a stream the Tailscale daemon's logs as they arrive.
 // Close the context to stop the stream.
-func (lc *LocalClient) TailDaemonLogs(ctx context.Context) (io.Reader, error) {
+func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/logtap", nil)
 	if err != nil {
 		return nil, err
@@ -401,7 +414,7 @@ func (lc *LocalClient) TailDaemonLogs(ctx context.Context) (io.Reader, error) {
 }
 
 // Pprof returns a pprof profile of the Tailscale daemon.
-func (lc *LocalClient) Pprof(ctx context.Context, pprofType string, sec int) ([]byte, error) {
+func (lc *Client) Pprof(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string
 	if sec < 0 || sec > 300 {
 		return nil, errors.New("duration out of range")
@@ -434,7 +447,7 @@ type BugReportOpts struct {
 //
 // The opts type specifies options to pass to the Tailscale daemon when
 // generating this bug report.
-func (lc *LocalClient) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) {
+func (lc *Client) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) {
 	qparams := make(url.Values)
 	if opts.Note != "" {
 		qparams.Set("note", opts.Note)
@@ -479,13 +492,13 @@ func (lc *LocalClient) BugReportWithOpts(ctx context.Context, opts BugReportOpts
 //
 // This is the same as calling BugReportWithOpts and only specifying the Note
 // field.
-func (lc *LocalClient) BugReport(ctx context.Context, note string) (string, error) {
+func (lc *Client) BugReport(ctx context.Context, note string) (string, error) {
 	return lc.BugReportWithOpts(ctx, BugReportOpts{Note: note})
 }
 
 // DebugAction invokes a debug action, such as "rebind" or "restun".
 // These are development tools and subject to change or removal over time.
-func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
+func (lc *Client) DebugAction(ctx context.Context, action string) error {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil)
 	if err != nil {
 		return fmt.Errorf("error %w: %s", err, body)
@@ -496,7 +509,7 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
 // DebugActionBody invokes a debug action with a body parameter, such as
 // "debug-force-prefer-derp".
 // These are development tools and subject to change or removal over time.
-func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
+func (lc *Client) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, rbody)
 	if err != nil {
 		return fmt.Errorf("error %w: %s", err, body)
@@ -506,7 +519,7 @@ func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody
 
 // DebugResultJSON invokes a debug action and returns its result as something JSON-able.
 // These are development tools and subject to change or removal over time.
-func (lc *LocalClient) DebugResultJSON(ctx context.Context, action string) (any, error) {
+func (lc *Client) DebugResultJSON(ctx context.Context, action string) (any, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error %w: %s", err, body)
@@ -549,7 +562,7 @@ type DebugPortmapOpts struct {
 // process.
 //
 // opts can be nil; if so, default values will be used.
-func (lc *LocalClient) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) {
+func (lc *Client) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) {
 	vals := make(url.Values)
 	if opts == nil {
 		opts = &DebugPortmapOpts{}
@@ -584,7 +597,7 @@ func (lc *LocalClient) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts)
 
 // SetDevStoreKeyValue set a statestore key/value. It's only meant for development.
 // The schema (including when keys are re-read) is not a stable interface.
-func (lc *LocalClient) SetDevStoreKeyValue(ctx context.Context, key, value string) error {
+func (lc *Client) SetDevStoreKeyValue(ctx context.Context, key, value string) error {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/dev-set-state-store?"+(url.Values{
 		"key":   {key},
 		"value": {value},
@@ -598,7 +611,7 @@ func (lc *LocalClient) SetDevStoreKeyValue(ctx context.Context, key, value strin
 // SetComponentDebugLogging sets component's debug logging enabled for
 // the provided duration. If the duration is in the past, the debug logging
 // is disabled.
-func (lc *LocalClient) SetComponentDebugLogging(ctx context.Context, component string, d time.Duration) error {
+func (lc *Client) SetComponentDebugLogging(ctx context.Context, component string, d time.Duration) error {
 	body, err := lc.send(ctx, "POST",
 		fmt.Sprintf("/localapi/v0/component-debug-logging?component=%s&secs=%d",
 			url.QueryEscape(component), int64(d.Seconds())), 200, nil)
@@ -619,25 +632,25 @@ func (lc *LocalClient) SetComponentDebugLogging(ctx context.Context, component s
 
 // Status returns the Tailscale daemon's status.
 func Status(ctx context.Context) (*ipnstate.Status, error) {
-	return defaultLocalClient.Status(ctx)
+	return defaultClient.Status(ctx)
 }
 
 // Status returns the Tailscale daemon's status.
-func (lc *LocalClient) Status(ctx context.Context) (*ipnstate.Status, error) {
+func (lc *Client) Status(ctx context.Context) (*ipnstate.Status, error) {
 	return lc.status(ctx, "")
 }
 
 // StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
 func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
-	return defaultLocalClient.StatusWithoutPeers(ctx)
+	return defaultClient.StatusWithoutPeers(ctx)
 }
 
 // StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
-func (lc *LocalClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
+func (lc *Client) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 	return lc.status(ctx, "?peers=false")
 }
 
-func (lc *LocalClient) status(ctx context.Context, queryString string) (*ipnstate.Status, error) {
+func (lc *Client) status(ctx context.Context, queryString string) (*ipnstate.Status, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/status"+queryString)
 	if err != nil {
 		return nil, err
@@ -648,7 +661,7 @@ func (lc *LocalClient) status(ctx context.Context, queryString string) (*ipnstat
 // IDToken is a request to get an OIDC ID token for an audience.
 // The token can be presented to any resource provider which offers OIDC
 // Federation.
-func (lc *LocalClient) IDToken(ctx context.Context, aud string) (*tailcfg.TokenResponse, error) {
+func (lc *Client) IDToken(ctx context.Context, aud string) (*tailcfg.TokenResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/id-token?aud="+url.QueryEscape(aud))
 	if err != nil {
 		return nil, err
@@ -660,14 +673,14 @@ func (lc *LocalClient) IDToken(ctx context.Context, aud string) (*tailcfg.TokenR
 // received by the Tailscale daemon in its staging/cache directory but not yet
 // transferred by the user's CLI or GUI client and written to a user's home
 // directory somewhere.
-func (lc *LocalClient) WaitingFiles(ctx context.Context) ([]apitype.WaitingFile, error) {
+func (lc *Client) WaitingFiles(ctx context.Context) ([]apitype.WaitingFile, error) {
 	return lc.AwaitWaitingFiles(ctx, 0)
 }
 
 // AwaitWaitingFiles is like WaitingFiles but takes a duration to await for an answer.
 // If the duration is 0, it will return immediately. The duration is respected at second
 // granularity only. If no files are available, it returns (nil, nil).
-func (lc *LocalClient) AwaitWaitingFiles(ctx context.Context, d time.Duration) ([]apitype.WaitingFile, error) {
+func (lc *Client) AwaitWaitingFiles(ctx context.Context, d time.Duration) ([]apitype.WaitingFile, error) {
 	path := "/localapi/v0/files/?waitsec=" + fmt.Sprint(int(d.Seconds()))
 	body, err := lc.get200(ctx, path)
 	if err != nil {
@@ -676,12 +689,12 @@ func (lc *LocalClient) AwaitWaitingFiles(ctx context.Context, d time.Duration) (
 	return decodeJSON[[]apitype.WaitingFile](body)
 }
 
-func (lc *LocalClient) DeleteWaitingFile(ctx context.Context, baseName string) error {
+func (lc *Client) DeleteWaitingFile(ctx context.Context, baseName string) error {
 	_, err := lc.send(ctx, "DELETE", "/localapi/v0/files/"+url.PathEscape(baseName), http.StatusNoContent, nil)
 	return err
 }
 
-func (lc *LocalClient) GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, size int64, err error) {
+func (lc *Client) GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, size int64, err error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/files/"+url.PathEscape(baseName), nil)
 	if err != nil {
 		return nil, 0, err
@@ -702,7 +715,7 @@ func (lc *LocalClient) GetWaitingFile(ctx context.Context, baseName string) (rc
 	return res.Body, res.ContentLength, nil
 }
 
-func (lc *LocalClient) FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
+func (lc *Client) FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/file-targets")
 	if err != nil {
 		return nil, err
@@ -714,7 +727,7 @@ func (lc *LocalClient) FileTargets(ctx context.Context) ([]apitype.FileTarget, e
 //
 // A size of -1 means unknown.
 // The name parameter is the original filename, not escaped.
-func (lc *LocalClient) PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
+func (lc *Client) PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
 	req, err := http.NewRequestWithContext(ctx, "PUT", "http://"+apitype.LocalAPIHost+"/localapi/v0/file-put/"+string(target)+"/"+url.PathEscape(name), r)
 	if err != nil {
 		return err
@@ -737,7 +750,7 @@ func (lc *LocalClient) PushFile(ctx context.Context, target tailcfg.StableNodeID
 // CheckIPForwarding asks the local Tailscale daemon whether it looks like the
 // machine is properly configured to forward IP packets as a subnet router
 // or exit node.
-func (lc *LocalClient) CheckIPForwarding(ctx context.Context) error {
+func (lc *Client) CheckIPForwarding(ctx context.Context) error {
 	body, err := lc.get200(ctx, "/localapi/v0/check-ip-forwarding")
 	if err != nil {
 		return err
@@ -757,7 +770,7 @@ func (lc *LocalClient) CheckIPForwarding(ctx context.Context) error {
 // CheckUDPGROForwarding asks the local Tailscale daemon whether it looks like
 // the machine is optimally configured to forward UDP packets as a subnet router
 // or exit node.
-func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
+func (lc *Client) CheckUDPGROForwarding(ctx context.Context) error {
 	body, err := lc.get200(ctx, "/localapi/v0/check-udp-gro-forwarding")
 	if err != nil {
 		return err
@@ -778,7 +791,7 @@ func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
 // node. This can be done to improve performance of tailnet nodes acting as exit
 // nodes or subnet routers.
 // See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
-func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
+func (lc *Client) SetUDPGROForwarding(ctx context.Context) error {
 	body, err := lc.get200(ctx, "/localapi/v0/set-udp-gro-forwarding")
 	if err != nil {
 		return err
@@ -801,12 +814,12 @@ func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
 // work. Currently (2022-04-18) this only checks for SSH server compatibility.
 // Note that EditPrefs does the same validation as this, so call CheckPrefs before
 // EditPrefs is not necessary.
-func (lc *LocalClient) CheckPrefs(ctx context.Context, p *ipn.Prefs) error {
+func (lc *Client) CheckPrefs(ctx context.Context, p *ipn.Prefs) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/check-prefs", http.StatusOK, jsonBody(p))
 	return err
 }
 
-func (lc *LocalClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
+func (lc *Client) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/prefs")
 	if err != nil {
 		return nil, err
@@ -818,7 +831,12 @@ func (lc *LocalClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
 	return &p, nil
 }
 
-func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
+// EditPrefs updates the [ipn.Prefs] of the current Tailscale profile, applying the changes in mp.
+// It returns an error if the changes cannot be applied, such as due to the caller's access rights
+// or a policy restriction. An optional reason or justification for the request can be
+// provided as a context value using [apitype.RequestReasonKey]. If permitted by policy,
+// access may be granted, and the reason will be logged for auditing purposes.
+func (lc *Client) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
 	body, err := lc.send(ctx, "PATCH", "/localapi/v0/prefs", http.StatusOK, jsonBody(mp))
 	if err != nil {
 		return nil, err
@@ -827,7 +845,7 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 }
 
 // GetEffectivePolicy returns the effective policy for the specified scope.
-func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+func (lc *Client) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
 	scopeID, err := scope.MarshalText()
 	if err != nil {
 		return nil, err
@@ -841,7 +859,7 @@ func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.Pol
 
 // ReloadEffectivePolicy reloads the effective policy for the specified scope
 // by reading and merging policy settings from all applicable policy sources.
-func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+func (lc *Client) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
 	scopeID, err := scope.MarshalText()
 	if err != nil {
 		return nil, err
@@ -855,7 +873,7 @@ func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.
 
 // GetDNSOSConfig returns the system DNS configuration for the current device.
 // That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
-func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
+func (lc *Client) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
 	if err != nil {
 		return nil, err
@@ -870,7 +888,7 @@ func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig
 // QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
 // It returns the raw DNS response bytes and the resolvers that were used to answer the query
 // (often just one, but can be more if we raced multiple resolvers).
-func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
+func (lc *Client) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
 	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
 	if err != nil {
 		return nil, nil, err
@@ -883,20 +901,20 @@ func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType stri
 }
 
 // StartLoginInteractive starts an interactive login.
-func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
+func (lc *Client) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)
 	return err
 }
 
 // Start applies the configuration specified in opts, and starts the
 // state machine.
-func (lc *LocalClient) Start(ctx context.Context, opts ipn.Options) error {
+func (lc *Client) Start(ctx context.Context, opts ipn.Options) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/start", http.StatusNoContent, jsonBody(opts))
 	return err
 }
 
 // Logout logs out the current node.
-func (lc *LocalClient) Logout(ctx context.Context) error {
+func (lc *Client) Logout(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/logout", http.StatusNoContent, nil)
 	return err
 }
@@ -915,7 +933,7 @@ func (lc *LocalClient) Logout(ctx context.Context) error {
 // This is a low-level interface; it's expected that most Tailscale
 // users use a higher level interface to getting/using TLS
 // certificates.
-func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
+func (lc *Client) SetDNS(ctx context.Context, name, value string) error {
 	v := url.Values{}
 	v.Set("name", name)
 	v.Set("value", value)
@@ -929,7 +947,7 @@ func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
 // tailscaled), a FQDN, or an IP address.
 //
 // The ctx is only used for the duration of the call, not the lifetime of the net.Conn.
-func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
+func (lc *Client) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
 	return lc.UserDial(ctx, "tcp", host, port)
 }
 
@@ -940,7 +958,7 @@ func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (n
 //
 // The ctx is only used for the duration of the call, not the lifetime of the
 // net.Conn.
-func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
+func (lc *Client) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
 	connCh := make(chan net.Conn, 1)
 	trace := httptrace.ClientTrace{
 		GotConn: func(info httptrace.GotConnInfo) {
@@ -991,7 +1009,7 @@ func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port
 
 // CurrentDERPMap returns the current DERPMap that is being used by the local tailscaled.
 // It is intended to be used with netcheck to see availability of DERPs.
-func (lc *LocalClient) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
+func (lc *Client) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
 	var derpMap tailcfg.DERPMap
 	res, err := lc.send(ctx, "GET", "/localapi/v0/derpmap", 200, nil)
 	if err != nil {
@@ -1007,9 +1025,9 @@ func (lc *LocalClient) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, er
 //
 // It returns a cached certificate from disk if it's still valid.
 //
-// Deprecated: use LocalClient.CertPair.
+// Deprecated: use Client.CertPair.
 func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
-	return defaultLocalClient.CertPair(ctx, domain)
+	return defaultClient.CertPair(ctx, domain)
 }
 
 // CertPair returns a cert and private key for the provided DNS domain.
@@ -1017,7 +1035,7 @@ func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err e
 // It returns a cached certificate from disk if it's still valid.
 //
 // API maturity: this is considered a stable API.
-func (lc *LocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+func (lc *Client) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
 	return lc.CertPairWithValidity(ctx, domain, 0)
 }
 
@@ -1030,7 +1048,7 @@ func (lc *LocalClient) CertPair(ctx context.Context, domain string) (certPEM, ke
 // valid, but for less than minValidity, it will be synchronously renewed.
 //
 // API maturity: this is considered a stable API.
-func (lc *LocalClient) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) {
+func (lc *Client) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) {
 	res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil)
 	if err != nil {
 		return nil, nil, err
@@ -1056,9 +1074,9 @@ func (lc *LocalClient) CertPairWithValidity(ctx context.Context, domain string,
 // It's the right signature to use as the value of
 // tls.Config.GetCertificate.
 //
-// Deprecated: use LocalClient.GetCertificate.
+// Deprecated: use Client.GetCertificate.
 func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	return defaultLocalClient.GetCertificate(hi)
+	return defaultClient.GetCertificate(hi)
 }
 
 // GetCertificate fetches a TLS certificate for the TLS ClientHello in hi.
@@ -1069,7 +1087,7 @@ func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
 // tls.Config.GetCertificate.
 //
 // API maturity: this is considered a stable API.
-func (lc *LocalClient) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+func (lc *Client) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
 	if hi == nil || hi.ServerName == "" {
 		return nil, errors.New("no SNI ServerName")
 	}
@@ -1095,13 +1113,13 @@ func (lc *LocalClient) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate
 
 // ExpandSNIName expands bare label name into the most likely actual TLS cert name.
 //
-// Deprecated: use LocalClient.ExpandSNIName.
+// Deprecated: use Client.ExpandSNIName.
 func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
-	return defaultLocalClient.ExpandSNIName(ctx, name)
+	return defaultClient.ExpandSNIName(ctx, name)
 }
 
 // ExpandSNIName expands bare label name into the most likely actual TLS cert name.
-func (lc *LocalClient) ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
+func (lc *Client) ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
 	st, err := lc.StatusWithoutPeers(ctx)
 	if err != nil {
 		return "", false
@@ -1129,7 +1147,7 @@ type PingOpts struct {
 
 // Ping sends a ping of the provided type to the provided IP and waits
 // for its response. The opts type specifies additional options.
-func (lc *LocalClient) PingWithOpts(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType, opts PingOpts) (*ipnstate.PingResult, error) {
+func (lc *Client) PingWithOpts(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType, opts PingOpts) (*ipnstate.PingResult, error) {
 	v := url.Values{}
 	v.Set("ip", ip.String())
 	v.Set("size", strconv.Itoa(opts.Size))
@@ -1143,12 +1161,12 @@ func (lc *LocalClient) PingWithOpts(ctx context.Context, ip netip.Addr, pingtype
 
 // Ping sends a ping of the provided type to the provided IP and waits
 // for its response.
-func (lc *LocalClient) Ping(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType) (*ipnstate.PingResult, error) {
+func (lc *Client) Ping(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType) (*ipnstate.PingResult, error) {
 	return lc.PingWithOpts(ctx, ip, pingtype, PingOpts{})
 }
 
 // NetworkLockStatus fetches information about the tailnet key authority, if one is configured.
-func (lc *LocalClient) NetworkLockStatus(ctx context.Context) (*ipnstate.NetworkLockStatus, error) {
+func (lc *Client) NetworkLockStatus(ctx context.Context) (*ipnstate.NetworkLockStatus, error) {
 	body, err := lc.send(ctx, "GET", "/localapi/v0/tka/status", 200, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error: %w", err)
@@ -1159,7 +1177,7 @@ func (lc *LocalClient) NetworkLockStatus(ctx context.Context) (*ipnstate.Network
 // NetworkLockInit initializes the tailnet key authority.
 //
 // TODO(tom): Plumb through disablement secrets.
-func (lc *LocalClient) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.NetworkLockStatus, error) {
+func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.NetworkLockStatus, error) {
 	var b bytes.Buffer
 	type initRequest struct {
 		Keys               []tka.Key
@@ -1180,7 +1198,7 @@ func (lc *LocalClient) NetworkLockInit(ctx context.Context, keys []tka.Key, disa
 
 // NetworkLockWrapPreauthKey wraps a pre-auth key with information to
 // enable unattended bringup in the locked tailnet.
-func (lc *LocalClient) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) {
+func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) {
 	encodedPrivate, err := tkaKey.MarshalText()
 	if err != nil {
 		return "", err
@@ -1203,7 +1221,7 @@ func (lc *LocalClient) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey
 }
 
 // NetworkLockModify adds and/or removes key(s) to the tailnet key authority.
-func (lc *LocalClient) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error {
+func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error {
 	var b bytes.Buffer
 	type modifyRequest struct {
 		AddKeys    []tka.Key
@@ -1222,7 +1240,7 @@ func (lc *LocalClient) NetworkLockModify(ctx context.Context, addKeys, removeKey
 
 // NetworkLockSign signs the specified node-key and transmits that signature to the control plane.
 // rotationPublic, if specified, must be an ed25519 public key.
-func (lc *LocalClient) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error {
+func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error {
 	var b bytes.Buffer
 	type signRequest struct {
 		NodeKey        key.NodePublic
@@ -1240,7 +1258,7 @@ func (lc *LocalClient) NetworkLockSign(ctx context.Context, nodeKey key.NodePubl
 }
 
 // NetworkLockAffectedSigs returns all signatures signed by the specified keyID.
-func (lc *LocalClient) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) {
+func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/tka/affected-sigs", 200, bytes.NewReader(keyID))
 	if err != nil {
 		return nil, fmt.Errorf("error: %w", err)
@@ -1249,7 +1267,7 @@ func (lc *LocalClient) NetworkLockAffectedSigs(ctx context.Context, keyID tkatyp
 }
 
 // NetworkLockLog returns up to maxEntries number of changes to network-lock state.
-func (lc *LocalClient) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.NetworkLockUpdate, error) {
+func (lc *Client) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.NetworkLockUpdate, error) {
 	v := url.Values{}
 	v.Set("limit", fmt.Sprint(maxEntries))
 	body, err := lc.send(ctx, "GET", "/localapi/v0/tka/log?"+v.Encode(), 200, nil)
@@ -1260,7 +1278,7 @@ func (lc *LocalClient) NetworkLockLog(ctx context.Context, maxEntries int) ([]ip
 }
 
 // NetworkLockForceLocalDisable forcibly shuts down network lock on this node.
-func (lc *LocalClient) NetworkLockForceLocalDisable(ctx context.Context) error {
+func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error {
 	// This endpoint expects an empty JSON stanza as the payload.
 	var b bytes.Buffer
 	if err := json.NewEncoder(&b).Encode(struct{}{}); err != nil {
@@ -1275,7 +1293,7 @@ func (lc *LocalClient) NetworkLockForceLocalDisable(ctx context.Context) error {
 
 // NetworkLockVerifySigningDeeplink verifies the network lock deeplink contained
 // in url and returns information extracted from it.
-func (lc *LocalClient) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) {
+func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) {
 	vr := struct {
 		URL string
 	}{url}
@@ -1289,7 +1307,7 @@ func (lc *LocalClient) NetworkLockVerifySigningDeeplink(ctx context.Context, url
 }
 
 // NetworkLockGenRecoveryAUM generates an AUM for recovering from a tailnet-lock key compromise.
-func (lc *LocalClient) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) {
+func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) {
 	vr := struct {
 		Keys     []tkatype.KeyID
 		ForkFrom string
@@ -1304,7 +1322,7 @@ func (lc *LocalClient) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys
 }
 
 // NetworkLockCosignRecoveryAUM co-signs a recovery AUM using the node's tailnet lock key.
-func (lc *LocalClient) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) {
+func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) {
 	r := bytes.NewReader(aum.Serialize())
 	body, err := lc.send(ctx, "POST", "/localapi/v0/tka/cosign-recovery-aum", 200, r)
 	if err != nil {
@@ -1315,7 +1333,7 @@ func (lc *LocalClient) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka
 }
 
 // NetworkLockSubmitRecoveryAUM submits a recovery AUM to the control plane.
-func (lc *LocalClient) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error {
+func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error {
 	r := bytes.NewReader(aum.Serialize())
 	_, err := lc.send(ctx, "POST", "/localapi/v0/tka/submit-recovery-aum", 200, r)
 	if err != nil {
@@ -1326,7 +1344,7 @@ func (lc *LocalClient) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka
 
 // SetServeConfig sets or replaces the serving settings.
 // If config is nil, settings are cleared and serving is disabled.
-func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConfig) error {
+func (lc *Client) SetServeConfig(ctx context.Context, config *ipn.ServeConfig) error {
 	h := make(http.Header)
 	if config != nil {
 		h.Set("If-Match", config.ETag)
@@ -1341,7 +1359,7 @@ func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConf
 // DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be
 // run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over
 // to another replica whilst there is still some grace period for the existing connections to terminate.
-func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
+func (lc *Client) DisconnectControl(ctx context.Context) error {
 	_, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil)
 	if err != nil {
 		return fmt.Errorf("error disconnecting control: %w", err)
@@ -1350,7 +1368,7 @@ func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
 }
 
 // NetworkLockDisable shuts down network-lock across the tailnet.
-func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) error {
+func (lc *Client) NetworkLockDisable(ctx context.Context, secret []byte) error {
 	if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil {
 		return fmt.Errorf("error: %w", err)
 	}
@@ -1360,7 +1378,7 @@ func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) er
 // GetServeConfig return the current serve config.
 //
 // If the serve config is empty, it returns (nil, nil).
-func (lc *LocalClient) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) {
+func (lc *Client) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) {
 	body, h, err := lc.sendWithHeaders(ctx, "GET", "/localapi/v0/serve-config", 200, nil, nil)
 	if err != nil {
 		return nil, fmt.Errorf("getting serve config: %w", err)
@@ -1435,7 +1453,7 @@ func (r jsonReader) Read(p []byte) (n int, err error) {
 }
 
 // ProfileStatus returns the current profile and the list of all profiles.
-func (lc *LocalClient) ProfileStatus(ctx context.Context) (current ipn.LoginProfile, all []ipn.LoginProfile, err error) {
+func (lc *Client) ProfileStatus(ctx context.Context) (current ipn.LoginProfile, all []ipn.LoginProfile, err error) {
 	body, err := lc.send(ctx, "GET", "/localapi/v0/profiles/current", 200, nil)
 	if err != nil {
 		return
@@ -1453,7 +1471,7 @@ func (lc *LocalClient) ProfileStatus(ctx context.Context) (current ipn.LoginProf
 }
 
 // ReloadConfig reloads the config file, if possible.
-func (lc *LocalClient) ReloadConfig(ctx context.Context) (ok bool, err error) {
+func (lc *Client) ReloadConfig(ctx context.Context) (ok bool, err error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/reload-config", 200, nil)
 	if err != nil {
 		return
@@ -1471,13 +1489,13 @@ func (lc *LocalClient) ReloadConfig(ctx context.Context) (ok bool, err error) {
 // SwitchToEmptyProfile creates and switches to a new unnamed profile. The new
 // profile is not assigned an ID until it is persisted after a successful login.
 // In order to login to the new profile, the user must call LoginInteractive.
-func (lc *LocalClient) SwitchToEmptyProfile(ctx context.Context) error {
+func (lc *Client) SwitchToEmptyProfile(ctx context.Context) error {
 	_, err := lc.send(ctx, "PUT", "/localapi/v0/profiles/", http.StatusCreated, nil)
 	return err
 }
 
 // SwitchProfile switches to the given profile.
-func (lc *LocalClient) SwitchProfile(ctx context.Context, profile ipn.ProfileID) error {
+func (lc *Client) SwitchProfile(ctx context.Context, profile ipn.ProfileID) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/profiles/"+url.PathEscape(string(profile)), 204, nil)
 	return err
 }
@@ -1485,7 +1503,7 @@ func (lc *LocalClient) SwitchProfile(ctx context.Context, profile ipn.ProfileID)
 // DeleteProfile removes the profile with the given ID.
 // If the profile is the current profile, an empty profile
 // will be selected as if SwitchToEmptyProfile was called.
-func (lc *LocalClient) DeleteProfile(ctx context.Context, profile ipn.ProfileID) error {
+func (lc *Client) DeleteProfile(ctx context.Context, profile ipn.ProfileID) error {
 	_, err := lc.send(ctx, "DELETE", "/localapi/v0/profiles"+url.PathEscape(string(profile)), http.StatusNoContent, nil)
 	return err
 }
@@ -1502,7 +1520,7 @@ func (lc *LocalClient) DeleteProfile(ctx context.Context, profile ipn.ProfileID)
 // to block until the feature has been enabled.
 //
 // 2023-08-09: Valid feature values are "serve" and "funnel".
-func (lc *LocalClient) QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) {
+func (lc *Client) QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) {
 	v := url.Values{"feature": {feature}}
 	body, err := lc.send(ctx, "POST", "/localapi/v0/query-feature?"+v.Encode(), 200, nil)
 	if err != nil {
@@ -1511,7 +1529,7 @@ func (lc *LocalClient) QueryFeature(ctx context.Context, feature string) (*tailc
 	return decodeJSON[*tailcfg.QueryFeatureResponse](body)
 }
 
-func (lc *LocalClient) DebugDERPRegion(ctx context.Context, regionIDOrCode string) (*ipnstate.DebugDERPRegionReport, error) {
+func (lc *Client) DebugDERPRegion(ctx context.Context, regionIDOrCode string) (*ipnstate.DebugDERPRegionReport, error) {
 	v := url.Values{"region": {regionIDOrCode}}
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug-derp-region?"+v.Encode(), 200, nil)
 	if err != nil {
@@ -1521,7 +1539,7 @@ func (lc *LocalClient) DebugDERPRegion(ctx context.Context, regionIDOrCode strin
 }
 
 // DebugPacketFilterRules returns the packet filter rules for the current device.
-func (lc *LocalClient) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterRule, error) {
+func (lc *Client) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterRule, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug-packet-filter-rules", 200, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error %w: %s", err, body)
@@ -1532,7 +1550,7 @@ func (lc *LocalClient) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.Fi
 // DebugSetExpireIn marks the current node key to expire in d.
 //
 // This is meant primarily for debug and testing.
-func (lc *LocalClient) DebugSetExpireIn(ctx context.Context, d time.Duration) error {
+func (lc *Client) DebugSetExpireIn(ctx context.Context, d time.Duration) error {
 	v := url.Values{"expiry": {fmt.Sprint(time.Now().Add(d).Unix())}}
 	_, err := lc.send(ctx, "POST", "/localapi/v0/set-expiry-sooner?"+v.Encode(), 200, nil)
 	return err
@@ -1542,7 +1560,7 @@ func (lc *LocalClient) DebugSetExpireIn(ctx context.Context, d time.Duration) er
 //
 // The provided context does not determine the lifetime of the
 // returned io.ReadCloser.
-func (lc *LocalClient) StreamDebugCapture(ctx context.Context) (io.ReadCloser, error) {
+func (lc *Client) StreamDebugCapture(ctx context.Context) (io.ReadCloser, error) {
 	req, err := http.NewRequestWithContext(ctx, "POST", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-capture", nil)
 	if err != nil {
 		return nil, err
@@ -1568,7 +1586,7 @@ func (lc *LocalClient) StreamDebugCapture(ctx context.Context) (io.ReadCloser, e
 // resources.
 //
 // A default set of ipn.Notify messages are returned but the set can be modified by mask.
-func (lc *LocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*IPNBusWatcher, error) {
+func (lc *Client) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*IPNBusWatcher, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET",
 		"http://"+apitype.LocalAPIHost+"/localapi/v0/watch-ipn-bus?mask="+fmt.Sprint(mask),
 		nil)
@@ -1594,7 +1612,7 @@ func (lc *LocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt)
 // CheckUpdate returns a tailcfg.ClientVersion indicating whether or not an update is available
 // to be installed via the LocalAPI. In case the LocalAPI can't install updates, it returns a
 // ClientVersion that says that we are up to date.
-func (lc *LocalClient) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion, error) {
+func (lc *Client) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/update/check")
 	if err != nil {
 		return nil, err
@@ -1610,7 +1628,7 @@ func (lc *LocalClient) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion,
 // To turn it on, there must have been a previously used exit node.
 // The most previously used one is reused.
 // This is a convenience method for GUIs. To select an actual one, update the prefs.
-func (lc *LocalClient) SetUseExitNode(ctx context.Context, on bool) error {
+func (lc *Client) SetUseExitNode(ctx context.Context, on bool) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/set-use-exit-node-enabled?enabled="+strconv.FormatBool(on), http.StatusOK, nil)
 	return err
 }
@@ -1618,7 +1636,7 @@ func (lc *LocalClient) SetUseExitNode(ctx context.Context, on bool) error {
 // DriveSetServerAddr instructs Taildrive to use the server at addr to access
 // the filesystem. This is used on platforms like Windows and MacOS to let
 // Taildrive know to use the file server running in the GUI app.
-func (lc *LocalClient) DriveSetServerAddr(ctx context.Context, addr string) error {
+func (lc *Client) DriveSetServerAddr(ctx context.Context, addr string) error {
 	_, err := lc.send(ctx, "PUT", "/localapi/v0/drive/fileserver-address", http.StatusCreated, strings.NewReader(addr))
 	return err
 }
@@ -1626,14 +1644,14 @@ func (lc *LocalClient) DriveSetServerAddr(ctx context.Context, addr string) erro
 // DriveShareSet adds or updates the given share in the list of shares that
 // Taildrive will serve to remote nodes. If a share with the same name already
 // exists, the existing share is replaced/updated.
-func (lc *LocalClient) DriveShareSet(ctx context.Context, share *drive.Share) error {
+func (lc *Client) DriveShareSet(ctx context.Context, share *drive.Share) error {
 	_, err := lc.send(ctx, "PUT", "/localapi/v0/drive/shares", http.StatusCreated, jsonBody(share))
 	return err
 }
 
 // DriveShareRemove removes the share with the given name from the list of
 // shares that Taildrive will serve to remote nodes.
-func (lc *LocalClient) DriveShareRemove(ctx context.Context, name string) error {
+func (lc *Client) DriveShareRemove(ctx context.Context, name string) error {
 	_, err := lc.send(
 		ctx,
 		"DELETE",
@@ -1644,7 +1662,7 @@ func (lc *LocalClient) DriveShareRemove(ctx context.Context, name string) error
 }
 
 // DriveShareRename renames the share from old to new name.
-func (lc *LocalClient) DriveShareRename(ctx context.Context, oldName, newName string) error {
+func (lc *Client) DriveShareRename(ctx context.Context, oldName, newName string) error {
 	_, err := lc.send(
 		ctx,
 		"POST",
@@ -1656,7 +1674,7 @@ func (lc *LocalClient) DriveShareRename(ctx context.Context, oldName, newName st
 
 // DriveShareList returns the list of shares that drive is currently serving
 // to remote nodes.
-func (lc *LocalClient) DriveShareList(ctx context.Context) ([]*drive.Share, error) {
+func (lc *Client) DriveShareList(ctx context.Context) ([]*drive.Share, error) {
 	result, err := lc.get200(ctx, "/localapi/v0/drive/shares")
 	if err != nil {
 		return nil, err
@@ -1667,7 +1685,7 @@ func (lc *LocalClient) DriveShareList(ctx context.Context) ([]*drive.Share, erro
 }
 
 // IPNBusWatcher is an active subscription (watch) of the local tailscaled IPN bus.
-// It's returned by LocalClient.WatchIPNBus.
+// It's returned by Client.WatchIPNBus.
 //
 // It must be closed when done.
 type IPNBusWatcher struct {
@@ -1691,7 +1709,7 @@ func (w *IPNBusWatcher) Close() error {
 }
 
 // Next returns the next ipn.Notify from the stream.
-// If the context from LocalClient.WatchIPNBus is done, that error is returned.
+// If the context from Client.WatchIPNBus is done, that error is returned.
 func (w *IPNBusWatcher) Next() (ipn.Notify, error) {
 	var n ipn.Notify
 	if err := w.dec.Decode(&n); err != nil {
@@ -1704,7 +1722,7 @@ func (w *IPNBusWatcher) Next() (ipn.Notify, error) {
 }
 
 // SuggestExitNode requests an exit node suggestion and returns the exit node's details.
-func (lc *LocalClient) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) {
+func (lc *Client) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/suggest-exit-node")
 	if err != nil {
 		return apitype.ExitNodeSuggestionResponse{}, err

client/local/local_test.go

@@ -3,7 +3,7 @@
 
 //go:build go1.19
 
-package tailscale
+package local
 
 import (
 	"context"
@@ -41,7 +41,7 @@ func TestWhoIsPeerNotFound(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	lc := &LocalClient{
+	lc := &Client{
 		Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
 			var std net.Dialer
 			return std.DialContext(ctx, network, ts.Listener.Addr().(*net.TCPAddr).String())

client/systray/logo.go

@@ -0,0 +1,319 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+package systray
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"image/color"
+	"image/png"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/fogleman/gg"
+)
+
+// tsLogo represents the Tailscale logo displayed as the systray icon.
+type tsLogo struct {
+	// dots represents the state of the 3x3 dot grid in the logo.
+	// A 0 represents a gray dot, any other value is a white dot.
+	dots [9]byte
+
+	// dotMask returns an image mask to be used when rendering the logo dots.
+	dotMask func(dc *gg.Context, borderUnits int, radius int) *image.Alpha
+
+	// overlay is called after the dots are rendered to draw an additional overlay.
+	overlay func(dc *gg.Context, borderUnits int, radius int)
+}
+
+var (
+	// disconnected is all gray dots
+	disconnected = tsLogo{dots: [9]byte{
+		0, 0, 0,
+		0, 0, 0,
+		0, 0, 0,
+	}}
+
+	// connected is the normal Tailscale logo
+	connected = tsLogo{dots: [9]byte{
+		0, 0, 0,
+		1, 1, 1,
+		0, 1, 0,
+	}}
+
+	// loading is a special tsLogo value that is not meant to be rendered directly,
+	// but indicates that the loading animation should be shown.
+	loading = tsLogo{dots: [9]byte{'l', 'o', 'a', 'd', 'i', 'n', 'g'}}
+
+	// loadingIcons are shown in sequence as an animated loading icon.
+	loadingLogos = []tsLogo{
+		{dots: [9]byte{
+			0, 1, 1,
+			1, 0, 1,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 1, 1,
+			0, 0, 1,
+			0, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 1, 1,
+			0, 0, 0,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 0, 1,
+			0, 1, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 1, 0,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 1,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 1,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			1, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			1, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 0, 0,
+			1, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 1,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 1, 0,
+			0, 1, 1,
+			1, 0, 1,
+		}},
+	}
+
+	// exitNodeOnline is the Tailscale logo with an additional arrow overlay in the corner.
+	exitNodeOnline = tsLogo{
+		dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 1, 0,
+		},
+		// draw an arrow mask in the bottom right corner with a reasonably thick line width.
+		dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 3.5)
+			y := r * (bu + 7)
+			x2 := x1 + (r * 5)
+
+			mc := gg.NewContext(dc.Width(), dc.Height())
+			mc.DrawLine(x1, y, x2, y)                 // arrow center line
+			mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
+			mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
+			mc.SetLineWidth(r * 3)
+			mc.Stroke()
+			return mc.AsMask()
+		},
+		// draw an arrow in the bottom right corner over the masked area.
+		overlay: func(dc *gg.Context, borderUnits int, radius int) {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 3.5)
+			y := r * (bu + 7)
+			x2 := x1 + (r * 5)
+
+			dc.DrawLine(x1, y, x2, y)                 // arrow center line
+			dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
+			dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
+			dc.SetColor(fg)
+			dc.SetLineWidth(r)
+			dc.Stroke()
+		},
+	}
+
+	// exitNodeOffline is the Tailscale logo with a red "x" in the corner.
+	exitNodeOffline = tsLogo{
+		dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 1, 0,
+		},
+		// Draw a square that hides the four dots in the bottom right corner,
+		dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
+			bu, r := float64(borderUnits), float64(radius)
+			x := r * (bu + 3)
+
+			mc := gg.NewContext(dc.Width(), dc.Height())
+			mc.DrawRectangle(x, x, r*6, r*6)
+			mc.Fill()
+			return mc.AsMask()
+		},
+		// draw a red "x" over the bottom right corner.
+		overlay: func(dc *gg.Context, borderUnits int, radius int) {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 4)
+			x2 := x1 + (r * 3.5)
+			dc.DrawLine(x1, x1, x2, x2) // top-left to bottom-right stroke
+			dc.DrawLine(x1, x2, x2, x1) // bottom-left to top-right stroke
+			dc.SetColor(red)
+			dc.SetLineWidth(r)
+			dc.Stroke()
+		},
+	}
+)
+
+var (
+	bg   = color.NRGBA{0, 0, 0, 255}
+	fg   = color.NRGBA{255, 255, 255, 255}
+	gray = color.NRGBA{255, 255, 255, 102}
+	red  = color.NRGBA{229, 111, 74, 255}
+)
+
+// render returns a PNG image of the logo.
+func (logo tsLogo) render() *bytes.Buffer {
+	const borderUnits = 1
+	return logo.renderWithBorder(borderUnits)
+}
+
+// renderWithBorder returns a PNG image of the logo with the specified border width.
+// One border unit is equal to the radius of a tailscale logo dot.
+func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer {
+	const radius = 25
+	dim := radius * (8 + borderUnits*2)
+
+	dc := gg.NewContext(dim, dim)
+	dc.DrawRectangle(0, 0, float64(dim), float64(dim))
+	dc.SetColor(bg)
+	dc.Fill()
+
+	if logo.dotMask != nil {
+		mask := logo.dotMask(dc, borderUnits, radius)
+		dc.SetMask(mask)
+		dc.InvertMask()
+	}
+
+	for y := 0; y < 3; y++ {
+		for x := 0; x < 3; x++ {
+			px := (borderUnits + 1 + 3*x) * radius
+			py := (borderUnits + 1 + 3*y) * radius
+			col := fg
+			if logo.dots[y*3+x] == 0 {
+				col = gray
+			}
+			dc.DrawCircle(float64(px), float64(py), radius)
+			dc.SetColor(col)
+			dc.Fill()
+		}
+	}
+
+	if logo.overlay != nil {
+		dc.ResetClip()
+		logo.overlay(dc, borderUnits, radius)
+	}
+
+	b := bytes.NewBuffer(nil)
+	png.Encode(b, dc.Image())
+	return b
+}
+
+// setAppIcon renders logo and sets it as the systray icon.
+func setAppIcon(icon tsLogo) {
+	if icon.dots == loading.dots {
+		startLoadingAnimation()
+	} else {
+		stopLoadingAnimation()
+		systray.SetIcon(icon.render().Bytes())
+	}
+}
+
+var (
+	loadingMu sync.Mutex // protects loadingCancel
+
+	// loadingCancel stops the loading animation in the systray icon.
+	// This is nil if the animation is not currently active.
+	loadingCancel func()
+)
+
+// startLoadingAnimation starts the animated loading icon in the system tray.
+// The animation continues until [stopLoadingAnimation] is called.
+// If the loading animation is already active, this func does nothing.
+func startLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		// loading icon already displayed
+		return
+	}
+
+	ctx := context.Background()
+	ctx, loadingCancel = context.WithCancel(ctx)
+
+	go func() {
+		t := time.NewTicker(500 * time.Millisecond)
+		var i int
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				systray.SetIcon(loadingLogos[i].render().Bytes())
+				i++
+				if i >= len(loadingLogos) {
+					i = 0
+				}
+			}
+		}
+	}()
+}
+
+// stopLoadingAnimation stops the animated loading icon in the system tray.
+// If the loading animation is not currently active, this func does nothing.
+func stopLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		loadingCancel()
+		loadingCancel = nil
+	}
+}

client/systray/systray.go

@@ -0,0 +1,740 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+// Package systray provides a minimal Tailscale systray application.
+package systray
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/atotto/clipboard"
+	dbus "github.com/godbus/dbus/v5"
+	"github.com/toqueteos/webbrowser"
+	"tailscale.com/client/local"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/slicesx"
+	"tailscale.com/util/stringsx"
+)
+
+var (
+	// newMenuDelay is the amount of time to sleep after creating a new menu,
+	// but before adding items to it. This works around a bug in some dbus implementations.
+	newMenuDelay time.Duration
+
+	// if true, treat all mullvad exit node countries as single-city.
+	// Instead of rendering a submenu with cities, just select the highest-priority peer.
+	hideMullvadCities bool
+)
+
+// Run starts the systray menu and blocks until the menu exits.
+func (menu *Menu) Run() {
+	menu.updateState()
+
+	// exit cleanly on SIGINT and SIGTERM
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		select {
+		case <-interrupt:
+			menu.onExit()
+		case <-menu.bgCtx.Done():
+		}
+	}()
+	go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1)
+
+	systray.Run(menu.onReady, menu.onExit)
+}
+
+// Menu represents the systray menu, its items, and the current Tailscale state.
+type Menu struct {
+	mu sync.Mutex // protects the entire Menu
+
+	lc          local.Client
+	status      *ipnstate.Status
+	curProfile  ipn.LoginProfile
+	allProfiles []ipn.LoginProfile
+
+	// readonly is whether the systray app is running in read-only mode.
+	// This is set if LocalAPI returns a permission error,
+	// typically because the user needs to run `tailscale set --operator=$USER`.
+	readonly bool
+
+	bgCtx    context.Context // ctx for background tasks not involving menu item clicks
+	bgCancel context.CancelFunc
+
+	// Top-level menu items
+	connect    *systray.MenuItem
+	disconnect *systray.MenuItem
+	self       *systray.MenuItem
+	exitNodes  *systray.MenuItem
+	more       *systray.MenuItem
+	quit       *systray.MenuItem
+
+	rebuildCh  chan struct{} // triggers a menu rebuild
+	accountsCh chan ipn.ProfileID
+	exitNodeCh chan tailcfg.StableNodeID // ID of selected exit node
+
+	eventCancel context.CancelFunc // cancel eventLoop
+
+	notificationIcon *os.File // icon used for desktop notifications
+}
+
+func (menu *Menu) init() {
+	if menu.bgCtx != nil {
+		// already initialized
+		return
+	}
+
+	menu.rebuildCh = make(chan struct{}, 1)
+	menu.accountsCh = make(chan ipn.ProfileID)
+	menu.exitNodeCh = make(chan tailcfg.StableNodeID)
+
+	// dbus wants a file path for notification icons, so copy to a temp file.
+	menu.notificationIcon, _ = os.CreateTemp("", "tailscale-systray.png")
+	io.Copy(menu.notificationIcon, connected.renderWithBorder(3))
+
+	menu.bgCtx, menu.bgCancel = context.WithCancel(context.Background())
+	go menu.watchIPNBus()
+}
+
+func init() {
+	if runtime.GOOS != "linux" {
+		// so far, these tweaks are only needed on Linux
+		return
+	}
+
+	desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP"))
+	switch desktop {
+	case "gnome":
+		// GNOME expands submenus downward in the main menu, rather than flyouts to the side.
+		// Either as a result of that or another limitation, there seems to be a maximum depth of submenus.
+		// Mullvad countries that have a city submenu are not being rendered, and so can't be selected.
+		// Handle this by simply treating all mullvad countries as single-city and select the best peer.
+		hideMullvadCities = true
+	case "kde":
+		// KDE doesn't need a delay, and actually won't render submenus
+		// if we delay for more than about 400µs.
+		newMenuDelay = 0
+	default:
+		// Add a slight delay to ensure the menu is created before adding items.
+		//
+		// Systray implementations that use libdbusmenu sometimes process messages out of order,
+		// resulting in errors such as:
+		//    (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu'
+		//
+		// See also: https://github.com/fyne-io/systray/issues/12
+		newMenuDelay = 10 * time.Millisecond
+	}
+}
+
+// onReady is called by the systray package when the menu is ready to be built.
+func (menu *Menu) onReady() {
+	log.Printf("starting")
+	setAppIcon(disconnected)
+	menu.rebuild()
+}
+
+// updateState updates the Menu state from the Tailscale local client.
+func (menu *Menu) updateState() {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+	menu.init()
+
+	menu.readonly = false
+
+	var err error
+	menu.status, err = menu.lc.Status(menu.bgCtx)
+	if err != nil {
+		log.Print(err)
+	}
+	menu.curProfile, menu.allProfiles, err = menu.lc.ProfileStatus(menu.bgCtx)
+	if err != nil {
+		if local.IsAccessDeniedError(err) {
+			menu.readonly = true
+		}
+		log.Print(err)
+	}
+}
+
+// rebuild the systray menu based on the current Tailscale state.
+//
+// We currently rebuild the entire menu because it is not easy to update the existing menu.
+// You cannot iterate over the items in a menu, nor can you remove some items like separators.
+// So for now we rebuild the whole thing, and can optimize this later if needed.
+func (menu *Menu) rebuild() {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+	menu.init()
+
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+	ctx := context.Background()
+	ctx, menu.eventCancel = context.WithCancel(ctx)
+
+	systray.ResetMenu()
+
+	if menu.readonly {
+		const readonlyMsg = "No permission to manage Tailscale.\nSee tailscale.com/s/cli-operator"
+		m := systray.AddMenuItem(readonlyMsg, "")
+		onClick(ctx, m, func(_ context.Context) {
+			webbrowser.Open("https://tailscale.com/s/cli-operator")
+		})
+		systray.AddSeparator()
+	}
+
+	menu.connect = systray.AddMenuItem("Connect", "")
+	menu.disconnect = systray.AddMenuItem("Disconnect", "")
+	menu.disconnect.Hide()
+	systray.AddSeparator()
+
+	// delay to prevent race setting icon on first start
+	time.Sleep(newMenuDelay)
+
+	// Set systray menu icon and title.
+	// Also adjust connect/disconnect menu items if needed.
+	var backendState string
+	if menu.status != nil {
+		backendState = menu.status.BackendState
+	}
+	switch backendState {
+	case ipn.Running.String():
+		if menu.status.ExitNodeStatus != nil && !menu.status.ExitNodeStatus.ID.IsZero() {
+			if menu.status.ExitNodeStatus.Online {
+				setTooltip("Using exit node")
+				setAppIcon(exitNodeOnline)
+			} else {
+				setTooltip("Exit node offline")
+				setAppIcon(exitNodeOffline)
+			}
+		} else {
+			setTooltip(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name))
+			setAppIcon(connected)
+		}
+		menu.connect.SetTitle("Connected")
+		menu.connect.Disable()
+		menu.disconnect.Show()
+		menu.disconnect.Enable()
+	case ipn.Starting.String():
+		setTooltip("Connecting")
+		setAppIcon(loading)
+	default:
+		setTooltip("Disconnected")
+		setAppIcon(disconnected)
+	}
+
+	if menu.readonly {
+		menu.connect.Disable()
+		menu.disconnect.Disable()
+	}
+
+	account := "Account"
+	if pt := profileTitle(menu.curProfile); pt != "" {
+		account = pt
+	}
+	if !menu.readonly {
+		accounts := systray.AddMenuItem(account, "")
+		setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL)
+		time.Sleep(newMenuDelay)
+		for _, profile := range menu.allProfiles {
+			title := profileTitle(profile)
+			var item *systray.MenuItem
+			if profile.ID == menu.curProfile.ID {
+				item = accounts.AddSubMenuItemCheckbox(title, "", true)
+			} else {
+				item = accounts.AddSubMenuItem(title, "")
+			}
+			setRemoteIcon(item, profile.UserProfile.ProfilePicURL)
+			onClick(ctx, item, func(ctx context.Context) {
+				select {
+				case <-ctx.Done():
+				case menu.accountsCh <- profile.ID:
+				}
+			})
+		}
+	}
+
+	if menu.status != nil && menu.status.Self != nil && len(menu.status.Self.TailscaleIPs) > 0 {
+		title := fmt.Sprintf("This Device: %s (%s)", menu.status.Self.HostName, menu.status.Self.TailscaleIPs[0])
+		menu.self = systray.AddMenuItem(title, "")
+	} else {
+		menu.self = systray.AddMenuItem("This Device: not connected", "")
+		menu.self.Disable()
+	}
+	systray.AddSeparator()
+
+	if !menu.readonly {
+		menu.rebuildExitNodeMenu(ctx)
+	}
+
+	if menu.status != nil {
+		menu.more = systray.AddMenuItem("More settings", "")
+		onClick(ctx, menu.more, func(_ context.Context) {
+			webbrowser.Open("http://100.100.100.100/")
+		})
+	}
+
+	menu.quit = systray.AddMenuItem("Quit", "Quit the app")
+	menu.quit.Enable()
+
+	go menu.eventLoop(ctx)
+}
+
+// profileTitle returns the title string for a profile menu item.
+func profileTitle(profile ipn.LoginProfile) string {
+	title := profile.Name
+	if profile.NetworkProfile.DomainName != "" {
+		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+			// windows and mac don't support multi-line menu
+			title += " (" + profile.NetworkProfile.DomainName + ")"
+		} else {
+			title += "\n" + profile.NetworkProfile.DomainName
+		}
+	}
+	return title
+}
+
+var (
+	cacheMu   sync.Mutex
+	httpCache = map[string][]byte{} // URL => response body
+)
+
+// setRemoteIcon sets the icon for menu to the specified remote image.
+// Remote images are fetched as needed and cached.
+func setRemoteIcon(menu *systray.MenuItem, urlStr string) {
+	if menu == nil || urlStr == "" {
+		return
+	}
+
+	cacheMu.Lock()
+	b, ok := httpCache[urlStr]
+	if !ok {
+		resp, err := http.Get(urlStr)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			b, _ = io.ReadAll(resp.Body)
+			httpCache[urlStr] = b
+			resp.Body.Close()
+		}
+	}
+	cacheMu.Unlock()
+
+	if len(b) > 0 {
+		menu.SetIcon(b)
+	}
+}
+
+// setTooltip sets the tooltip text for the systray icon.
+func setTooltip(text string) {
+	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+		systray.SetTooltip(text)
+	} else {
+		// on Linux, SetTitle actually sets the tooltip
+		systray.SetTitle(text)
+	}
+}
+
+// eventLoop is the main event loop for handling click events on menu items
+// and responding to Tailscale state changes.
+// This method does not return until ctx.Done is closed.
+func (menu *Menu) eventLoop(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-menu.rebuildCh:
+			menu.updateState()
+			menu.rebuild()
+		case <-menu.connect.ClickedCh:
+			_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: true,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("error connecting: %v", err)
+			}
+
+		case <-menu.disconnect.ClickedCh:
+			_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: false,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("error disconnecting: %v", err)
+			}
+
+		case <-menu.self.ClickedCh:
+			menu.copyTailscaleIP(menu.status.Self)
+
+		case id := <-menu.accountsCh:
+			if err := menu.lc.SwitchProfile(ctx, id); err != nil {
+				log.Printf("error switching to profile ID %v: %v", id, err)
+			}
+
+		case exitNode := <-menu.exitNodeCh:
+			if exitNode.IsZero() {
+				log.Print("disable exit node")
+				if err := menu.lc.SetUseExitNode(ctx, false); err != nil {
+					log.Printf("error disabling exit node: %v", err)
+				}
+			} else {
+				log.Printf("enable exit node: %v", exitNode)
+				mp := &ipn.MaskedPrefs{
+					Prefs: ipn.Prefs{
+						ExitNodeID: exitNode,
+					},
+					ExitNodeIDSet: true,
+				}
+				if _, err := menu.lc.EditPrefs(ctx, mp); err != nil {
+					log.Printf("error setting exit node: %v", err)
+				}
+			}
+
+		case <-menu.quit.ClickedCh:
+			systray.Quit()
+		}
+	}
+}
+
+// onClick registers a click handler for a menu item.
+func onClick(ctx context.Context, item *systray.MenuItem, fn func(ctx context.Context)) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-item.ClickedCh:
+				fn(ctx)
+			}
+		}
+	}()
+}
+
+// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
+// This method does not return.
+func (menu *Menu) watchIPNBus() {
+	for {
+		if err := menu.watchIPNBusInner(); err != nil {
+			log.Println(err)
+			if errors.Is(err, context.Canceled) {
+				// If the context got canceled, we will never be able to
+				// reconnect to IPN bus, so exit the process.
+				log.Fatalf("watchIPNBus: %v", err)
+			}
+		}
+		// If our watch connection breaks, wait a bit before reconnecting. No
+		// reason to spam the logs if e.g. tailscaled is restarting or goes
+		// down.
+		time.Sleep(3 * time.Second)
+	}
+}
+
+func (menu *Menu) watchIPNBusInner() error {
+	watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return fmt.Errorf("watching ipn bus: %w", err)
+	}
+	defer watcher.Close()
+	for {
+		select {
+		case <-menu.bgCtx.Done():
+			return nil
+		default:
+			n, err := watcher.Next()
+			if err != nil {
+				return fmt.Errorf("ipnbus error: %w", err)
+			}
+			var rebuild bool
+			if n.State != nil {
+				log.Printf("new state: %v", n.State)
+				rebuild = true
+			}
+			if n.Prefs != nil {
+				rebuild = true
+			}
+			if rebuild {
+				menu.rebuildCh <- struct{}{}
+			}
+		}
+	}
+}
+
+// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
+// and sends a notification with the copied value.
+func (menu *Menu) copyTailscaleIP(device *ipnstate.PeerStatus) {
+	if device == nil || len(device.TailscaleIPs) == 0 {
+		return
+	}
+	name := strings.Split(device.DNSName, ".")[0]
+	ip := device.TailscaleIPs[0].String()
+	err := clipboard.WriteAll(ip)
+	if err != nil {
+		log.Printf("clipboard error: %v", err)
+	}
+
+	menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
+}
+
+// sendNotification sends a desktop notification with the given title and content.
+func (menu *Menu) sendNotification(title, content string) {
+	conn, err := dbus.SessionBus()
+	if err != nil {
+		log.Printf("dbus: %v", err)
+		return
+	}
+	timeout := 3 * time.Second
+	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
+	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
+		menu.notificationIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
+	if call.Err != nil {
+		log.Printf("dbus: %v", call.Err)
+	}
+}
+
+func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) {
+	if menu.status == nil {
+		return
+	}
+
+	status := menu.status
+	menu.exitNodes = systray.AddMenuItem("Exit Nodes", "")
+	time.Sleep(newMenuDelay)
+
+	// register a click handler for a menu item to set nodeID as the exit node.
+	setExitNodeOnClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) {
+		onClick(ctx, item, func(ctx context.Context) {
+			select {
+			case <-ctx.Done():
+			case menu.exitNodeCh <- nodeID:
+			}
+		})
+	}
+
+	noExitNodeMenu := menu.exitNodes.AddSubMenuItemCheckbox("None", "", status.ExitNodeStatus == nil)
+	setExitNodeOnClick(noExitNodeMenu, "")
+
+	// Show recommended exit node if available.
+	if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) {
+		sugg, err := menu.lc.SuggestExitNode(ctx)
+		if err == nil {
+			title := "Recommended: "
+			if loc := sugg.Location; loc.Valid() && loc.Country() != "" {
+				flag := countryFlag(loc.CountryCode())
+				title += fmt.Sprintf("%s %s: %s", flag, loc.Country(), loc.City())
+			} else {
+				title += strings.Split(sugg.Name, ".")[0]
+			}
+			menu.exitNodes.AddSeparator()
+			rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false)
+			setExitNodeOnClick(rm, sugg.ID)
+			if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID {
+				rm.Check()
+			}
+		}
+	}
+
+	// Add tailnet exit nodes if present.
+	var tailnetExitNodes []*ipnstate.PeerStatus
+	for _, ps := range status.Peer {
+		if ps.ExitNodeOption && ps.Location == nil {
+			tailnetExitNodes = append(tailnetExitNodes, ps)
+		}
+	}
+	if len(tailnetExitNodes) > 0 {
+		menu.exitNodes.AddSeparator()
+		menu.exitNodes.AddSubMenuItem("Tailnet Exit Nodes", "").Disable()
+		for _, ps := range status.Peer {
+			if !ps.ExitNodeOption || ps.Location != nil {
+				continue
+			}
+			name := strings.Split(ps.DNSName, ".")[0]
+			if !ps.Online {
+				name += " (offline)"
+			}
+			sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false)
+			if !ps.Online {
+				sm.Disable()
+			}
+			if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID {
+				sm.Check()
+			}
+			setExitNodeOnClick(sm, ps.ID)
+		}
+	}
+
+	// Add mullvad exit nodes if present.
+	var mullvadExitNodes mullvadPeers
+	if status.Self.CapMap.Contains("mullvad") {
+		mullvadExitNodes = newMullvadPeers(status)
+	}
+	if len(mullvadExitNodes.countries) > 0 {
+		menu.exitNodes.AddSeparator()
+		menu.exitNodes.AddSubMenuItem("Location-based Exit Nodes", "").Disable()
+		mullvadMenu := menu.exitNodes.AddSubMenuItemCheckbox("Mullvad VPN", "", false)
+
+		for _, country := range mullvadExitNodes.sortedCountries() {
+			flag := countryFlag(country.code)
+			countryMenu := mullvadMenu.AddSubMenuItemCheckbox(flag+" "+country.name, "", false)
+
+			// single-city country, no submenu
+			if len(country.cities) == 1 || hideMullvadCities {
+				setExitNodeOnClick(countryMenu, country.best.ID)
+				if status.ExitNodeStatus != nil {
+					for _, city := range country.cities {
+						for _, ps := range city.peers {
+							if status.ExitNodeStatus.ID == ps.ID {
+								mullvadMenu.Check()
+								countryMenu.Check()
+							}
+						}
+					}
+				}
+				continue
+			}
+
+			// multi-city country, build submenu with "best available" option and cities.
+			time.Sleep(newMenuDelay)
+			bm := countryMenu.AddSubMenuItemCheckbox("Best Available", "", false)
+			setExitNodeOnClick(bm, country.best.ID)
+			countryMenu.AddSeparator()
+
+			for _, city := range country.sortedCities() {
+				cityMenu := countryMenu.AddSubMenuItemCheckbox(city.name, "", false)
+				setExitNodeOnClick(cityMenu, city.best.ID)
+				if status.ExitNodeStatus != nil {
+					for _, ps := range city.peers {
+						if status.ExitNodeStatus.ID == ps.ID {
+							mullvadMenu.Check()
+							countryMenu.Check()
+							cityMenu.Check()
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// TODO: "Allow Local Network Access" and "Run Exit Node" menu items
+}
+
+// mullvadPeers contains all mullvad peer nodes, sorted by country and city.
+type mullvadPeers struct {
+	countries map[string]*mvCountry // country code (uppercase) => country
+}
+
+// sortedCountries returns countries containing mullvad nodes, sorted by name.
+func (mp mullvadPeers) sortedCountries() []*mvCountry {
+	countries := slicesx.MapValues(mp.countries)
+	slices.SortFunc(countries, func(a, b *mvCountry) int {
+		return stringsx.CompareFold(a.name, b.name)
+	})
+	return countries
+}
+
+type mvCountry struct {
+	code   string
+	name   string
+	best   *ipnstate.PeerStatus // highest priority peer in the country
+	cities map[string]*mvCity   // city code => city
+}
+
+// sortedCities returns cities containing mullvad nodes, sorted by name.
+func (mc *mvCountry) sortedCities() []*mvCity {
+	cities := slicesx.MapValues(mc.cities)
+	slices.SortFunc(cities, func(a, b *mvCity) int {
+		return stringsx.CompareFold(a.name, b.name)
+	})
+	return cities
+}
+
+// countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag.
+// It returns the empty string on error.
+func countryFlag(code string) string {
+	if len(code) != 2 {
+		return ""
+	}
+	runes := make([]rune, 0, 2)
+	for i := range 2 {
+		b := code[i] | 32 // lowercase
+		if b < 'a' || b > 'z' {
+			return ""
+		}
+		// https://en.wikipedia.org/wiki/Regional_indicator_symbol
+		runes = append(runes, 0x1F1E6+rune(b-'a'))
+	}
+	return string(runes)
+}
+
+type mvCity struct {
+	name  string
+	best  *ipnstate.PeerStatus // highest priority peer in the city
+	peers []*ipnstate.PeerStatus
+}
+
+func newMullvadPeers(status *ipnstate.Status) mullvadPeers {
+	countries := make(map[string]*mvCountry)
+	for _, ps := range status.Peer {
+		if !ps.ExitNodeOption || ps.Location == nil {
+			continue
+		}
+		loc := ps.Location
+		country, ok := countries[loc.CountryCode]
+		if !ok {
+			country = &mvCountry{
+				code:   loc.CountryCode,
+				name:   loc.Country,
+				cities: make(map[string]*mvCity),
+			}
+			countries[loc.CountryCode] = country
+		}
+		city, ok := countries[loc.CountryCode].cities[loc.CityCode]
+		if !ok {
+			city = &mvCity{
+				name: loc.City,
+			}
+			countries[loc.CountryCode].cities[loc.CityCode] = city
+		}
+		city.peers = append(city.peers, ps)
+		if city.best == nil || ps.Location.Priority > city.best.Location.Priority {
+			city.best = ps
+		}
+		if country.best == nil || ps.Location.Priority > country.best.Location.Priority {
+			country.best = ps
+		}
+	}
+	return mullvadPeers{countries}
+}
+
+// onExit is called by the systray package when the menu is exiting.
+func (menu *Menu) onExit() {
+	log.Printf("exiting")
+	if menu.bgCancel != nil {
+		menu.bgCancel()
+	}
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+
+	os.Remove(menu.notificationIcon.Name())
+}

client/tailscale/acl.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/netip"
+	"net/url"
 )
 
 // ACLRow defines a rule that grants access by a set of users or groups to a set
@@ -83,7 +84,7 @@ func (c *Client) ACL(ctx context.Context) (acl *ACL, err error) {
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl")
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -97,7 +98,7 @@ func (c *Client) ACL(ctx context.Context) (acl *ACL, err error) {
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	// Otherwise, try to decode the response.
@@ -126,7 +127,7 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl?details=1", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl", url.Values{"details": {"1"}})
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -138,7 +139,7 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	data := struct {
@@ -146,7 +147,7 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 		Warnings []string `json:"warnings"`
 	}{}
 	if err := json.Unmarshal(b, &data); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("json.Unmarshal %q: %w", b, err)
 	}
 
 	acl = &ACLHuJSON{
@@ -184,7 +185,7 @@ func (e ACLTestError) Error() string {
 }
 
 func (c *Client) aclPOSTRequest(ctx context.Context, body []byte, avoidCollisions bool, etag, acceptHeader string) ([]byte, string, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewBuffer(body))
 	if err != nil {
 		return nil, "", err
@@ -328,7 +329,7 @@ type ACLPreview struct {
 }
 
 func (c *Client) previewACLPostRequest(ctx context.Context, body []byte, previewType string, previewFor string) (res *ACLPreviewResponse, err error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl/preview", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl", "preview")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewBuffer(body))
 	if err != nil {
 		return nil, err
@@ -350,7 +351,7 @@ func (c *Client) previewACLPostRequest(ctx context.Context, body []byte, preview
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 	if err = json.Unmarshal(b, &res); err != nil {
 		return nil, err
@@ -488,7 +489,7 @@ func (c *Client) ValidateACLJSON(ctx context.Context, source, dest string) (test
 		return nil, err
 	}
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl/validate", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl", "validate")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewBuffer(postData))
 	if err != nil {
 		return nil, err

client/tailscale/apitype/apitype.go

@@ -7,11 +7,29 @@ package apitype
 import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/dnstype"
+	"tailscale.com/util/ctxkey"
 )
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
 
+// RequestReasonHeader is the header used to pass justification for a LocalAPI request,
+// such as when a user wants to perform an action they don't have permission for,
+// and a policy allows it with justification. As of 2025-01-29, it is only used to
+// allow a user to disconnect Tailscale when the "always-on" mode is enabled.
+//
+// The header value is base64-encoded using the standard encoding defined in RFC 4648.
+//
+// See tailscale/corp#26146.
+const RequestReasonHeader = "X-Tailscale-Reason"
+
+// RequestReasonKey is the context key used to pass the request reason
+// when making a LocalAPI request via [local.Client].
+// It's value is a raw string. An empty string means no reason was provided.
+//
+// See tailscale/corp#26146.
+var RequestReasonKey = ctxkey.New(RequestReasonHeader, "")
+
 // WhoIsResponse is the JSON type returned by tailscaled debug server's /whois?ip=$IP handler.
 // In successful whois responses, Node and UserProfile are never nil.
 type WhoIsResponse struct {

client/tailscale/devices.go

@@ -131,7 +131,7 @@ func (c *Client) Devices(ctx context.Context, fields *DeviceFieldsOpts) (deviceL
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/devices", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("devices")
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -149,7 +149,7 @@ func (c *Client) Devices(ctx context.Context, fields *DeviceFieldsOpts) (deviceL
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var devices GetDevicesResponse
@@ -188,7 +188,7 @@ func (c *Client) Device(ctx context.Context, deviceID string, fields *DeviceFiel
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	err = json.Unmarshal(b, &device)
@@ -221,7 +221,7 @@ func (c *Client) DeleteDevice(ctx context.Context, deviceID string) (err error)
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 	return nil
 }
@@ -253,7 +253,7 @@ func (c *Client) SetAuthorized(ctx context.Context, deviceID string, authorized
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 
 	return nil
@@ -281,7 +281,7 @@ func (c *Client) SetTags(ctx context.Context, deviceID string, tags []string) er
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 
 	return nil

client/tailscale/dns.go

@@ -44,7 +44,7 @@ type DNSPreferences struct {
 }
 
 func (c *Client) dnsGETRequest(ctx context.Context, endpoint string) ([]byte, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/dns/%s", c.baseURL(), c.tailnet, endpoint)
+	path := c.BuildTailnetURL("dns", endpoint)
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -57,14 +57,14 @@ func (c *Client) dnsGETRequest(ctx context.Context, endpoint string) ([]byte, er
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	return b, nil
 }
 
 func (c *Client) dnsPOSTRequest(ctx context.Context, endpoint string, postData any) ([]byte, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/dns/%s", c.baseURL(), c.tailnet, endpoint)
+	path := c.BuildTailnetURL("dns", endpoint)
 	data, err := json.Marshal(&postData)
 	if err != nil {
 		return nil, err
@@ -84,7 +84,7 @@ func (c *Client) dnsPOSTRequest(ctx context.Context, endpoint string, postData a
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	return b, nil

client/tailscale/example/servetls/servetls.go

@@ -11,13 +11,14 @@ import (
 	"log"
 	"net/http"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 )
 
 func main() {
+	var lc local.Client
 	s := &http.Server{
 		TLSConfig: &tls.Config{
-			GetCertificate: tailscale.GetCertificate,
+			GetCertificate: lc.GetCertificate,
 		},
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			io.WriteString(w, "<h1>Hello from Tailscale!</h1> It works.")

client/tailscale/keys.go

@@ -40,7 +40,7 @@ type KeyDeviceCreateCapabilities struct {
 
 // Keys returns the list of keys for the current user.
 func (c *Client) Keys(ctx context.Context) ([]string, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("keys")
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -51,7 +51,7 @@ func (c *Client) Keys(ctx context.Context) ([]string, error) {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var keys struct {
@@ -99,7 +99,7 @@ func (c *Client) CreateKeyWithExpiry(ctx context.Context, caps KeyCapabilities,
 		return "", nil, err
 	}
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("keys")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewReader(bs))
 	if err != nil {
 		return "", nil, err
@@ -110,7 +110,7 @@ func (c *Client) CreateKeyWithExpiry(ctx context.Context, caps KeyCapabilities,
 		return "", nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return "", nil, handleErrorResponse(b, resp)
+		return "", nil, HandleErrorResponse(b, resp)
 	}
 
 	var key struct {
@@ -126,7 +126,7 @@ func (c *Client) CreateKeyWithExpiry(ctx context.Context, caps KeyCapabilities,
 // Key returns the metadata for the given key ID. Currently, capabilities are
 // only returned for auth keys, API keys only return general metadata.
 func (c *Client) Key(ctx context.Context, id string) (*Key, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys/%s", c.baseURL(), c.tailnet, id)
+	path := c.BuildTailnetURL("keys", id)
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -137,7 +137,7 @@ func (c *Client) Key(ctx context.Context, id string) (*Key, error) {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var key Key
@@ -149,7 +149,7 @@ func (c *Client) Key(ctx context.Context, id string) (*Key, error) {
 
 // DeleteKey deletes the key with the given ID.
 func (c *Client) DeleteKey(ctx context.Context, id string) error {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys/%s", c.baseURL(), c.tailnet, id)
+	path := c.BuildTailnetURL("keys", id)
 	req, err := http.NewRequestWithContext(ctx, "DELETE", path, nil)
 	if err != nil {
 		return err
@@ -160,7 +160,7 @@ func (c *Client) DeleteKey(ctx context.Context, id string) error {
 		return err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 	return nil
 }

client/tailscale/localclient_aliases.go

@@ -0,0 +1,106 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package tailscale
+
+import (
+	"context"
+	"crypto/tls"
+
+	"tailscale.com/client/local"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/ipn/ipnstate"
+)
+
+// ErrPeerNotFound is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+var ErrPeerNotFound = local.ErrPeerNotFound
+
+// LocalClient is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type LocalClient = local.Client
+
+// IPNBusWatcher is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type IPNBusWatcher = local.IPNBusWatcher
+
+// BugReportOpts is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type BugReportOpts = local.BugReportOpts
+
+// DebugPortMapOpts is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type DebugPortmapOpts = local.DebugPortmapOpts
+
+// PingOpts is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type PingOpts = local.PingOpts
+
+// GetCertificate is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return local.GetCertificate(hi)
+}
+
+// SetVersionMismatchHandler is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func SetVersionMismatchHandler(f func(clientVer, serverVer string)) {
+	local.SetVersionMismatchHandler(f)
+}
+
+// IsAccessDeniedError is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func IsAccessDeniedError(err error) bool {
+	return local.IsAccessDeniedError(err)
+}
+
+// IsPreconditionsFailedError is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func IsPreconditionsFailedError(err error) bool {
+	return local.IsPreconditionsFailedError(err)
+}
+
+// WhoIs is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
+	return local.WhoIs(ctx, remoteAddr)
+}
+
+// Status is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func Status(ctx context.Context) (*ipnstate.Status, error) {
+	return local.Status(ctx)
+}
+
+// StatusWithoutPeers is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
+	return local.StatusWithoutPeers(ctx)
+}
+
+// CertPair is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+	return local.CertPair(ctx, domain)
+}
+
+// ExpandSNIName is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
+	return local.ExpandSNIName(ctx, name)
+}

client/tailscale/routes.go

@@ -44,7 +44,7 @@ func (c *Client) Routes(ctx context.Context, deviceID string) (routes *Routes, e
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var sr Routes
@@ -84,7 +84,7 @@ func (c *Client) SetRoutes(ctx context.Context, deviceID string, subnets []netip
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var srr *Routes

client/tailscale/tailnet.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"net/url"
 
 	"tailscale.com/util/httpm"
 )
@@ -22,7 +21,7 @@ func (c *Client) TailnetDeleteRequest(ctx context.Context, tailnetID string) (er
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s", c.baseURL(), url.PathEscape(string(tailnetID)))
+	path := c.BuildTailnetURL("tailnet")
 	req, err := http.NewRequestWithContext(ctx, httpm.DELETE, path, nil)
 	if err != nil {
 		return err
@@ -35,7 +34,7 @@ func (c *Client) TailnetDeleteRequest(ctx context.Context, tailnetID string) (er
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 
 	return nil

client/tailscale/tailscale.go

@@ -3,11 +3,12 @@
 
 //go:build go1.19
 
-// Package tailscale contains Go clients for the Tailscale LocalAPI and
-// Tailscale control plane API.
+// Package tailscale contains a Go client for the Tailscale control plane API.
 //
-// Warning: this package is in development and makes no API compatibility
-// promises as of 2022-04-29. It is subject to change at any time.
+// This package is only intended for internal and transitional use.
+//
+// Deprecated: the official control plane client is available at
+// tailscale.com/client/tailscale/v2.
 package tailscale
 
 import (
@@ -16,13 +17,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
+	"path"
 )
 
 // I_Acknowledge_This_API_Is_Unstable must be set true to use this package
-// for now. It was added 2022-04-29 when it was moved to this git repo
-// and will be removed when the public API has settled.
-//
-// TODO(bradfitz): remove this after the we're happy with the public API.
+// for now. This package is being replaced by tailscale.com/client/tailscale/v2.
 var I_Acknowledge_This_API_Is_Unstable = false
 
 // TODO: use url.PathEscape() for deviceID and tailnets when constructing requests.
@@ -36,6 +36,8 @@ const maxReadSize = 10 << 20
 //
 // Use NewClient to instantiate one. Exported fields should be set before
 // the client is used and not changed thereafter.
+//
+// Deprecated: use tailscale.com/client/tailscale/v2 instead.
 type Client struct {
 	// tailnet is the globally unique identifier for a Tailscale network, such
 	// as "example.com" or "user@gmail.com".
@@ -63,6 +65,46 @@ func (c *Client) httpClient() *http.Client {
 	return http.DefaultClient
 }
 
+// BuildURL builds a url to http(s)://<apiserver>/api/v2/<slash-separated-pathElements>
+// using the given pathElements. It url escapes each path element, so the
+// caller doesn't need to worry about that. The last item of pathElements can
+// be of type url.Values to add a query string to the URL.
+//
+// For example, BuildURL(devices, 5) with the default server URL would result in
+// https://api.tailscale.com/api/v2/devices/5.
+func (c *Client) BuildURL(pathElements ...any) string {
+	elem := make([]string, 1, len(pathElements)+1)
+	elem[0] = "/api/v2"
+	var query string
+	for i, pathElement := range pathElements {
+		if uv, ok := pathElement.(url.Values); ok && i == len(pathElements)-1 {
+			query = uv.Encode()
+		} else {
+			elem = append(elem, url.PathEscape(fmt.Sprint(pathElement)))
+		}
+	}
+	url := c.baseURL() + path.Join(elem...)
+	if query != "" {
+		url += "?" + query
+	}
+	return url
+}
+
+// BuildTailnetURL builds a url to http(s)://<apiserver>/api/v2/tailnet/<tailnet>/<slash-separated-pathElements>
+// using the given pathElements. It url escapes each path element, so the
+// caller doesn't need to worry about that. The last item of pathElements can
+// be of type url.Values to add a query string to the URL.
+//
+// For example, BuildTailnetURL(policy, validate) with the default server URL and a tailnet of "example.com"
+// would result in https://api.tailscale.com/api/v2/tailnet/example.com/policy/validate.
+func (c *Client) BuildTailnetURL(pathElements ...any) string {
+	allElements := make([]any, 2, len(pathElements)+2)
+	allElements[0] = "tailnet"
+	allElements[1] = c.tailnet
+	allElements = append(allElements, pathElements...)
+	return c.BuildURL(allElements...)
+}
+
 func (c *Client) baseURL() string {
 	if c.BaseURL != "" {
 		return c.BaseURL
@@ -98,6 +140,8 @@ func (c *Client) setAuth(r *http.Request) {
 // If httpClient is nil, then http.DefaultClient is used.
 // "api.tailscale.com" is set as the BaseURL for the returned client
 // and can be changed manually by the user.
+//
+// Deprecated: use tailscale.com/client/tailscale/v2 instead.
 func NewClient(tailnet string, auth AuthMethod) *Client {
 	return &Client{
 		tailnet:   tailnet,
@@ -148,12 +192,14 @@ func (e ErrResponse) Error() string {
 	return fmt.Sprintf("Status: %d, Message: %q", e.Status, e.Message)
 }
 
-// handleErrorResponse decodes the error message from the server and returns
+// HandleErrorResponse decodes the error message from the server and returns
 // an ErrResponse from it.
-func handleErrorResponse(b []byte, resp *http.Response) error {
+//
+// Deprecated: use tailscale.com/client/tailscale/v2 instead.
+func HandleErrorResponse(b []byte, resp *http.Response) error {
 	var errResp ErrResponse
 	if err := json.Unmarshal(b, &errResp); err != nil {
-		return err
+		return fmt.Errorf("json.Unmarshal %q: %w", b, err)
 	}
 	errResp.Status = resp.StatusCode
 	return errResp

client/tailscale/tailscale_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package tailscale
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestClientBuildURL(t *testing.T) {
+	c := Client{BaseURL: "http://127.0.0.1:1234"}
+	for _, tt := range []struct {
+		desc     string
+		elements []any
+		want     string
+	}{
+		{
+			desc:     "single-element",
+			elements: []any{"devices"},
+			want:     "http://127.0.0.1:1234/api/v2/devices",
+		},
+		{
+			desc:     "multiple-elements",
+			elements: []any{"tailnet", "example.com"},
+			want:     "http://127.0.0.1:1234/api/v2/tailnet/example.com",
+		},
+		{
+			desc:     "escape-element",
+			elements: []any{"tailnet", "example dot com?foo=bar"},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example%20dot%20com%3Ffoo=bar`,
+		},
+		{
+			desc:     "url.Values",
+			elements: []any{"tailnet", "example.com", "acl", url.Values{"details": {"1"}}},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example.com/acl?details=1`,
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			got := c.BuildURL(tt.elements...)
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestClientBuildTailnetURL(t *testing.T) {
+	c := Client{
+		BaseURL: "http://127.0.0.1:1234",
+		tailnet: "example.com",
+	}
+	for _, tt := range []struct {
+		desc     string
+		elements []any
+		want     string
+	}{
+		{
+			desc:     "single-element",
+			elements: []any{"devices"},
+			want:     "http://127.0.0.1:1234/api/v2/tailnet/example.com/devices",
+		},
+		{
+			desc:     "multiple-elements",
+			elements: []any{"devices", 123},
+			want:     "http://127.0.0.1:1234/api/v2/tailnet/example.com/devices/123",
+		},
+		{
+			desc:     "escape-element",
+			elements: []any{"foo bar?baz=qux"},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example.com/foo%20bar%3Fbaz=qux`,
+		},
+		{
+			desc:     "url.Values",
+			elements: []any{"acl", url.Values{"details": {"1"}}},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example.com/acl?details=1`,
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			got := c.BuildTailnetURL(tt.elements...)
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

client/web/src/components/views/login-view.tsx

@@ -1,13 +1,11 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-import React, { useState } from "react"
+import React from "react"
 import { useAPI } from "src/api"
 import TailscaleIcon from "src/assets/icons/tailscale-icon.svg?react"
 import { NodeData } from "src/types"
 import Button from "src/ui/button"
-import Collapsible from "src/ui/collapsible"
-import Input from "src/ui/input"
 
 /**
  * LoginView is rendered when the client is not authenticated
@@ -15,8 +13,6 @@ import Input from "src/ui/input"
  */
 export default function LoginView({ data }: { data: NodeData }) {
   const api = useAPI()
-  const [controlURL, setControlURL] = useState<string>("")
-  const [authKey, setAuthKey] = useState<string>("")
 
   return (
     <div className="mb-8 py-6 px-8 bg-white rounded-md shadow-2xl">
@@ -88,8 +84,6 @@ export default function LoginView({ data }: { data: NodeData }) {
                 action: "up",
                 data: {
                   Reauthenticate: true,
-                  ControlURL: controlURL,
-                  AuthKey: authKey,
                 },
               })
             }
@@ -98,34 +92,6 @@ export default function LoginView({ data }: { data: NodeData }) {
           >
             Log In
           </Button>
-          <Collapsible trigger="Advanced options">
-            <h4 className="font-medium mb-1 mt-2">Auth Key</h4>
-            <p className="text-sm text-gray-500">
-              Connect with a pre-authenticated key.{" "}
-              <a
-                href="https://tailscale.com/kb/1085/auth-keys/"
-                className="link"
-                target="_blank"
-                rel="noreferrer"
-              >
-                Learn more &rarr;
-              </a>
-            </p>
-            <Input
-              className="mt-2"
-              value={authKey}
-              onChange={(e) => setAuthKey(e.target.value)}
-              placeholder="tskey-auth-XXX"
-            />
-            <h4 className="font-medium mt-3 mb-1">Server URL</h4>
-            <p className="text-sm text-gray-500">Base URL of control server.</p>
-            <Input
-              className="mt-2"
-              value={controlURL}
-              onChange={(e) => setControlURL(e.target.value)}
-              placeholder="https://login.tailscale.com/"
-            />
-          </Collapsible>
         </>
       )}
     </div>

client/web/web.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/gorilla/csrf"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
@@ -50,7 +50,7 @@ type Server struct {
 	mode ServerMode
 
 	logf    logger.Logf
-	lc      *tailscale.LocalClient
+	lc      *local.Client
 	timeNow func() time.Time
 
 	// devMode indicates that the server run with frontend assets
@@ -89,8 +89,8 @@ type Server struct {
 type ServerMode string
 
 const (
-	// LoginServerMode serves a readonly login client for logging a
-	// node into a tailnet, and viewing a readonly interface of the
+	// LoginServerMode serves a read-only login client for logging a
+	// node into a tailnet, and viewing a read-only interface of the
 	// node's current Tailscale settings.
 	//
 	// In this mode, API calls are authenticated via platform auth.
@@ -110,7 +110,7 @@ const (
 	// This mode restricts the app to only being assessible over Tailscale,
 	// and API calls are authenticated via browser sessions associated with
 	// the source's Tailscale identity. If the source browser does not have
-	// a valid session, a readonly version of the app is displayed.
+	// a valid session, a read-only version of the app is displayed.
 	ManageServerMode ServerMode = "manage"
 )
 
@@ -125,9 +125,9 @@ type ServerOpts struct {
 	// PathPrefix is the URL prefix added to requests by CGI or reverse proxy.
 	PathPrefix string
 
-	// LocalClient is the tailscale.LocalClient to use for this web server.
+	// LocalClient is the local.Client to use for this web server.
 	// If nil, a new one will be created.
-	LocalClient *tailscale.LocalClient
+	LocalClient *local.Client
 
 	// TimeNow optionally provides a time function.
 	// time.Now is used as default.
@@ -166,7 +166,7 @@ func NewServer(opts ServerOpts) (s *Server, err error) {
 		return nil, fmt.Errorf("invalid Mode provided")
 	}
 	if opts.LocalClient == nil {
-		opts.LocalClient = &tailscale.LocalClient{}
+		opts.LocalClient = &local.Client{}
 	}
 	s = &Server{
 		mode:        opts.Mode,
@@ -203,25 +203,9 @@ func NewServer(opts ServerOpts) (s *Server, err error) {
 	}
 	s.assetsHandler, s.assetsCleanup = assetsHandler(s.devMode)
 
-	var metric string // clientmetric to report on startup
-
-	// Create handler for "/api" requests with CSRF protection.
-	// We don't require secure cookies, since the web client is regularly used
-	// on network appliances that are served on local non-https URLs.
-	// The client is secured by limiting the interface it listens on,
-	// or by authenticating requests before they reach the web client.
-	csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false))
-	switch s.mode {
-	case LoginServerMode:
-		s.apiHandler = csrfProtect(http.HandlerFunc(s.serveLoginAPI))
-		metric = "web_login_client_initialization"
-	case ReadOnlyServerMode:
-		s.apiHandler = csrfProtect(http.HandlerFunc(s.serveLoginAPI))
-		metric = "web_readonly_client_initialization"
-	case ManageServerMode:
-		s.apiHandler = csrfProtect(http.HandlerFunc(s.serveAPI))
-		metric = "web_client_initialization"
-	}
+	var metric string
+	s.apiHandler, metric = s.modeAPIHandler(s.mode)
+	s.apiHandler = s.withCSRF(s.apiHandler)
 
 	// Don't block startup on reporting metric.
 	// Report in separate go routine with 5 second timeout.
@@ -234,6 +218,39 @@ func NewServer(opts ServerOpts) (s *Server, err error) {
 	return s, nil
 }
 
+func (s *Server) withCSRF(h http.Handler) http.Handler {
+	csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false))
+
+	// ref https://github.com/tailscale/tailscale/pull/14822
+	// signal to the CSRF middleware that the request is being served over
+	// plaintext HTTP to skip TLS-only header checks.
+	withSetPlaintext := func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			r = csrf.PlaintextHTTPRequest(r)
+			h.ServeHTTP(w, r)
+		})
+	}
+
+	// NB: the order of the withSetPlaintext and csrfProtect calls is important
+	// to ensure that we signal to the CSRF middleware that the request is being
+	// served over plaintext HTTP and not over TLS as it presumes by default.
+	return withSetPlaintext(csrfProtect(h))
+}
+
+func (s *Server) modeAPIHandler(mode ServerMode) (http.Handler, string) {
+	switch mode {
+	case LoginServerMode:
+		return http.HandlerFunc(s.serveLoginAPI), "web_login_client_initialization"
+	case ReadOnlyServerMode:
+		return http.HandlerFunc(s.serveLoginAPI), "web_readonly_client_initialization"
+	case ManageServerMode:
+		return http.HandlerFunc(s.serveAPI), "web_client_initialization"
+	default: // invalid mode
+		log.Fatalf("invalid mode: %v", mode)
+	}
+	return nil, ""
+}
+
 func (s *Server) Shutdown() {
 	s.logf("web.Server: shutting down")
 	if s.assetsCleanup != nil {
@@ -695,16 +712,16 @@ func (s *Server) serveAPIAuth(w http.ResponseWriter, r *http.Request) {
 	switch {
 	case sErr != nil && errors.Is(sErr, errNotUsingTailscale):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_local", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && errors.Is(sErr, errNotOwner):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_not_owner", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && errors.Is(sErr, errTaggedLocalSource):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_local_tag", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && errors.Is(sErr, errTaggedRemoteSource):
 		s.lc.IncrementCounter(r.Context(), "web_client_viewing_remote_tag", 1)
-		resp.Authorized = false // restricted to the readonly view
+		resp.Authorized = false // restricted to the read-only view
 	case sErr != nil && !errors.Is(sErr, errNoSession):
 		// Any other error.
 		http.Error(w, sErr.Error(), http.StatusInternalServerError)
@@ -804,8 +821,8 @@ type nodeData struct {
 	DeviceName  string
 	TailnetName string // TLS cert name
 	DomainName  string
-	IPv4        string
-	IPv6        string
+	IPv4        netip.Addr
+	IPv6        netip.Addr
 	OS          string
 	IPNVersion  string
 
@@ -864,10 +881,14 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	filterRules, _ := s.lc.DebugPacketFilterRules(r.Context())
+	ipv4, ipv6 := s.selfNodeAddresses(r, st)
+
 	data := &nodeData{
 		ID:               st.Self.ID,
 		Status:           st.BackendState,
 		DeviceName:       strings.Split(st.Self.DNSName, ".")[0],
+		IPv4:             ipv4,
+		IPv6:             ipv6,
 		OS:               st.Self.OS,
 		IPNVersion:       strings.Split(st.Version, "-")[0],
 		Profile:          st.User[st.Self.UserID],
@@ -887,10 +908,6 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 		ACLAllowsAnyIncomingTraffic: s.aclsAllowAccess(filterRules),
 	}
 
-	ipv4, ipv6 := s.selfNodeAddresses(r, st)
-	data.IPv4 = ipv4.String()
-	data.IPv6 = ipv6.String()
-
 	if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn && data.URLPrefix == "" {
 		// X-Ingress-Path is the path prefix in use for Home Assistant
 		// https://developers.home-assistant.io/docs/add-ons/presentation#ingress

client/web/web_test.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/http/cookiejar"
 	"net/http/httptest"
 	"net/netip"
 	"net/url"
@@ -20,7 +21,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/client/tailscale"
+	"github.com/gorilla/csrf"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -120,7 +122,7 @@ func TestServeAPI(t *testing.T) {
 
 	s := &Server{
 		mode:    ManageServerMode,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 		timeNow: time.Now,
 	}
 
@@ -288,7 +290,7 @@ func TestGetTailscaleBrowserSession(t *testing.T) {
 
 	s := &Server{
 		timeNow: time.Now,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 	}
 
 	// Add some browser sessions to cache state.
@@ -457,7 +459,7 @@ func TestAuthorizeRequest(t *testing.T) {
 
 	s := &Server{
 		mode:    ManageServerMode,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 		timeNow: time.Now,
 	}
 	validCookie := "ts-cookie"
@@ -572,7 +574,7 @@ func TestServeAuth(t *testing.T) {
 
 	s := &Server{
 		mode:        ManageServerMode,
-		lc:          &tailscale.LocalClient{Dial: lal.Dial},
+		lc:          &local.Client{Dial: lal.Dial},
 		timeNow:     func() time.Time { return timeNow },
 		newAuthURL:  mockNewAuthURL,
 		waitAuthURL: mockWaitAuthURL,
@@ -914,7 +916,7 @@ func TestServeAPIAuthMetricLogging(t *testing.T) {
 
 	s := &Server{
 		mode:        ManageServerMode,
-		lc:          &tailscale.LocalClient{Dial: lal.Dial},
+		lc:          &local.Client{Dial: lal.Dial},
 		timeNow:     func() time.Time { return timeNow },
 		newAuthURL:  mockNewAuthURL,
 		waitAuthURL: mockWaitAuthURL,
@@ -1126,7 +1128,7 @@ func TestRequireTailscaleIP(t *testing.T) {
 
 	s := &Server{
 		mode:    ManageServerMode,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 		timeNow: time.Now,
 		logf:    t.Logf,
 	}
@@ -1477,3 +1479,83 @@ func mockWaitAuthURL(_ context.Context, id string, src tailcfg.NodeID) (*tailcfg
 		return nil, errors.New("unknown id")
 	}
 }
+
+func TestCSRFProtect(t *testing.T) {
+	s := &Server{}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /test/csrf-token", func(w http.ResponseWriter, r *http.Request) {
+		token := csrf.Token(r)
+		_, err := io.WriteString(w, token)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	mux.HandleFunc("POST /test/csrf-protected", func(w http.ResponseWriter, r *http.Request) {
+		_, err := io.WriteString(w, "ok")
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	h := s.withCSRF(mux)
+	ser := httptest.NewServer(h)
+	defer ser.Close()
+
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatalf("unable to construct cookie jar: %v", err)
+	}
+
+	client := ser.Client()
+	client.Jar = jar
+
+	// make GET request to populate cookie jar
+	resp, err := client.Get(ser.URL + "/test/csrf-token")
+	if err != nil {
+		t.Fatalf("unable to make request: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v", resp.Status)
+	}
+	tokenBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unable to read body: %v", err)
+	}
+
+	csrfToken := strings.TrimSpace(string(tokenBytes))
+	if csrfToken == "" {
+		t.Fatal("empty csrf token")
+	}
+
+	// make a POST request without the CSRF header; ensure it fails
+	resp, err = client.Post(ser.URL+"/test/csrf-protected", "text/plain", nil)
+	if err != nil {
+		t.Fatalf("unable to make request: %v", err)
+	}
+	if resp.StatusCode != http.StatusForbidden {
+		t.Fatalf("unexpected status: %v", resp.Status)
+	}
+
+	// make a POST request with the CSRF header; ensure it succeeds
+	req, err := http.NewRequest("POST", ser.URL+"/test/csrf-protected", nil)
+	if err != nil {
+		t.Fatalf("error building request: %v", err)
+	}
+	req.Header.Set("X-CSRF-Token", csrfToken)
+	resp, err = client.Do(req)
+	if err != nil {
+		t.Fatalf("unable to make request: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v", resp.Status)
+	}
+	defer resp.Body.Close()
+	out, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unable to read body: %v", err)
+	}
+	if string(out) != "ok" {
+		t.Fatalf("unexpected body: %q", out)
+	}
+}

clientupdate/clientupdate.go

@@ -27,6 +27,7 @@ import (
 	"strconv"
 	"strings"
 
+	"tailscale.com/hostinfo"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 	"tailscale.com/version"
@@ -169,6 +170,12 @@ func NewUpdater(args Arguments) (*Updater, error) {
 type updateFunction func() error
 
 func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
+	hi := hostinfo.New()
+	// We don't know how to update custom tsnet binaries, it's up to the user.
+	if hi.Package == "tsnet" {
+		return nil, false
+	}
+
 	switch runtime.GOOS {
 	case "windows":
 		return up.updateWindows, true

cmd/addlicense/main.go

@@ -18,12 +18,12 @@ var (
 )
 
 func usage() {
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 usage: addlicense -file FILE <subcommand args...>
 `[1:])
 
 	flag.PrintDefaults()
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 addlicense adds a Tailscale license to the beginning of file.
 
 It is intended for use with 'go generate', so it also runs a subcommand,

cmd/containerboot/certs.go

@@ -0,0 +1,147 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/util/goroutines"
+	"tailscale.com/util/mak"
+)
+
+// certManager is responsible for issuing certificates for known domains and for
+// maintaining a loop that re-attempts issuance daily.
+// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for
+// HA Ingress HTTPS endpoints ('write' replicas).
+type certManager struct {
+	lc      localClient
+	tracker goroutines.Tracker // tracks running goroutines
+	mu      sync.Mutex         // guards the following
+	// certLoops contains a map of DNS names, for which we currently need to
+	// manage certs to cancel functions that allow stopping a goroutine when
+	// we no longer need to manage certs for the DNS name.
+	certLoops map[string]context.CancelFunc
+}
+
+// ensureCertLoops ensures that, for all currently managed Service HTTPS
+// endpoints, there is a cert loop responsible for issuing and ensuring the
+// renewal of the TLS certs.
+// ServeConfig must not be nil.
+func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error {
+	if sc == nil {
+		return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
+	}
+	currentDomains := make(map[string]bool)
+	const httpsPort = "443"
+	for _, service := range sc.Services {
+		for hostPort := range service.Web {
+			domain, port, err := net.SplitHostPort(string(hostPort))
+			if err != nil {
+				return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
+			}
+			if port != httpsPort { // HA Ingress' HTTP endpoint
+				continue
+			}
+			currentDomains[domain] = true
+		}
+	}
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	for domain := range currentDomains {
+		if _, exists := cm.certLoops[domain]; !exists {
+			cancelCtx, cancel := context.WithCancel(ctx)
+			mak.Set(&cm.certLoops, domain, cancel)
+			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
+		}
+	}
+
+	// Stop goroutines for domain names that are no longer in the config.
+	for domain, cancel := range cm.certLoops {
+		if !currentDomains[domain] {
+			cancel()
+			delete(cm.certLoops, domain)
+		}
+	}
+	return nil
+}
+
+// runCertLoop:
+// - calls localAPI certificate endpoint to ensure that certs are issued for the
+// given domain name
+// - calls localAPI certificate endpoint daily to ensure that certs are renewed
+// - if certificate issuance failed retries after an exponential backoff period
+// starting at 1 minute and capped at 24 hours. Reset the backoff once issuance succeeds.
+// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get
+// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write'
+// replica does not get any HTTPS requests.
+// https://letsencrypt.org/docs/integration-guide/#retrying-failures
+func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
+	const (
+		normalInterval   = 24 * time.Hour  // regular renewal check
+		initialRetry     = 1 * time.Minute // initial backoff after a failure
+		maxRetryInterval = 24 * time.Hour  // max backoff period
+	)
+	timer := time.NewTimer(0) // fire off timer immediately
+	defer timer.Stop()
+	retryCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			// We call the certificate endpoint, but don't do anything
+			// with the returned certs here.
+			// The call to the certificate endpoint will ensure that
+			// certs are issued/renewed as needed and stored in the
+			// relevant state store. For example, for HA Ingress
+			// 'write' replica, the cert and key will be stored in a
+			// Kubernetes Secret named after the domain for which we
+			// are issuing.
+			// Note that renewals triggered by the call to the
+			// certificates endpoint here and by renewal check
+			// triggered during a call to node's HTTPS endpoint
+			// share the same state/renewal lock mechanism, so we
+			// should not run into redundant issuances during
+			// concurrent renewal checks.
+			// TODO(irbekrm): maybe it is worth adding a new
+			// issuance endpoint that explicitly only triggers
+			// issuance and stores certs in the relevant store, but
+			// does not return certs to the caller?
+			_, _, err := cm.lc.CertPair(ctx, domain)
+			if err != nil {
+				log.Printf("error refreshing certificate for %s: %v", domain, err)
+			}
+			var nextInterval time.Duration
+			// TODO(irbekrm): distinguish between LE rate limit
+			// errors and other error types like transient network
+			// errors.
+			if err == nil {
+				retryCount = 0
+				nextInterval = normalInterval
+			} else {
+				retryCount++
+				// Calculate backoff: initialRetry * 2^(retryCount-1)
+				// For retryCount=1: 1min * 2^0 = 1min
+				// For retryCount=2: 1min * 2^1 = 2min
+				// For retryCount=3: 1min * 2^2 = 4min
+				backoff := initialRetry * time.Duration(1<<(retryCount-1))
+				if backoff > maxRetryInterval {
+					backoff = maxRetryInterval
+				}
+				nextInterval = backoff
+				log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n",
+					domain, retryCount, err, nextInterval)
+			}
+			timer.Reset(nextInterval)
+		}
+	}
+}

cmd/containerboot/certs_test.go

@@ -0,0 +1,229 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+)
+
+// TestEnsureCertLoops tests that the certManager correctly starts and stops
+// update loops for certs when the serve config changes. It tracks goroutine
+// count and uses that as a validator that the expected number of cert loops are
+// running.
+func TestEnsureCertLoops(t *testing.T) {
+	tests := []struct {
+		name              string
+		initialConfig     *ipn.ServeConfig
+		updatedConfig     *ipn.ServeConfig
+		initialGoroutines int64 // after initial serve config is applied
+		updatedGoroutines int64 // after updated serve config is applied
+		wantErr           bool
+	}{
+		{
+			name:              "empty_serve_config",
+			initialConfig:     &ipn.ServeConfig{},
+			initialGoroutines: 0,
+		},
+		{
+			name:              "nil_serve_config",
+			initialConfig:     nil,
+			initialGoroutines: 0,
+			wantErr:           true,
+		},
+		{
+			name:          "empty_to_one_service",
+			initialConfig: &ipn.ServeConfig{},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 0,
+			updatedGoroutines: 1,
+		},
+		{
+			name: "single_service",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+		},
+		{
+			name: "multiple_services",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // one loop per domain across all services
+		},
+		{
+			name: "ignore_non_https_ports",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+							"my-app.tailnetxyz.ts.net:80":  {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1, // only one loop for the 443 endpoint
+		},
+		{
+			name: "remove_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // initially two loops (one per service)
+			updatedGoroutines: 1, // one loop after removing service2
+		},
+		{
+			name: "add_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+			updatedGoroutines: 2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			cm := &certManager{
+				lc:        &fakeLocalClient{},
+				certLoops: make(map[string]context.CancelFunc),
+			}
+
+			allDone := make(chan bool, 1)
+			defer cm.tracker.AddDoneCallback(func() {
+				cm.mu.Lock()
+				defer cm.mu.Unlock()
+				if cm.tracker.RunningGoroutines() > 0 {
+					return
+				}
+				select {
+				case allDone <- true:
+				default:
+				}
+			})()
+
+			err := cm.ensureCertLoops(ctx, tt.initialConfig)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ensureCertLoops() error = %v", err)
+			}
+
+			if got := cm.tracker.RunningGoroutines(); got != tt.initialGoroutines {
+				t.Errorf("after initial config: got %d running goroutines, want %d", got, tt.initialGoroutines)
+			}
+
+			if tt.updatedConfig != nil {
+				if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil {
+					t.Fatalf("ensureCertLoops() error on update = %v", err)
+				}
+
+				// Although starting goroutines and cancelling
+				// the context happens in the main goroutine, it
+				// the actual goroutine exit when a context is
+				// cancelled does not- so wait for a bit for the
+				// running goroutine count to reach the expected
+				// number.
+				deadline := time.After(5 * time.Second)
+				for {
+					if got := cm.tracker.RunningGoroutines(); got == tt.updatedGoroutines {
+						break
+					}
+					select {
+					case <-deadline:
+						t.Fatalf("timed out waiting for goroutine count to reach %d, currently at %d",
+							tt.updatedGoroutines, cm.tracker.RunningGoroutines())
+					case <-time.After(10 * time.Millisecond):
+						continue
+					}
+				}
+			}
+
+			if tt.updatedGoroutines == 0 {
+				return // no goroutines to wait for
+			}
+			// cancel context to make goroutines exit
+			cancel()
+			select {
+			case <-time.After(5 * time.Second):
+				t.Fatal("timed out waiting for goroutine to finish")
+			case <-allDone:
+			}
+		})
+	}
+}

cmd/containerboot/healthz.go

@@ -6,9 +6,12 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"net/http"
 	"sync"
+
+	"tailscale.com/kube/kubetypes"
 )
 
 // healthz is a simple health check server, if enabled it returns 200 OK if
@@ -17,6 +20,7 @@ import (
 type healthz struct {
 	sync.Mutex
 	hasAddrs bool
+	podIPv4  string
 }
 
 func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -24,7 +28,10 @@ func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer h.Unlock()
 
 	if h.hasAddrs {
-		w.Write([]byte("ok"))
+		w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4)
+		if _, err := w.Write([]byte("ok")); err != nil {
+			http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError)
+		}
 	} else {
 		http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable)
 	}
@@ -43,8 +50,8 @@ func (h *healthz) update(healthy bool) {
 // healthHandlers registers a simple health handler at /healthz.
 // A containerized tailscale instance is considered healthy if
 // it has at least one tailnet IP address.
-func healthHandlers(mux *http.ServeMux) *healthz {
-	h := &healthz{}
+func healthHandlers(mux *http.ServeMux, podIPv4 string) *healthz {
+	h := &healthz{podIPv4: podIPv4}
 	mux.Handle("GET /healthz", h)
 	return h
 }

cmd/containerboot/kube.go

@@ -8,15 +8,22 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"log"
 	"net/http"
 	"net/netip"
 	"os"
+	"strings"
+	"time"
 
+	"tailscale.com/ipn"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
 	"tailscale.com/kube/kubetypes"
+	"tailscale.com/logtail/backoff"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 )
 
 // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
@@ -126,3 +133,62 @@ func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error {
 	}
 	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }
+
+// waitForConsistentState waits for tailscaled to finish writing state if it
+// looks like it's started. It is designed to reduce the likelihood that
+// tailscaled gets shut down in the window between authenticating to control
+// and finishing writing state. However, it's not bullet proof because we can't
+// atomically authenticate and write state.
+func (kc *kubeClient) waitForConsistentState(ctx context.Context) error {
+	var logged bool
+
+	bo := backoff.NewBackoff("", logger.Discard, 2*time.Second)
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		secret, err := kc.GetSecret(ctx, kc.stateSecret)
+		if ctx.Err() != nil || kubeclient.IsNotFoundErr(err) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("getting Secret %q: %v", kc.stateSecret, err)
+		}
+
+		if hasConsistentState(secret.Data) {
+			return nil
+		}
+
+		if !logged {
+			log.Printf("Waiting for tailscaled to finish writing state to Secret %q", kc.stateSecret)
+			logged = true
+		}
+		bo.BackOff(ctx, errors.New("")) // Fake error to trigger actual sleep.
+	}
+}
+
+// hasConsistentState returns true is there is either no state or the full set
+// of expected keys are present.
+func hasConsistentState(d map[string][]byte) bool {
+	var (
+		_, hasCurrent = d[string(ipn.CurrentProfileStateKey)]
+		_, hasKnown   = d[string(ipn.KnownProfilesStateKey)]
+		_, hasMachine = d[string(ipn.MachineKeyStateKey)]
+		hasProfile    bool
+	)
+
+	for k := range d {
+		if strings.HasPrefix(k, "profile-") {
+			if hasProfile {
+				return false // We only expect one profile.
+			}
+			hasProfile = true
+		}
+	}
+
+	// Approximate check, we don't want to reimplement all of profileManager.
+	return (hasCurrent && hasKnown && hasMachine && hasProfile) ||
+		(!hasCurrent && !hasKnown && !hasMachine && !hasProfile)
+}

cmd/containerboot/kube_test.go

@@ -9,8 +9,10 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"tailscale.com/ipn"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
 )
@@ -205,3 +207,34 @@ func TestSetupKube(t *testing.T) {
 		})
 	}
 }
+
+func TestWaitForConsistentState(t *testing.T) {
+	data := map[string][]byte{
+		// Missing _current-profile.
+		string(ipn.KnownProfilesStateKey): []byte(""),
+		string(ipn.MachineKeyStateKey):    []byte(""),
+		"profile-foo":                     []byte(""),
+	}
+	kc := &kubeClient{
+		Client: &kubeclient.FakeClient{
+			GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+				return &kubeapi.Secret{
+					Data: data,
+				}, nil
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if err := kc.waitForConsistentState(ctx); err != context.DeadlineExceeded {
+		t.Fatalf("expected DeadlineExceeded, got %v", err)
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	data[string(ipn.CurrentProfileStateKey)] = []byte("")
+	if err := kc.waitForConsistentState(ctx); err != nil {
+		t.Fatalf("expected nil, got %v", err)
+	}
+}

cmd/containerboot/main.go

@@ -137,53 +137,83 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) {
 }
 
 func main() {
+	if err := run(); err != nil && !errors.Is(err, context.Canceled) {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
 	log.SetPrefix("boot: ")
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	cfg, err := configFromEnv()
 	if err != nil {
-		log.Fatalf("invalid configuration: %v", err)
+		return fmt.Errorf("invalid configuration: %w", err)
 	}
 
 	if !cfg.UserspaceMode {
 		if err := ensureTunFile(cfg.Root); err != nil {
-			log.Fatalf("Unable to create tuntap device file: %v", err)
+			return fmt.Errorf("unable to create tuntap device file: %w", err)
 		}
 		if cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.Routes != nil || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" {
 			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTargetIP, cfg.TailnetTargetIP, cfg.TailnetTargetFQDN, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
-					log.Fatalf("You can either set the sysctls as a privileged initContainer, or run the tailscale container with privileged=true.")
+					return fmt.Errorf("you can either set the sysctls as a privileged initContainer, or run the tailscale container with privileged=true.")
 				} else {
-					log.Fatalf("You can fix this by running the container with privileged=true, or the equivalent in your container runtime that permits access to sysctls.")
+					return fmt.Errorf("you can fix this by running the container with privileged=true, or the equivalent in your container runtime that permits access to sysctls.")
 				}
 			}
 		}
 	}
 
-	// Context is used for all setup stuff until we're in steady
+	// Root context for the whole containerboot process, used to make sure
+	// shutdown signals are promptly and cleanly handled.
+	ctx, cancel := contextWithExitSignalWatch()
+	defer cancel()
+
+	// bootCtx is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
-	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	bootCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 
 	var kc *kubeClient
 	if cfg.InKubernetes {
 		kc, err = newKubeClient(cfg.Root, cfg.KubeSecret)
 		if err != nil {
-			log.Fatalf("error initializing kube client: %v", err)
+			return fmt.Errorf("error initializing kube client: %w", err)
 		}
 		if err := cfg.setupKube(bootCtx, kc); err != nil {
-			log.Fatalf("error setting up for running on Kubernetes: %v", err)
+			return fmt.Errorf("error setting up for running on Kubernetes: %w", err)
 		}
 	}
 
 	client, daemonProcess, err := startTailscaled(bootCtx, cfg)
 	if err != nil {
-		log.Fatalf("failed to bring up tailscale: %v", err)
+		return fmt.Errorf("failed to bring up tailscale: %w", err)
 	}
 	killTailscaled := func() {
+		if hasKubeStateStore(cfg) {
+			// Check we're not shutting tailscaled down while it's still writing
+			// state. If we authenticate and fail to write all the state, we'll
+			// never recover automatically.
+			//
+			// The default termination grace period for a Pod is 30s. We wait 25s at
+			// most so that we still reserve some of that budget for tailscaled
+			// to receive and react to a SIGTERM before the SIGKILL that k8s
+			// will send at the end of the grace period.
+			ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+			defer cancel()
+
+			log.Printf("Checking for consistent state")
+			err := kc.waitForConsistentState(ctx)
+			if err != nil {
+				log.Printf("Error waiting for consistent state on shutdown: %v", err)
+			}
+		}
+		log.Printf("Sending SIGTERM to tailscaled")
 		if err := daemonProcess.Signal(unix.SIGTERM); err != nil {
 			log.Fatalf("error shutting tailscaled down: %v", err)
 		}
@@ -191,17 +221,18 @@ func main() {
 	defer killTailscaled()
 
 	var healthCheck *healthz
+	ep := &egressProxy{}
 	if cfg.HealthCheckAddrPort != "" {
 		mux := http.NewServeMux()
 
 		log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
-		healthCheck = healthHandlers(mux)
+		healthCheck = healthHandlers(mux, cfg.PodIPv4)
 
 		close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
 		defer close()
 	}
 
-	if cfg.localMetricsEnabled() || cfg.localHealthEnabled() {
+	if cfg.localMetricsEnabled() || cfg.localHealthEnabled() || cfg.egressSvcsTerminateEPEnabled() {
 		mux := http.NewServeMux()
 
 		if cfg.localMetricsEnabled() {
@@ -211,7 +242,11 @@ func main() {
 
 		if cfg.localHealthEnabled() {
 			log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
-			healthCheck = healthHandlers(mux)
+			healthCheck = healthHandlers(mux, cfg.PodIPv4)
+		}
+		if cfg.EgressProxiesCfgPath != "" {
+			log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP)
+			ep.registerHandlers(mux)
 		}
 
 		close := runHTTPServer(mux, cfg.LocalAddrPort)
@@ -226,7 +261,7 @@ func main() {
 
 	w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
 	if err != nil {
-		log.Fatalf("failed to watch tailscaled for updates: %v", err)
+		return fmt.Errorf("failed to watch tailscaled for updates: %w", err)
 	}
 
 	// Now that we've started tailscaled, we can symlink the socket to the
@@ -262,18 +297,18 @@ func main() {
 		didLogin = true
 		w.Close()
 		if err := tailscaleUp(bootCtx, cfg); err != nil {
-			return fmt.Errorf("failed to auth tailscale: %v", err)
+			return fmt.Errorf("failed to auth tailscale: %w", err)
 		}
 		w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 		if err != nil {
-			return fmt.Errorf("rewatching tailscaled for updates after auth: %v", err)
+			return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err)
 		}
 		return nil
 	}
 
 	if isTwoStepConfigAlwaysAuth(cfg) {
 		if err := authTailscale(); err != nil {
-			log.Fatalf("failed to auth tailscale: %v", err)
+			return fmt.Errorf("failed to auth tailscale: %w", err)
 		}
 	}
 
@@ -281,7 +316,7 @@ authLoop:
 	for {
 		n, err := w.Next()
 		if err != nil {
-			log.Fatalf("failed to read from tailscaled: %v", err)
+			return fmt.Errorf("failed to read from tailscaled: %w", err)
 		}
 
 		if n.State != nil {
@@ -290,10 +325,10 @@ authLoop:
 				if isOneStepConfig(cfg) {
 					// This could happen if this is the first time tailscaled was run for this
 					// device and the auth key was not passed via the configfile.
-					log.Fatalf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
+					return fmt.Errorf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
 				}
 				if err := authTailscale(); err != nil {
-					log.Fatalf("failed to auth tailscale: %v", err)
+					return fmt.Errorf("failed to auth tailscale: %w", err)
 				}
 			case ipn.NeedsMachineAuth:
 				log.Printf("machine authorization required, please visit the admin panel")
@@ -313,14 +348,11 @@ authLoop:
 
 	w.Close()
 
-	ctx, cancel := contextWithExitSignalWatch()
-	defer cancel()
-
 	if isTwoStepConfigAuthOnce(cfg) {
 		// Now that we are authenticated, we can set/reset any of the
 		// settings that we need to.
 		if err := tailscaleSet(ctx, cfg); err != nil {
-			log.Fatalf("failed to auth tailscale: %v", err)
+			return fmt.Errorf("failed to auth tailscale: %w", err)
 		}
 	}
 
@@ -329,11 +361,11 @@ authLoop:
 	if cfg.ServeConfigPath != "" {
 		log.Printf("serve proxy: unsetting previous config")
 		if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil {
-			log.Fatalf("failed to unset serve config: %v", err)
+			return fmt.Errorf("failed to unset serve config: %w", err)
 		}
 		if hasKubeStateStore(cfg) {
 			if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil {
-				log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err)
+				return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err)
 			}
 		}
 	}
@@ -344,19 +376,25 @@ authLoop:
 		// wipe it, but it's good hygiene.
 		log.Printf("Deleting authkey from kube secret")
 		if err := kc.deleteAuthKey(ctx); err != nil {
-			log.Fatalf("deleting authkey from kube secret: %v", err)
+			return fmt.Errorf("deleting authkey from kube secret: %w", err)
 		}
 	}
 
 	if hasKubeStateStore(cfg) {
 		if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil {
-			log.Fatalf("storing capability version and UID: %v", err)
+			return fmt.Errorf("storing capability version and UID: %w", err)
 		}
 	}
 
 	w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 	if err != nil {
-		log.Fatalf("rewatching tailscaled for updates after auth: %v", err)
+		return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err)
+	}
+
+	// If tailscaled config was read from a mounted file, watch the file for updates and reload.
+	cfgWatchErrChan := make(chan error)
+	if cfg.TailscaledConfigFilePath != "" {
+		go watchTailscaledConfigChanges(ctx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan)
 	}
 
 	var (
@@ -380,7 +418,7 @@ authLoop:
 	if isL3Proxy(cfg) {
 		nfr, err = newNetfilterRunner(log.Printf)
 		if err != nil {
-			log.Fatalf("error creating new netfilter runner: %v", err)
+			return fmt.Errorf("error creating new netfilter runner: %w", err)
 		}
 	}
 
@@ -451,7 +489,9 @@ runLoop:
 			killTailscaled()
 			break runLoop
 		case err := <-errChan:
-			log.Fatalf("failed to read from tailscaled: %v", err)
+			return fmt.Errorf("failed to read from tailscaled: %w", err)
+		case err := <-cfgWatchErrChan:
+			return fmt.Errorf("failed to watch tailscaled config: %w", err)
 		case n := <-notifyChan:
 			if n.State != nil && *n.State != ipn.Running {
 				// Something's gone wrong and we've left the authenticated state.
@@ -459,7 +499,7 @@ runLoop:
 				// control flow required to make it work now is hard. So, just crash
 				// the container and rely on the container runtime to restart us,
 				// whereupon we'll go through initial auth again.
-				log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
+				return fmt.Errorf("tailscaled left running state (now in state %q), exiting", *n.State)
 			}
 			if n.NetMap != nil {
 				addrs = n.NetMap.SelfNode.Addresses().AsSlice()
@@ -477,7 +517,7 @@ runLoop:
 				deviceID := n.NetMap.SelfNode.StableID()
 				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceID, &deviceID) {
 					if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil {
-						log.Fatalf("storing device ID in Kubernetes Secret: %v", err)
+						return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err)
 					}
 				}
 				if cfg.TailnetTargetFQDN != "" {
@@ -514,12 +554,12 @@ runLoop:
 								rulesInstalled = true
 								log.Printf("Installing forwarding rules for destination %v", ea.String())
 								if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil {
-									log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err)
+									return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err)
 								}
 							}
 						}
 						if !rulesInstalled {
-							log.Fatalf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT())
+							return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT())
 						}
 					}
 					currentEgressIPs = newCurentEgressIPs
@@ -527,7 +567,7 @@ runLoop:
 				if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged {
 					log.Printf("Installing proxy rules")
 					if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil {
-						log.Fatalf("installing ingress proxy rules: %v", err)
+						return fmt.Errorf("installing ingress proxy rules: %w", err)
 					}
 				}
 				if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged {
@@ -543,7 +583,7 @@ runLoop:
 					if backendsHaveChanged {
 						log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs)
 						if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil {
-							log.Fatalf("error installing ingress proxy rules: %v", err)
+							return fmt.Errorf("error installing ingress proxy rules: %w", err)
 						}
 					}
 					resetTimer(false)
@@ -565,7 +605,7 @@ runLoop:
 				if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 {
 					log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP)
 					if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil {
-						log.Fatalf("installing egress proxy rules: %v", err)
+						return fmt.Errorf("installing egress proxy rules: %w", err)
 					}
 				}
 				// If this is a L7 cluster ingress proxy (set up
@@ -577,7 +617,7 @@ runLoop:
 				if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 {
 					log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP)
 					if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil {
-						log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err)
+						return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err)
 					}
 				}
 				currentIPs = newCurrentIPs
@@ -596,7 +636,7 @@ runLoop:
 				deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()}
 				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceEndpoints, &deviceEndpoints) {
 					if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
-						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
+						return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err)
 					}
 				}
 
@@ -606,7 +646,7 @@ runLoop:
 
 				if cfg.ServeConfigPath != "" {
 					triggerWatchServeConfigChanges.Do(func() {
-						go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc)
+						go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg)
 					})
 				}
 
@@ -631,20 +671,21 @@ runLoop:
 					// will then continuously monitor the config file and netmap updates and
 					// reconfigure the firewall rules as needed. If any of its operations fail, it
 					// will crash this node.
-					if cfg.EgressSvcsCfgPath != "" {
-						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressSvcsCfgPath)
+					if cfg.EgressProxiesCfgPath != "" {
+						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath)
 						egressSvcsNotify = make(chan ipn.Notify)
-						ep := egressProxy{
-							cfgPath:      cfg.EgressSvcsCfgPath,
+						opts := egressProxyRunOpts{
+							cfgPath:      cfg.EgressProxiesCfgPath,
 							nfr:          nfr,
 							kc:           kc,
+							tsClient:     client,
 							stateSecret:  cfg.KubeSecret,
 							netmapChan:   egressSvcsNotify,
 							podIPv4:      cfg.PodIPv4,
 							tailnetAddrs: addrs,
 						}
 						go func() {
-							if err := ep.run(ctx, n); err != nil {
+							if err := ep.run(ctx, n, opts); err != nil {
 								egressSvcsErrorChan <- err
 							}
 						}()
@@ -686,16 +727,18 @@ runLoop:
 			if backendsHaveChanged && len(addrs) != 0 {
 				log.Printf("Backend address change detected, installing proxy rules for backends %v", newBackendAddrs)
 				if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil {
-					log.Fatalf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err)
+					return fmt.Errorf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err)
 				}
 			}
 			backendAddrs = newBackendAddrs
 			resetTimer(false)
 		case e := <-egressSvcsErrorChan:
-			log.Fatalf("egress proxy failed: %v", e)
+			return fmt.Errorf("egress proxy failed: %v", e)
 		}
 	}
 	wg.Wait()
+
+	return nil
 }
 
 // ensureTunFile checks that /dev/net/tun exists, creating it if
@@ -724,13 +767,13 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	ip4s, err := net.DefaultResolver.LookupIP(ctx, "ip4", name)
 	if err != nil {
 		if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) {
-			return nil, fmt.Errorf("error looking up IPv4 addresses: %v", err)
+			return nil, fmt.Errorf("error looking up IPv4 addresses: %w", err)
 		}
 	}
 	ip6s, err := net.DefaultResolver.LookupIP(ctx, "ip6", name)
 	if err != nil {
 		if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) {
-			return nil, fmt.Errorf("error looking up IPv6 addresses: %v", err)
+			return nil, fmt.Errorf("error looking up IPv6 addresses: %w", err)
 		}
 	}
 	if len(ip4s) == 0 && len(ip6s) == 0 {
@@ -743,7 +786,7 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
 func contextWithExitSignalWatch() (context.Context, func()) {
-	closeChan := make(chan string)
+	closeChan := make(chan struct{})
 	ctx, cancel := context.WithCancel(context.Background())
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
@@ -755,8 +798,11 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 			return
 		}
 	}()
+	closeOnce := sync.Once{}
 	f := func() {
-		closeChan <- "goodbye"
+		closeOnce.Do(func() {
+			close(closeChan)
+		})
 	}
 	return ctx, f
 }
@@ -809,7 +855,11 @@ func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) {
 
 	go func() {
 		if err := srv.Serve(ln); err != nil {
-			log.Fatalf("failed running server: %v", err)
+			if err != http.ErrServerClosed {
+				log.Fatalf("failed running server: %v", err)
+			} else {
+				log.Printf("HTTP server at %s closed", addr)
+			}
 		}
 	}()
 

cmd/containerboot/main_test.go

@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -32,6 +33,8 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/netmap"
@@ -48,26 +51,13 @@ func TestContainerBoot(t *testing.T) {
 	defer lapi.Close()
 
 	kube := kubeServer{FSRoot: d}
-	if err := kube.Start(); err != nil {
-		t.Fatal(err)
-	}
+	kube.Start(t)
 	defer kube.Close()
 
 	tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"}
-	tailscaledConfBytes, err := json.Marshal(tailscaledConf)
-	if err != nil {
-		t.Fatalf("error unmarshaling tailscaled config: %v", err)
-	}
 	serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}}
-	serveConfBytes, err := json.Marshal(serveConf)
-	if err != nil {
-		t.Fatalf("error unmarshaling serve config: %v", err)
-	}
-	egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}}
-	egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg)
-	if err != nil {
-		t.Fatalf("error unmarshaling egress services config: %v", err)
-	}
+	egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net")
+	egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net")
 
 	dirs := []string{
 		"var/lib",
@@ -84,16 +74,17 @@ func TestContainerBoot(t *testing.T) {
 		}
 	}
 	files := map[string][]byte{
-		"usr/bin/tailscaled":                         fakeTailscaled,
-		"usr/bin/tailscale":                          fakeTailscale,
-		"usr/bin/iptables":                           fakeTailscale,
-		"usr/bin/ip6tables":                          fakeTailscale,
-		"dev/net/tun":                                []byte(""),
-		"proc/sys/net/ipv4/ip_forward":               []byte("0"),
-		"proc/sys/net/ipv6/conf/all/forwarding":      []byte("0"),
-		"etc/tailscaled/cap-95.hujson":               tailscaledConfBytes,
-		"etc/tailscaled/serve-config.json":           serveConfBytes,
-		"etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes,
+		"usr/bin/tailscaled":                    fakeTailscaled,
+		"usr/bin/tailscale":                     fakeTailscale,
+		"usr/bin/iptables":                      fakeTailscale,
+		"usr/bin/ip6tables":                     fakeTailscale,
+		"dev/net/tun":                           []byte(""),
+		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
+		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
+		"etc/tailscaled/cap-95.hujson":          mustJSON(t, tailscaledConf),
+		"etc/tailscaled/serve-config.json":      mustJSON(t, serveConf),
+		filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg),
+		filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings):       []byte("4"),
 	}
 	resetFiles := func() {
 		for path, content := range files {
@@ -132,6 +123,9 @@ func TestContainerBoot(t *testing.T) {
 	healthURL := func(port int) string {
 		return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
 	}
+	egressSvcTerminateURL := func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d%s", port, kubetypes.EgessServicesPreshutdownEP)
+	}
 
 	capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
 
@@ -143,15 +137,29 @@ func TestContainerBoot(t *testing.T) {
 
 		// WantCmds is the commands that containerboot should run in this phase.
 		WantCmds []string
+
 		// WantKubeSecret is the secret keys/values that should exist in the
 		// kube secret.
 		WantKubeSecret map[string]string
+
+		// Update the kube secret with these keys/values at the beginning of the
+		// phase (simulates our fake tailscaled doing it).
+		UpdateKubeSecret map[string]string
+
 		// WantFiles files that should exist in the container and their
 		// contents.
 		WantFiles map[string]string
-		// WantFatalLog is the fatal log message we expect from containerboot.
-		// If set for a phase, the test will finish on that phase.
-		WantFatalLog string
+
+		// WantLog is a log message we expect from containerboot.
+		WantLog string
+
+		// If set for a phase, the test will expect containerboot to exit with
+		// this error code, and the test will finish on that phase without
+		// waiting for the successful startup log message.
+		WantExitCode *int
+
+		// The signal to send to containerboot at the start of the phase.
+		Signal *syscall.Signal
 
 		EndpointStatuses map[string]int
 	}
@@ -439,7 +447,8 @@ func TestContainerBoot(t *testing.T) {
 							},
 						},
 					},
-					WantFatalLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false",
+					WantLog:      "no forwarding rules for egress addresses [::1/128], host supports IPv6: false",
+					WantExitCode: ptr.To(1),
 				},
 			},
 		},
@@ -896,9 +905,10 @@ func TestContainerBoot(t *testing.T) {
 		{
 			Name: "egress_svcs_config_kube",
 			Env: map[string]string{
-				"KUBERNETES_SERVICE_HOST":        kube.Host,
-				"KUBERNETES_SERVICE_PORT_HTTPS":  kube.Port,
-				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"),
+				"TS_LOCAL_ADDR_PORT":            fmt.Sprintf("[::]:%d", localAddrPort),
 			},
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
@@ -912,28 +922,92 @@ func TestContainerBoot(t *testing.T) {
 					WantKubeSecret: map[string]string{
 						"authkey": "tskey-key",
 					},
+					EndpointStatuses: map[string]int{
+						egressSvcTerminateURL(localAddrPort): 200,
+					},
 				},
 				{
 					Notify: runningNotify,
 					WantKubeSecret: map[string]string{
+						"egress-services":  mustBase64(t, egressStatus),
 						"authkey":          "tskey-key",
 						"device_fqdn":      "test-node.test.ts.net",
 						"device_id":        "myID",
 						"device_ips":       `["100.64.0.1"]`,
 						"tailscale_capver": capver,
 					},
+					EndpointStatuses: map[string]int{
+						egressSvcTerminateURL(localAddrPort): 200,
+					},
 				},
 			},
 		},
 		{
 			Name: "egress_svcs_config_no_kube",
 			Env: map[string]string{
-				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
-				"TS_AUTHKEY":                     "tskey-key",
+				"TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"),
+				"TS_AUTHKEY":                    "tskey-key",
 			},
 			Phases: []phase{
 				{
-					WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
+					WantLog:      "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
+					WantExitCode: ptr.To(1),
+				},
+			},
+		},
+		{
+			Name: "kube_shutdown_during_state_write",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_ENABLE_HEALTH_CHECK":        "true",
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			Phases: []phase{
+				{
+					// Normal startup.
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					// SIGTERM before state is finished writing, should wait for
+					// consistent state before propagating SIGTERM to tailscaled.
+					Signal: ptr.To(unix.SIGTERM),
+					UpdateKubeSecret: map[string]string{
+						"_machinekey":  "foo",
+						"_profiles":    "foo",
+						"profile-baff": "foo",
+						// Missing "_current-profile" key.
+					},
+					WantKubeSecret: map[string]string{
+						"authkey":      "tskey-key",
+						"_machinekey":  "foo",
+						"_profiles":    "foo",
+						"profile-baff": "foo",
+					},
+					WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"",
+				},
+				{
+					// tailscaled has finished writing state, should propagate SIGTERM.
+					UpdateKubeSecret: map[string]string{
+						"_current-profile": "foo",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey":          "tskey-key",
+						"_machinekey":      "foo",
+						"_profiles":        "foo",
+						"profile-baff":     "foo",
+						"_current-profile": "foo",
+					},
+					WantLog:      "HTTP server at [::]:9002 closed",
+					WantExitCode: ptr.To(0),
 				},
 			},
 		},
@@ -981,26 +1055,36 @@ func TestContainerBoot(t *testing.T) {
 
 			var wantCmds []string
 			for i, p := range test.Phases {
+				for k, v := range p.UpdateKubeSecret {
+					kube.SetSecret(k, v)
+				}
 				lapi.Notify(p.Notify)
-				if p.WantFatalLog != "" {
+				if p.Signal != nil {
+					cmd.Process.Signal(*p.Signal)
+				}
+				if p.WantLog != "" {
 					err := tstest.WaitFor(2*time.Second, func() error {
-						state, err := cmd.Process.Wait()
-						if err != nil {
-							return err
-						}
-						if state.ExitCode() != 1 {
-							return fmt.Errorf("process exited with code %d but wanted %d", state.ExitCode(), 1)
-						}
-						waitLogLine(t, time.Second, cbOut, p.WantFatalLog)
+						waitLogLine(t, time.Second, cbOut, p.WantLog)
 						return nil
 					})
 					if err != nil {
 						t.Fatal(err)
 					}
+				}
+
+				if p.WantExitCode != nil {
+					state, err := cmd.Process.Wait()
+					if err != nil {
+						t.Fatal(err)
+					}
+					if state.ExitCode() != *p.WantExitCode {
+						t.Fatalf("phase %d: want exit code %d, got %d", i, *p.WantExitCode, state.ExitCode())
+					}
 
 					// Early test return, we don't expect the successful startup log message.
 					return
 				}
+
 				wantCmds = append(wantCmds, p.WantCmds...)
 				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
 				err := tstest.WaitFor(2*time.Second, func() error {
@@ -1056,6 +1140,9 @@ func TestContainerBoot(t *testing.T) {
 				}
 			}
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
+			if cmd.ProcessState != nil {
+				t.Fatalf("containerboot should be running but exited with exit code %d", cmd.ProcessState.ExitCode())
+			}
 		})
 	}
 }
@@ -1287,18 +1374,18 @@ func (k *kubeServer) Reset() {
 	k.secret = map[string]string{}
 }
 
-func (k *kubeServer) Start() error {
+func (k *kubeServer) Start(t *testing.T) {
 	root := filepath.Join(k.FSRoot, "var/run/secrets/kubernetes.io/serviceaccount")
 
 	if err := os.MkdirAll(root, 0700); err != nil {
-		return err
+		t.Fatal(err)
 	}
 
 	if err := os.WriteFile(filepath.Join(root, "namespace"), []byte("default"), 0600); err != nil {
-		return err
+		t.Fatal(err)
 	}
 	if err := os.WriteFile(filepath.Join(root, "token"), []byte("bearer_token"), 0600); err != nil {
-		return err
+		t.Fatal(err)
 	}
 
 	k.srv = httptest.NewTLSServer(k)
@@ -1307,13 +1394,11 @@ func (k *kubeServer) Start() error {
 
 	var cert bytes.Buffer
 	if err := pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: k.srv.Certificate().Raw}); err != nil {
-		return err
+		t.Fatal(err)
 	}
 	if err := os.WriteFile(filepath.Join(root, "ca.crt"), cert.Bytes(), 0600); err != nil {
-		return err
+		t.Fatal(err)
 	}
-
-	return nil
 }
 
 func (k *kubeServer) Close() {
@@ -1362,6 +1447,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, fmt.Sprintf("reading request body: %v", err), http.StatusInternalServerError)
 		return
 	}
+	defer r.Body.Close()
 
 	switch r.Method {
 	case "GET":
@@ -1394,13 +1480,32 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 				panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
 			}
 			for _, op := range req {
-				if op.Op != "remove" {
+				switch op.Op {
+				case "remove":
+					if !strings.HasPrefix(op.Path, "/data/") {
+						panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
+					}
+					delete(k.secret, strings.TrimPrefix(op.Path, "/data/"))
+				case "replace":
+					path, ok := strings.CutPrefix(op.Path, "/data/")
+					if !ok {
+						panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
+					}
+					req := make([]kubeclient.JSONPatch, 0)
+					if err := json.Unmarshal(bs, &req); err != nil {
+						panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
+					}
+
+					for _, patch := range req {
+						val, ok := patch.Value.(string)
+						if !ok {
+							panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value))
+						}
+						k.secret[path] = val
+					}
+				default:
 					panic(fmt.Sprintf("unsupported json-patch op %q", op.Op))
 				}
-				if !strings.HasPrefix(op.Path, "/data/") {
-					panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
-				}
-				delete(k.secret, strings.TrimPrefix(op.Path, "/data/"))
 			}
 		case "application/strategic-merge-patch+json":
 			req := struct {
@@ -1416,6 +1521,44 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 			panic(fmt.Sprintf("unknown content type %q", r.Header.Get("Content-Type")))
 		}
 	default:
-		panic(fmt.Sprintf("unhandled HTTP method %q", r.Method))
+		panic(fmt.Sprintf("unhandled HTTP request %s %s", r.Method, r.URL))
+	}
+}
+
+func mustBase64(t *testing.T, v any) string {
+	b := mustJSON(t, v)
+	s := base64.StdEncoding.WithPadding('=').EncodeToString(b)
+	return s
+}
+
+func mustJSON(t *testing.T, v any) []byte {
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("error converting %v to json: %v", v, err)
+	}
+	return b
+}
+
+// egress services status given one named tailnet target specified by FQDN. As written by the proxy to its state Secret.
+func egressSvcStatus(name, fqdn string) egressservices.Status {
+	return egressservices.Status{
+		Services: map[string]*egressservices.ServiceStatus{
+			name: {
+				TailnetTarget: egressservices.TailnetTarget{
+					FQDN: fqdn,
+				},
+			},
+		},
+	}
+}
+
+// egress config given one named tailnet target specified by FQDN.
+func egressSvcConfig(name, fqdn string) egressservices.Configs {
+	return egressservices.Configs{
+		name: egressservices.Config{
+			TailnetTarget: egressservices.TailnetTarget{
+				FQDN: fqdn,
+			},
+		},
 	}
 }

cmd/containerboot/metrics.go

@@ -10,7 +10,7 @@ import (
 	"io"
 	"net/http"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 )
 
@@ -18,7 +18,7 @@ import (
 // the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
 type metrics struct {
 	debugEndpoint string
-	lc            *tailscale.LocalClient
+	lc            *local.Client
 }
 
 func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
@@ -68,7 +68,7 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
 // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
 // endpoint if configured to ease migration for a breaking change serving user
 // metrics instead of debug metrics on the "metrics" port.
-func metricsHandlers(mux *http.ServeMux, lc *tailscale.LocalClient, debugAddrPort string) {
+func metricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) {
 	m := &metrics{
 		lc:            lc,
 		debugEndpoint: debugAddrPort,

cmd/containerboot/serve.go

@@ -17,7 +17,7 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/netmap"
@@ -28,20 +28,23 @@ import (
 // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
 // is written to when the certDomain changes, causing the serve config to be
 // re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient, kc *kubeClient) {
+func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings) {
 	if certDomainAtomic == nil {
 		panic("certDomainAtomic must not be nil")
 	}
+
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	if w, err := fsnotify.NewWatcher(); err != nil {
+		// Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor.
+		// See https://github.com/tailscale/tailscale/issues/15081
 		log.Printf("serve proxy: failed to create fsnotify watcher, timer-only mode: %v", err)
 		ticker := time.NewTicker(5 * time.Second)
 		defer ticker.Stop()
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
+		if err := w.Add(filepath.Dir(cfg.ServeConfigPath)); err != nil {
 			log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
 		}
 		eventChan = w.Events
@@ -49,6 +52,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 
 	var certDomain string
 	var prevServeConfig *ipn.ServeConfig
+	var cm certManager
+	if cfg.CertShareMode == "rw" {
+		cm = certManager{
+			lc: lc,
+		}
+	}
 	for {
 		select {
 		case <-ctx.Done():
@@ -61,14 +70,17 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			// k8s handles these mounts. So just re-read the file and apply it
 			// if it's changed.
 		}
-		sc, err := readServeConfig(path, certDomain)
+		sc, err := readServeConfig(cfg.ServeConfigPath, certDomain)
 		if err != nil {
 			log.Fatalf("serve proxy: failed to read serve config: %v", err)
 		}
+		if sc == nil {
+			log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath)
+			continue
+		}
 		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
 			continue
 		}
-		validateHTTPSServe(certDomain, sc)
 		if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil {
 			log.Fatalf("serve proxy: error updating serve config: %v", err)
 		}
@@ -78,6 +90,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			}
 		}
 		prevServeConfig = sc
+		if cfg.CertShareMode != "rw" {
+			continue
+		}
+		if err := cm.ensureCertLoops(ctx, sc); err != nil {
+			log.Fatalf("serve proxy: error ensuring cert loops: %v", err)
+		}
 	}
 }
 
@@ -88,27 +106,35 @@ func certDomainFromNetmap(nm *netmap.NetworkMap) string {
 	return nm.DNS.CertDomains[0]
 }
 
-func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc *tailscale.LocalClient) error {
-	// TODO(irbekrm): This means that serve config that does not expose HTTPS endpoint will not be set for a tailnet
-	// that does not have HTTPS enabled. We probably want to fix this.
-	if certDomain == kubetypes.ValueNoHTTPS {
+// localClient is a subset of [local.Client] that can be mocked for testing.
+type localClient interface {
+	SetServeConfig(context.Context, *ipn.ServeConfig) error
+	CertPair(context.Context, string) ([]byte, []byte, error)
+}
+
+func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error {
+	if !isValidHTTPSConfig(certDomain, sc) {
 		return nil
 	}
 	log.Printf("serve proxy: applying serve config")
 	return lc.SetServeConfig(ctx, sc)
 }
 
-func validateHTTPSServe(certDomain string, sc *ipn.ServeConfig) {
-	if certDomain != kubetypes.ValueNoHTTPS || !hasHTTPSEndpoint(sc) {
-		return
-	}
-	log.Printf(
-		`serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet,
+func isValidHTTPSConfig(certDomain string, sc *ipn.ServeConfig) bool {
+	if certDomain == kubetypes.ValueNoHTTPS && hasHTTPSEndpoint(sc) {
+		log.Printf(
+			`serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet,
 		(perhaps a Kubernetes operator Ingress proxy) but it is not able to issue TLS certs, so this will likely not work.
 		To make it work, ensure that HTTPS is enabled for your tailnet, see https://tailscale.com/kb/1153/enabling-https for more details.`)
+		return false
+	}
+	return true
 }
 
 func hasHTTPSEndpoint(cfg *ipn.ServeConfig) bool {
+	if cfg == nil {
+		return false
+	}
 	for _, tcpCfg := range cfg.TCP {
 		if tcpCfg.HTTPS {
 			return true
@@ -125,8 +151,17 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
 	}
 	j, err := os.ReadFile(path)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
 		return nil, err
 	}
+	// Serve config can be provided by users as well as the Kubernetes Operator (for its proxies). User-provided
+	// config could be empty for reasons.
+	if len(j) == 0 {
+		log.Printf("serve proxy: serve config file is empty, skipping")
+		return nil, nil
+	}
 	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
 	var sc ipn.ServeConfig
 	if err := json.Unmarshal(j, &sc); err != nil {

cmd/containerboot/serve_test.go

@@ -0,0 +1,271 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/client/local"
+	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
+)
+
+func TestUpdateServeConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		sc         *ipn.ServeConfig
+		certDomain string
+		wantCall   bool
+	}{
+		{
+			name: "no_https_no_cert_domain",
+			sc: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80: {HTTP: true},
+				},
+			},
+			certDomain: kubetypes.ValueNoHTTPS, // tailnet has HTTPS disabled
+			wantCall:   true,                   // should set serve config as it doesn't have HTTPS endpoints
+		},
+		{
+			name: "https_with_cert_domain",
+			sc: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {HTTPS: true},
+				},
+				Web: map[ipn.HostPort]*ipn.WebServerConfig{
+					"${TS_CERT_DOMAIN}:443": {
+						Handlers: map[string]*ipn.HTTPHandler{
+							"/": {Proxy: "http://10.0.1.100:8080"},
+						},
+					},
+				},
+			},
+			certDomain: "test-node.tailnet.ts.net",
+			wantCall:   true,
+		},
+		{
+			name: "https_without_cert_domain",
+			sc: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {HTTPS: true},
+				},
+			},
+			certDomain: kubetypes.ValueNoHTTPS,
+			wantCall:   false, // incorrect configuration- should not set serve config
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fakeLC := &fakeLocalClient{}
+			err := updateServeConfig(context.Background(), tt.sc, tt.certDomain, fakeLC)
+			if err != nil {
+				t.Errorf("updateServeConfig() error = %v", err)
+			}
+			if fakeLC.setServeCalled != tt.wantCall {
+				t.Errorf("SetServeConfig() called = %v, want %v", fakeLC.setServeCalled, tt.wantCall)
+			}
+		})
+	}
+}
+
+func TestReadServeConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		gotSC      string
+		certDomain string
+		wantSC     *ipn.ServeConfig
+		wantErr    bool
+	}{
+		{
+			name: "empty_file",
+		},
+		{
+			name: "valid_config_with_cert_domain_placeholder",
+			gotSC: `{
+				"TCP": {
+					"443": {
+						"HTTPS": true
+					}
+				},
+				"Web": {
+					"${TS_CERT_DOMAIN}:443": {
+					"Handlers": {
+						"/api": {
+							"Proxy": "https://10.2.3.4/api"
+						}}}}}`,
+			certDomain: "example.com",
+			wantSC: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {
+						HTTPS: true,
+					},
+				},
+				Web: map[ipn.HostPort]*ipn.WebServerConfig{
+					ipn.HostPort("example.com:443"): {
+						Handlers: map[string]*ipn.HTTPHandler{
+							"/api": {
+								Proxy: "https://10.2.3.4/api",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "valid_config_for_http_proxy",
+			gotSC: `{
+				"TCP": {
+					"80": {
+						"HTTP": true
+					}
+				}}`,
+			wantSC: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80: {
+						HTTP: true,
+					},
+				},
+			},
+		},
+		{
+			name: "config_without_cert_domain",
+			gotSC: `{
+				"TCP": {
+					"443": {
+						"HTTPS": true
+					}
+				},
+				"Web": {
+					"localhost:443": {
+					"Handlers": {
+						"/api": {
+							"Proxy": "https://10.2.3.4/api"
+						}}}}}`,
+			certDomain: "",
+			wantErr:    false,
+			wantSC: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {
+						HTTPS: true,
+					},
+				},
+				Web: map[ipn.HostPort]*ipn.WebServerConfig{
+					ipn.HostPort("localhost:443"): {
+						Handlers: map[string]*ipn.HTTPHandler{
+							"/api": {
+								Proxy: "https://10.2.3.4/api",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:    "invalid_json",
+			gotSC:   "invalid json",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "serve-config.json")
+			if err := os.WriteFile(path, []byte(tt.gotSC), 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			got, err := readServeConfig(path, tt.certDomain)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("readServeConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !cmp.Equal(got, tt.wantSC) {
+				t.Errorf("readServeConfig() diff (-got +want):\n%s", cmp.Diff(got, tt.wantSC))
+			}
+		})
+	}
+}
+
+type fakeLocalClient struct {
+	*local.Client
+	setServeCalled bool
+}
+
+func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error {
+	m.setServeCalled = true
+	return nil
+}
+
+func (m *fakeLocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+	return nil, nil, nil
+}
+
+func TestHasHTTPSEndpoint(t *testing.T) {
+	tests := []struct {
+		name string
+		cfg  *ipn.ServeConfig
+		want bool
+	}{
+		{
+			name: "nil_config",
+			cfg:  nil,
+			want: false,
+		},
+		{
+			name: "empty_config",
+			cfg:  &ipn.ServeConfig{},
+			want: false,
+		},
+		{
+			name: "no_https_endpoints",
+			cfg: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80: {
+						HTTPS: false,
+					},
+				},
+			},
+			want: false,
+		},
+		{
+			name: "has_https_endpoint",
+			cfg: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					443: {
+						HTTPS: true,
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "mixed_endpoints",
+			cfg: &ipn.ServeConfig{
+				TCP: map[uint16]*ipn.TCPPortHandler{
+					80:  {HTTPS: false},
+					443: {HTTPS: true},
+				},
+			},
+			want: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := hasHTTPSEndpoint(tt.cfg)
+			if got != tt.want {
+				t.Errorf("hasHTTPSEndpoint() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/containerboot/services.go

@@ -11,18 +11,24 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net/http"
 	"net/netip"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/local"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/egressservices"
 	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/httpm"
 	"tailscale.com/util/linuxfw"
 	"tailscale.com/util/mak"
 )
@@ -37,13 +43,15 @@ const tailscaleTunInterface = "tailscale0"
 // egressProxy knows how to configure firewall rules to route cluster traffic to
 // one or more tailnet services.
 type egressProxy struct {
-	cfgPath string // path to egress service config file
+	cfgPath string // path to a directory with egress services config files
 
 	nfr linuxfw.NetfilterRunner // never nil
 
 	kc          kubeclient.Client // never nil
 	stateSecret string            // name of the kube state Secret
 
+	tsClient *local.Client // never nil
+
 	netmapChan chan ipn.Notify // chan to receive netmap updates on
 
 	podIPv4 string // never empty string, currently only IPv4 is supported
@@ -55,15 +63,29 @@ type egressProxy struct {
 	// memory at all.
 	targetFQDNs map[string][]netip.Prefix
 
-	// used to configure firewall rules.
-	tailnetAddrs []netip.Prefix
+	tailnetAddrs []netip.Prefix // tailnet IPs of this tailnet device
+
+	// shortSleep is the backoff sleep between healthcheck endpoint calls - can be overridden in tests.
+	shortSleep time.Duration
+	// longSleep is the time to sleep after the routing rules are updated to increase the chance that kube
+	// proxies on all nodes have updated their routing configuration. It can be configured to 0 in
+	// tests.
+	longSleep time.Duration
+	// client is a client that can send HTTP requests.
+	client httpClient
+}
+
+// httpClient is a client that can send HTTP requests and can be mocked in tests.
+type httpClient interface {
+	Do(*http.Request) (*http.Response, error)
 }
 
 // run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when:
 // - the mounted egress config has changed
 // - the proxy's tailnet IP addresses have changed
 // - tailnet IPs have changed for any backend targets specified by tailnet FQDN
-func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
+func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRunOpts) error {
+	ep.configure(opts)
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	// TODO (irbekrm): take a look if this can be pulled into a single func
@@ -75,7 +97,7 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
-		if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil {
+		if err := w.Add(ep.cfgPath); err != nil {
 			return fmt.Errorf("failed to add fsnotify watch: %w", err)
 		}
 		eventChan = w.Events
@@ -85,28 +107,52 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
 		return err
 	}
 	for {
-		var err error
 		select {
 		case <-ctx.Done():
 			return nil
 		case <-tickChan:
-			err = ep.sync(ctx, n)
+			log.Printf("periodic sync, ensuring firewall config is up to date...")
 		case <-eventChan:
 			log.Printf("config file change detected, ensuring firewall config is up to date...")
-			err = ep.sync(ctx, n)
 		case n = <-ep.netmapChan:
 			shouldResync := ep.shouldResync(n)
-			if shouldResync {
-				log.Printf("netmap change detected, ensuring firewall config is up to date...")
-				err = ep.sync(ctx, n)
+			if !shouldResync {
+				continue
 			}
+			log.Printf("netmap change detected, ensuring firewall config is up to date...")
 		}
-		if err != nil {
+		if err := ep.sync(ctx, n); err != nil {
 			return fmt.Errorf("error syncing egress service config: %w", err)
 		}
 	}
 }
 
+type egressProxyRunOpts struct {
+	cfgPath      string
+	nfr          linuxfw.NetfilterRunner
+	kc           kubeclient.Client
+	tsClient     *local.Client
+	stateSecret  string
+	netmapChan   chan ipn.Notify
+	podIPv4      string
+	tailnetAddrs []netip.Prefix
+}
+
+// applyOpts configures egress proxy using the provided options.
+func (ep *egressProxy) configure(opts egressProxyRunOpts) {
+	ep.cfgPath = opts.cfgPath
+	ep.nfr = opts.nfr
+	ep.kc = opts.kc
+	ep.tsClient = opts.tsClient
+	ep.stateSecret = opts.stateSecret
+	ep.netmapChan = opts.netmapChan
+	ep.podIPv4 = opts.podIPv4
+	ep.tailnetAddrs = opts.tailnetAddrs
+	ep.client = &http.Client{} // default HTTP client
+	ep.shortSleep = time.Second
+	ep.longSleep = time.Second * 10
+}
+
 // sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if
 // any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current
 // firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such
@@ -327,7 +373,8 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s
 
 // getConfigs gets the mounted egress service configuration.
 func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) {
-	j, err := os.ReadFile(ep.cfgPath)
+	svcsCfg := filepath.Join(ep.cfgPath, egressservices.KeyEgressServices)
+	j, err := os.ReadFile(svcsCfg)
 	if os.IsNotExist(err) {
 		return nil, nil
 	}
@@ -569,3 +616,142 @@ func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
 	st1.PodIPv4 = ""
 	return reflect.DeepEqual(*st, *st1)
 }
+
+// registerHandlers adds a new handler to the provided ServeMux that can be called as a Kubernetes prestop hook to
+// delay shutdown till it's safe to do so.
+func (ep *egressProxy) registerHandlers(mux *http.ServeMux) {
+	mux.Handle(fmt.Sprintf("GET %s", kubetypes.EgessServicesPreshutdownEP), ep)
+}
+
+// ServeHTTP serves /internal-egress-services-preshutdown endpoint, when it receives a request, it periodically polls
+// the configured health check endpoint for each egress service till it the health check endpoint no longer hits this
+// proxy Pod. It uses the Pod-IPv4 header to verify if health check response is received from this Pod.
+func (ep *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	cfgs, err := ep.getConfigs()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("error retrieving egress services configs: %v", err), http.StatusInternalServerError)
+		return
+	}
+	if cfgs == nil {
+		if _, err := w.Write([]byte("safe to terminate")); err != nil {
+			http.Error(w, fmt.Sprintf("error writing termination status: %v", err), http.StatusInternalServerError)
+			return
+		}
+	}
+	hp, err := ep.getHEPPings()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("error determining the number of times health check endpoint should be pinged: %v", err), http.StatusInternalServerError)
+		return
+	}
+	ep.waitTillSafeToShutdown(r.Context(), cfgs, hp)
+}
+
+// waitTillSafeToShutdown looks up all egress targets configured to be proxied via this instance and, for each target
+// whose configuration includes a healthcheck endpoint, pings the endpoint till none of the responses
+// are returned by this instance or till the HTTP request times out. In practice, the endpoint will be a Kubernetes Service for whom one of the backends
+// would normally be this Pod. When this Pod is being deleted, the operator should have removed it from the Service
+// backends and eventually kube proxy routing rules should be updated to no longer route traffic for the Service to this
+// Pod.
+func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs *egressservices.Configs, hp int) {
+	if cfgs == nil || len(*cfgs) == 0 { // avoid sleeping if no services are configured
+		return
+	}
+	log.Printf("Ensuring that cluster traffic for egress targets is no longer routed via this Pod...")
+	wg := syncs.WaitGroup{}
+
+	for s, cfg := range *cfgs {
+		hep := cfg.HealthCheckEndpoint
+		if hep == "" {
+			log.Printf("Tailnet target %q does not have a cluster healthcheck specified, unable to verify if cluster traffic for the target is still routed via this Pod", s)
+			continue
+		}
+		svc := s
+		wg.Go(func() {
+			log.Printf("Ensuring that cluster traffic is no longer routed to %q via this Pod...", svc)
+			for {
+				if ctx.Err() != nil { // kubelet's HTTP request timeout
+					log.Printf("Cluster traffic for %s did not stop being routed to this Pod.", svc)
+					return
+				}
+				found, err := lookupPodRoute(ctx, hep, ep.podIPv4, hp, ep.client)
+				if err != nil {
+					log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err)
+					break
+				}
+				if !found {
+					log.Printf("service %q is no longer routed through this Pod", svc)
+					break
+				}
+				log.Printf("service %q is still routed through this Pod, waiting...", svc)
+				time.Sleep(ep.shortSleep)
+			}
+		})
+	}
+	wg.Wait()
+	// The check above really only checked that the routing rules are updated on this node. Sleep for a bit to
+	// ensure that the routing rules are updated on other nodes. TODO(irbekrm): this may or may not be good enough.
+	// If it's not good enough, we'd probably want to do something more complex, where the proxies check each other.
+	log.Printf("Sleeping for %s before shutdown to ensure that kube proxies on all nodes have updated routing configuration", ep.longSleep)
+	time.Sleep(ep.longSleep)
+}
+
+// lookupPodRoute calls the healthcheck endpoint repeat times and returns true if the endpoint returns with the podIP
+// header at least once.
+func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client httpClient) (bool, error) {
+	for range repeat {
+		f, err := lookup(ctx, hep, podIP, client)
+		if err != nil {
+			return false, err
+		}
+		if f {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// lookup calls the healthcheck endpoint and returns true if the response contains the podIP header.
+func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, hep, nil)
+	if err != nil {
+		return false, fmt.Errorf("error creating new HTTP request: %v", err)
+	}
+
+	// Close the TCP connection to ensure that the next request is routed to a different backend.
+	req.Close = true
+
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Printf("Endpoint %q can not be reached: %v, likely because there are no (more) healthy backends", hep, err)
+		return true, nil
+	}
+	defer resp.Body.Close()
+	gotIP := resp.Header.Get(kubetypes.PodIPv4Header)
+	return strings.EqualFold(podIP, gotIP), nil
+}
+
+// getHEPPings gets the number of pings that should be sent to a health check endpoint to ensure that each configured
+// backend is hit. This assumes that a health check endpoint is a Kubernetes Service and traffic to backend Pods is
+// round robin load balanced.
+func (ep *egressProxy) getHEPPings() (int, error) {
+	hepPingsPath := filepath.Join(ep.cfgPath, egressservices.KeyHEPPings)
+	j, err := os.ReadFile(hepPingsPath)
+	if os.IsNotExist(err) {
+		return 0, nil
+	}
+	if err != nil {
+		return -1, err
+	}
+	if len(j) == 0 || string(j) == "" {
+		return 0, nil
+	}
+	hp, err := strconv.Atoi(string(j))
+	if err != nil {
+		return -1, fmt.Errorf("error parsing hep pings as int: %v", err)
+	}
+	if hp < 0 {
+		log.Printf("[unexpected] hep pings is negative: %d", hp)
+		return 0, nil
+	}
+	return hp, nil
+}

cmd/containerboot/services_test.go

@@ -6,11 +6,18 @@
 package main
 
 import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
 	"net/netip"
 	"reflect"
+	"strings"
+	"sync"
 	"testing"
 
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
 )
 
 func Test_updatesForSvc(t *testing.T) {
@@ -173,3 +180,145 @@ func Test_updatesForSvc(t *testing.T) {
 		})
 	}
 }
+
+// A failure of this test will most likely look like a timeout.
+func TestWaitTillSafeToShutdown(t *testing.T) {
+	podIP := "10.0.0.1"
+	anotherIP := "10.0.0.2"
+
+	tests := []struct {
+		name string
+		// services is a map of service name to the number of calls to make to the healthcheck endpoint before
+		// returning a response that does NOT contain this Pod's IP in headers.
+		services       map[string]int
+		replicas       int
+		healthCheckSet bool
+	}{
+		{
+			name: "no_configs",
+		},
+		{
+			name: "one_service_immediately_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 0,
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_immediately_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 0,
+				"svc2": 0,
+				"svc3": 0,
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_no_healthcheck_endpoints",
+			services: map[string]int{
+				"svc1": 0,
+				"svc2": 0,
+				"svc3": 0,
+			},
+			replicas: 2,
+		},
+		{
+			name: "one_service_eventually_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_eventually_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 1, // After 1 call to health check endpoint, no longer returns this Pod's IP
+				"svc2": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP
+				"svc3": 5, // After 5 calls to the health check endpoint, no longer returns this Pod's IP
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_eventually_safe_to_shutdown_with_higher_replica_count",
+			services: map[string]int{
+				"svc1": 7,
+				"svc2": 10,
+			},
+			replicas:       5,
+			healthCheckSet: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfgs := &egressservices.Configs{}
+			switches := make(map[string]int)
+
+			for svc, callsToSwitch := range tt.services {
+				endpoint := fmt.Sprintf("http://%s.local", svc)
+				if tt.healthCheckSet {
+					(*cfgs)[svc] = egressservices.Config{
+						HealthCheckEndpoint: endpoint,
+					}
+				}
+				switches[endpoint] = callsToSwitch
+			}
+
+			ep := &egressProxy{
+				podIPv4: podIP,
+				client: &mockHTTPClient{
+					podIP:     podIP,
+					anotherIP: anotherIP,
+					switches:  switches,
+				},
+			}
+
+			ep.waitTillSafeToShutdown(context.Background(), cfgs, tt.replicas)
+		})
+	}
+}
+
+// mockHTTPClient is a client that receives an HTTP call for an egress service endpoint and returns a response with an
+// IP address in a 'Pod-IPv4' header. It can be configured to return one IP address for N calls, then switch to another
+// IP address to simulate a scenario where an IP is eventually no longer a backend for an endpoint.
+// TODO(irbekrm): to test this more thoroughly, we should have the client take into account the number of replicas and
+// return as if traffic was round robin load balanced across different Pods.
+type mockHTTPClient struct {
+	// podIP - initial IP address to return, that matches the current proxy's IP address.
+	podIP     string
+	anotherIP string
+	// after how many calls to an endpoint, the client should start returning 'anotherIP' instead of 'podIP.
+	switches map[string]int
+	mu       sync.Mutex // protects the following
+	// calls tracks the number of calls received.
+	calls map[string]int
+}
+
+func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	m.mu.Lock()
+	if m.calls == nil {
+		m.calls = make(map[string]int)
+	}
+
+	endpoint := req.URL.String()
+	m.calls[endpoint]++
+	calls := m.calls[endpoint]
+	m.mu.Unlock()
+
+	resp := &http.Response{
+		StatusCode: http.StatusOK,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader("")),
+	}
+
+	if calls <= m.switches[endpoint] {
+		resp.Header.Set(kubetypes.PodIPv4Header, m.podIP) // Pod is still routable
+	} else {
+		resp.Header.Set(kubetypes.PodIPv4Header, m.anotherIP) // Pod is no longer routable
+	}
+	return resp, nil
+}

cmd/containerboot/settings.go

@@ -64,16 +64,22 @@ type settings struct {
 	// when setting up rules to proxy cluster traffic to cluster ingress
 	// target.
 	// Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters
-	PodIP               string
-	PodIPv4             string
-	PodIPv6             string
-	PodUID              string
-	HealthCheckAddrPort string
-	LocalAddrPort       string
-	MetricsEnabled      bool
-	HealthCheckEnabled  bool
-	DebugAddrPort       string
-	EgressSvcsCfgPath   string
+	PodIP                string
+	PodIPv4              string
+	PodIPv6              string
+	PodUID               string
+	HealthCheckAddrPort  string
+	LocalAddrPort        string
+	MetricsEnabled       bool
+	HealthCheckEnabled   bool
+	DebugAddrPort        string
+	EgressProxiesCfgPath string
+	// CertShareMode is set for Kubernetes Pods running cert share mode.
+	// Possible values are empty (containerboot doesn't run any certs
+	// logic),  'ro' (for Pods that shold never attempt to issue/renew
+	// certs) and 'rw' for Pods that should manage the TLS certs shared
+	// amongst the replicas.
+	CertShareMode string
 }
 
 func configFromEnv() (*settings, error) {
@@ -107,7 +113,7 @@ func configFromEnv() (*settings, error) {
 		MetricsEnabled:                        defaultBool("TS_ENABLE_METRICS", false),
 		HealthCheckEnabled:                    defaultBool("TS_ENABLE_HEALTH_CHECK", false),
 		DebugAddrPort:                         defaultEnv("TS_DEBUG_ADDR_PORT", ""),
-		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
+		EgressProxiesCfgPath:                  defaultEnv("TS_EGRESS_PROXIES_CONFIG_PATH", ""),
 		PodUID:                                defaultEnv("POD_UID", ""),
 	}
 	podIPs, ok := os.LookupEnv("POD_IPS")
@@ -128,6 +134,17 @@ func configFromEnv() (*settings, error) {
 			cfg.PodIPv6 = parsed.String()
 		}
 	}
+	// If cert share is enabled, set the replica as read or write. Only 0th
+	// replica should be able to write.
+	isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false)
+	if isInCertShareMode {
+		cfg.CertShareMode = "ro"
+		podName := os.Getenv("POD_NAME")
+		if strings.HasSuffix(podName, "-0") {
+			cfg.CertShareMode = "rw"
+		}
+	}
+
 	if err := cfg.validate(); err != nil {
 		return nil, fmt.Errorf("invalid configuration: %v", err)
 	}
@@ -186,7 +203,7 @@ func (s *settings) validate() error {
 			return fmt.Errorf("error parsing TS_HEALTHCHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
 		}
 	}
-	if s.localMetricsEnabled() || s.localHealthEnabled() {
+	if s.localMetricsEnabled() || s.localHealthEnabled() || s.EgressProxiesCfgPath != "" {
 		if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
 			return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
 		}
@@ -199,8 +216,8 @@ func (s *settings) validate() error {
 	if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" {
 		return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT")
 	}
-	if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
-		return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
+	if s.EgressProxiesCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
+		return errors.New("TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
 	}
 	return nil
 }
@@ -291,7 +308,7 @@ func isOneStepConfig(cfg *settings) bool {
 // as an L3 proxy, proxying to an endpoint provided via one of the config env
 // vars.
 func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != ""
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressProxiesCfgPath != ""
 }
 
 // hasKubeStateStore returns true if the state must be stored in a Kubernetes
@@ -308,6 +325,10 @@ func (cfg *settings) localHealthEnabled() bool {
 	return cfg.LocalAddrPort != "" && cfg.HealthCheckEnabled
 }
 
+func (cfg *settings) egressSvcsTerminateEPEnabled() bool {
+	return cfg.LocalAddrPort != "" && cfg.EgressProxiesCfgPath != ""
+}
+
 // defaultEnv returns the value of the given envvar name, or defVal if
 // unset.
 func defaultEnv(name, defVal string) string {

cmd/containerboot/tailscaled.go

@@ -13,14 +13,17 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"reflect"
 	"strings"
 	"syscall"
 	"time"
 
-	"tailscale.com/client/tailscale"
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/local"
 )
 
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Process, error) {
 	args := tailscaledArgs(cfg)
 	// tailscaled runs without context, since it needs to persist
 	// beyond the startup timeout in ctx.
@@ -30,6 +33,9 @@ func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
+	if cfg.CertShareMode != "" {
+		cmd.Env = append(os.Environ(), "TS_CERT_SHARE_MODE="+cfg.CertShareMode)
+	}
 	log.Printf("Starting tailscaled")
 	if err := cmd.Start(); err != nil {
 		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
@@ -39,19 +45,19 @@ func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient
 	log.Printf("Waiting for tailscaled socket")
 	for {
 		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
+			return nil, nil, errors.New("timed out waiting for tailscaled socket")
 		}
 		_, err := os.Stat(cfg.Socket)
 		if errors.Is(err, fs.ErrNotExist) {
 			time.Sleep(100 * time.Millisecond)
 			continue
 		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
+			return nil, nil, fmt.Errorf("error waiting for tailscaled socket: %w", err)
 		}
 		break
 	}
 
-	tsClient := &tailscale.LocalClient{
+	tsClient := &local.Client{
 		Socket:        cfg.Socket,
 		UseSocketOnly: true,
 	}
@@ -166,3 +172,75 @@ func tailscaleSet(ctx context.Context, cfg *settings) error {
 	}
 	return nil
 }
+
+func watchTailscaledConfigChanges(ctx context.Context, path string, lc *local.Client, errCh chan<- error) {
+	var (
+		tickChan          <-chan time.Time
+		eventChan         <-chan fsnotify.Event
+		errChan           <-chan error
+		tailscaledCfgDir  = filepath.Dir(path)
+		prevTailscaledCfg []byte
+	)
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		// Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor.
+		// See https://github.com/tailscale/tailscale/issues/15081
+		log.Printf("tailscaled config watch: failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(tailscaledCfgDir); err != nil {
+			errCh <- fmt.Errorf("failed to add fsnotify watch: %w", err)
+			return
+		}
+		eventChan = w.Events
+		errChan = w.Errors
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		errCh <- fmt.Errorf("error reading configfile: %w", err)
+		return
+	}
+	prevTailscaledCfg = b
+	// kubelet mounts Secrets to Pods using a series of symlinks, one of
+	// which is <mount-dir>/..data that Kubernetes recommends consumers to
+	// use if they need to monitor changes
+	// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
+	const kubeletMountedCfg = "..data"
+	toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedCfg)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case err := <-errChan:
+			errCh <- fmt.Errorf("watcher error: %w", err)
+			return
+		case <-tickChan:
+		case event := <-eventChan:
+			if event.Name != toWatch {
+				continue
+			}
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			errCh <- fmt.Errorf("error reading configfile: %w", err)
+			return
+		}
+		// For some proxy types the mounted volume also contains tailscaled state and other files. We
+		// don't want to reload config unnecessarily on unrelated changes to these files.
+		if reflect.DeepEqual(b, prevTailscaledCfg) {
+			continue
+		}
+		prevTailscaledCfg = b
+		log.Printf("tailscaled config watch: ensuring that config is up to date")
+		ok, err := lc.ReloadConfig(ctx)
+		if err != nil {
+			errCh <- fmt.Errorf("error reloading tailscaled config: %w", err)
+			return
+		}
+		if ok {
+			log.Printf("tailscaled config watch: config was reloaded")
+		}
+	}
+}

cmd/derper/bootstrap_dns_test.go

@@ -20,10 +20,10 @@ import (
 )
 
 func BenchmarkHandleBootstrapDNS(b *testing.B) {
-	tstest.Replace(b, bootstrapDNS, "log.tailscale.io,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
+	tstest.Replace(b, bootstrapDNS, "log.tailscale.com,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
 	refreshBootstrapDNS()
 	w := new(bitbucketResponseWriter)
-	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.io"), nil)
+	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.com"), nil)
 	b.ReportAllocs()
 	b.ResetTimer()
 	b.RunParallel(func(b *testing.PB) {
@@ -63,7 +63,7 @@ func TestUnpublishedDNS(t *testing.T) {
 	nettest.SkipIfNoNetwork(t)
 
 	const published = "login.tailscale.com"
-	const unpublished = "log.tailscale.io"
+	const unpublished = "log.tailscale.com"
 
 	prev1, prev2 := *bootstrapDNS, *unpublishedDNS
 	*bootstrapDNS = published
@@ -119,18 +119,18 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 
 	unpublishedDNSCache.Store(&dnsEntryMap{
 		IPs: map[string][]net.IP{
-			"log.tailscale.io":           {},
+			"log.tailscale.com":          {},
 			"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
 		},
 		Percent: map[string]float64{
-			"log.tailscale.io":           1.0,
+			"log.tailscale.com":          1.0,
 			"controlplane.tailscale.com": 1.0,
 		},
 	})
 
 	t.Run("CacheMiss", func(t *testing.T) {
 		// One domain in map but empty, one not in map at all
-		for _, q := range []string{"log.tailscale.io", "login.tailscale.com"} {
+		for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} {
 			resetMetrics()
 			ips := getBootstrapDNS(t, q)
 

cmd/derper/cert.go

@@ -4,16 +4,28 @@
 package main
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
+	"log"
+	"math/big"
 	"net"
 	"net/http"
+	"os"
 	"path/filepath"
 	"regexp"
+	"time"
 
 	"golang.org/x/crypto/acme/autocert"
+	"tailscale.com/tailcfg"
 )
 
 var unsafeHostnameCharacters = regexp.MustCompile(`[^a-zA-Z0-9-\.]`)
@@ -65,8 +77,18 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	crtPath := filepath.Join(certdir, keyname+".crt")
 	keyPath := filepath.Join(certdir, keyname+".key")
 	cert, err := tls.LoadX509KeyPair(crtPath, keyPath)
+	hostnameIP := net.ParseIP(hostname) // or nil if hostname isn't an IP address
 	if err != nil {
-		return nil, fmt.Errorf("can not load x509 key pair for hostname %q: %w", keyname, err)
+		// If the hostname is an IP address, automatically create a
+		// self-signed certificate for it.
+		var certp *tls.Certificate
+		if os.IsNotExist(err) && hostnameIP != nil {
+			certp, err = createSelfSignedIPCert(crtPath, keyPath, hostname)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("can not load x509 key pair for hostname %q: %w", keyname, err)
+		}
+		cert = *certp
 	}
 	// ensure hostname matches with the certificate
 	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
@@ -76,6 +98,18 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	if err := x509Cert.VerifyHostname(hostname); err != nil {
 		return nil, fmt.Errorf("cert invalid for hostname %q: %w", hostname, err)
 	}
+	if hostnameIP != nil {
+		// If the hostname is an IP address, print out information on how to
+		// confgure this in the derpmap.
+		dn := &tailcfg.DERPNode{
+			Name:     "custom",
+			RegionID: 900,
+			HostName: hostname,
+			CertName: fmt.Sprintf("sha256-raw:%-02x", sha256.Sum256(x509Cert.Raw)),
+		}
+		dnJSON, _ := json.Marshal(dn)
+		log.Printf("Using self-signed certificate for IP address %q. Configure it in DERPMap using: (https://tailscale.com/s/custom-derp)\n  %s", hostname, dnJSON)
+	}
 	return &manualCertManager{
 		cert:       &cert,
 		hostname:   hostname,
@@ -109,3 +143,69 @@ func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certif
 func (m *manualCertManager) HTTPHandler(fallback http.Handler) http.Handler {
 	return fallback
 }
+
+func createSelfSignedIPCert(crtPath, keyPath, ipStr string) (*tls.Certificate, error) {
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return nil, fmt.Errorf("invalid IP address: %s", ipStr)
+	}
+
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate EC private key: %v", err)
+	}
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate serial number: %v", err)
+	}
+
+	now := time.Now()
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName: ipStr,
+		},
+		NotBefore: now,
+		NotAfter:  now.AddDate(1, 0, 0), // expires in 1 year; a bit over that is rejected by macOS etc
+
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	// Set the IP as a SAN.
+	template.IPAddresses = []net.IP{ip}
+
+	// Create the self-signed certificate.
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create certificate: %v", err)
+	}
+
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+
+	keyBytes, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal EC private key: %v", err)
+	}
+
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes})
+
+	if err := os.MkdirAll(filepath.Dir(crtPath), 0700); err != nil {
+		return nil, fmt.Errorf("failed to create directory for certificate: %v", err)
+	}
+	if err := os.WriteFile(crtPath, certPEM, 0644); err != nil {
+		return nil, fmt.Errorf("failed to write certificate to %s: %v", crtPath, err)
+	}
+	if err := os.WriteFile(keyPath, keyPEM, 0600); err != nil {
+		return nil, fmt.Errorf("failed to write key to %s: %v", keyPath, err)
+	}
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tls.Certificate: %v", err)
+	}
+	return &tlsCert, nil
+}

cmd/derper/cert_test.go

@@ -4,19 +4,29 @@
 package main
 
 import (
+	"context"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"fmt"
 	"math/big"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/net/netmon"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 // Verify that in --certmode=manual mode, we can use a bare IP address
@@ -95,3 +105,66 @@ func TestCertIP(t *testing.T) {
 		t.Fatalf("GetCertificate returned nil")
 	}
 }
+
+// Test that we can dial a raw IP without using a hostname and without a WebPKI
+// cert, validating the cert against the signature of the cert in the DERP map's
+// DERPNode.
+//
+// See https://github.com/tailscale/tailscale/issues/11776.
+func TestPinnedCertRawIP(t *testing.T) {
+	td := t.TempDir()
+	cp, err := NewManualCertManager(td, "127.0.0.1")
+	if err != nil {
+		t.Fatalf("NewManualCertManager: %v", err)
+	}
+
+	cert, err := cp.TLSConfig().GetCertificate(&tls.ClientHelloInfo{
+		ServerName: "127.0.0.1",
+	})
+	if err != nil {
+		t.Fatalf("GetCertificate: %v", err)
+	}
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("Listen: %v", err)
+	}
+	defer ln.Close()
+
+	ds := derp.NewServer(key.NewNode(), t.Logf)
+
+	derpHandler := derphttp.Handler(ds)
+	mux := http.NewServeMux()
+	mux.Handle("/derp", derpHandler)
+
+	var hs http.Server
+	hs.Handler = mux
+	hs.TLSConfig = cp.TLSConfig()
+	go hs.ServeTLS(ln, "", "")
+
+	lnPort := ln.Addr().(*net.TCPAddr).Port
+
+	reg := &tailcfg.DERPRegion{
+		RegionID: 900,
+		Nodes: []*tailcfg.DERPNode{
+			{
+				RegionID: 900,
+				HostName: "127.0.0.1",
+				CertName: fmt.Sprintf("sha256-raw:%-02x", sha256.Sum256(cert.Leaf.Raw)),
+				DERPPort: lnPort,
+			},
+		},
+	}
+
+	netMon := netmon.NewStatic()
+	dc := derphttp.NewRegionClient(key.NewNode(), t.Logf, netMon, func() *tailcfg.DERPRegion {
+		return reg
+	})
+	defer dc.Close()
+
+	_, connClose, _, err := dc.DialRegionTLS(context.Background(), reg)
+	if err != nil {
+		t.Fatalf("DialRegionTLS: %v", err)
+	}
+	defer connClose.Close()
+}

cmd/derper/depaware.txt

@@ -28,7 +28,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
-   L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
    L 💣 github.com/mdlayher/netlink                                  from github.com/google/nftables+
@@ -36,11 +35,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
+        github.com/munnerz/goautoneg                                 from github.com/prometheus/common/expfmt
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
         github.com/prometheus/common/expfmt                          from github.com/prometheus/client_golang/prometheus+
-        github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
         github.com/prometheus/common/model                           from github.com/prometheus/client_golang/prometheus+
   LD    github.com/prometheus/procfs                                 from github.com/prometheus/client_golang/prometheus
   LD    github.com/prometheus/procfs/internal/fs                     from github.com/prometheus/procfs
@@ -52,9 +51,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
    L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
+        github.com/tailscale/setec/client/setec                      from tailscale.com/cmd/derper
+        github.com/tailscale/setec/types/api                         from github.com/tailscale/setec/client/setec
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
-     💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
+     💣 go4.org/mem                                                  from tailscale.com/client/local+
         go4.org/netipx                                               from tailscale.com/net/tsaddr
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         google.golang.org/protobuf/encoding/protodelim               from github.com/prometheus/common/expfmt
@@ -86,27 +87,29 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
-        tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
+     💣 tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
+        tailscale.com/client/local                                   from tailscale.com/client/tailscale+
         tailscale.com/client/tailscale                               from tailscale.com/derp
-        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale
+        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/derp                                           from tailscale.com/cmd/derper+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/derper
         tailscale.com/disco                                          from tailscale.com/derp
-        tailscale.com/drive                                          from tailscale.com/client/tailscale+
-        tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/drive                                          from tailscale.com/client/local+
+        tailscale.com/envknob                                        from tailscale.com/client/local+
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
-        tailscale.com/ipn                                            from tailscale.com/client/tailscale
-        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/ipn                                            from tailscale.com/client/local
+        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/local+
         tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
      💣 tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
      💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
-        tailscale.com/net/netutil                                    from tailscale.com/client/tailscale
+        tailscale.com/net/netutil                                    from tailscale.com/client/local
         tailscale.com/net/sockstats                                  from tailscale.com/derp/derphttp
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
@@ -116,11 +119,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper
-        tailscale.com/paths                                          from tailscale.com/client/tailscale
-     💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
+        tailscale.com/paths                                          from tailscale.com/client/local
+     💣 tailscale.com/safesocket                                     from tailscale.com/client/local
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
-        tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
-        tailscale.com/tka                                            from tailscale.com/client/tailscale+
+        tailscale.com/tailcfg                                        from tailscale.com/client/local+
+        tailscale.com/tka                                            from tailscale.com/client/local+
    W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
@@ -131,7 +134,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
-        tailscale.com/types/key                                      from tailscale.com/client/tailscale+
+        tailscale.com/types/key                                      from tailscale.com/client/local+
         tailscale.com/types/lazy                                     from tailscale.com/version+
         tailscale.com/types/logger                                   from tailscale.com/cmd/derper+
         tailscale.com/types/netmap                                   from tailscale.com/ipn
@@ -141,7 +144,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
         tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
-        tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
+        tailscale.com/types/tkatype                                  from tailscale.com/client/local+
         tailscale.com/types/views                                    from tailscale.com/ipn+
         tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/net/netmon+
@@ -188,11 +191,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls+
+        golang.org/x/crypto/internal/alias                           from golang.org/x/crypto/chacha20+
+        golang.org/x/crypto/internal/poly1305                        from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
         golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
@@ -201,10 +204,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/net/http/httpproxy                              from net/http+
         golang.org/x/net/http2/hpack                                 from net/http
         golang.org/x/net/idna                                        from golang.org/x/crypto/acme/autocert+
+        golang.org/x/net/internal/socks                              from golang.org/x/net/proxy
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
         golang.org/x/sync/errgroup                                   from github.com/mdlayher/socket+
-        golang.org/x/sys/cpu                                         from github.com/josharian/native+
+        golang.org/x/sync/singleflight                               from github.com/tailscale/setec/client/setec
+        golang.org/x/sys/cpu                                         from golang.org/x/crypto/argon2+
   LD    golang.org/x/sys/unix                                        from github.com/google/nftables+
    W    golang.org/x/sys/windows                                     from github.com/dblohm7/wingoes+
    W    golang.org/x/sys/windows/registry                            from github.com/dblohm7/wingoes+
@@ -223,7 +228,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
         crypto                                                       from crypto/ecdh+
-        crypto/aes                                                   from crypto/ecdsa+
+        crypto/aes                                                   from crypto/internal/hpke+
         crypto/cipher                                                from crypto/aes+
         crypto/des                                                   from crypto/tls+
         crypto/dsa                                                   from crypto/x509
@@ -232,18 +237,58 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         crypto/ed25519                                               from crypto/tls+
         crypto/elliptic                                              from crypto/ecdsa+
         crypto/hmac                                                  from crypto/tls+
+        crypto/internal/boring                                       from crypto/aes+
+        crypto/internal/boring/bbig                                  from crypto/ecdsa+
+        crypto/internal/boring/sig                                   from crypto/internal/boring
+        crypto/internal/entropy                                      from crypto/internal/fips140/drbg
+        crypto/internal/fips140                                      from crypto/internal/fips140/aes+
+        crypto/internal/fips140/aes                                  from crypto/aes+
+        crypto/internal/fips140/aes/gcm                              from crypto/cipher+
+        crypto/internal/fips140/alias                                from crypto/cipher+
+        crypto/internal/fips140/bigmod                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/check                                from crypto/internal/fips140/aes+
+        crypto/internal/fips140/drbg                                 from crypto/internal/fips140/aes/gcm+
+        crypto/internal/fips140/ecdh                                 from crypto/ecdh
+        crypto/internal/fips140/ecdsa                                from crypto/ecdsa
+        crypto/internal/fips140/ed25519                              from crypto/ed25519
+        crypto/internal/fips140/edwards25519                         from crypto/internal/fips140/ed25519
+        crypto/internal/fips140/edwards25519/field                   from crypto/ecdh+
+        crypto/internal/fips140/hkdf                                 from crypto/internal/fips140/tls13+
+        crypto/internal/fips140/hmac                                 from crypto/hmac+
+        crypto/internal/fips140/mlkem                                from crypto/tls
+        crypto/internal/fips140/nistec                               from crypto/elliptic+
+        crypto/internal/fips140/nistec/fiat                          from crypto/internal/fips140/nistec
+        crypto/internal/fips140/rsa                                  from crypto/rsa
+        crypto/internal/fips140/sha256                               from crypto/internal/fips140/check+
+        crypto/internal/fips140/sha3                                 from crypto/internal/fips140/hmac+
+        crypto/internal/fips140/sha512                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/subtle                               from crypto/internal/fips140/aes+
+        crypto/internal/fips140/tls12                                from crypto/tls
+        crypto/internal/fips140/tls13                                from crypto/tls
+        crypto/internal/fips140deps/byteorder                        from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/cpu                              from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/godebug                          from crypto/internal/fips140+
+        crypto/internal/fips140hash                                  from crypto/ecdsa+
+        crypto/internal/fips140only                                  from crypto/cipher+
+        crypto/internal/hpke                                         from crypto/tls
+        crypto/internal/impl                                         from crypto/internal/fips140/aes+
+        crypto/internal/randutil                                     from crypto/dsa+
+        crypto/internal/sysrand                                      from crypto/internal/entropy+
         crypto/md5                                                   from crypto/tls+
         crypto/rand                                                  from crypto/ed25519+
         crypto/rc4                                                   from crypto/tls
         crypto/rsa                                                   from crypto/tls+
         crypto/sha1                                                  from crypto/tls+
         crypto/sha256                                                from crypto/tls+
+        crypto/sha3                                                  from crypto/internal/fips140hash
         crypto/sha512                                                from crypto/ecdsa+
-        crypto/subtle                                                from crypto/aes+
+        crypto/subtle                                                from crypto/cipher+
         crypto/tls                                                   from golang.org/x/crypto/acme+
+        crypto/tls/internal/fips140tls                               from crypto/tls
         crypto/x509                                                  from crypto/tls+
+   D    crypto/x509/internal/macos                                   from crypto/x509
         crypto/x509/pkix                                             from crypto/x509+
-        embed                                                        from crypto/internal/nistec+
+        embed                                                        from google.golang.org/protobuf/internal/editiondefaults+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base32                                              from github.com/fxamacker/cbor/v2+
@@ -263,9 +308,49 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
         html/template                                                from tailscale.com/cmd/derper
+        internal/abi                                                 from crypto/x509/internal/macos+
+        internal/asan                                                from syscall+
+        internal/bisect                                              from internal/godebug
+        internal/bytealg                                             from bytes+
+        internal/byteorder                                           from crypto/cipher+
+        internal/chacha8rand                                         from math/rand/v2+
+        internal/coverage/rtcov                                      from runtime
+        internal/cpu                                                 from crypto/internal/fips140deps/cpu+
+        internal/filepathlite                                        from os+
+        internal/fmtsort                                             from fmt+
+        internal/goarch                                              from crypto/internal/fips140deps/cpu+
+        internal/godebug                                             from crypto/tls+
+        internal/godebugs                                            from internal/godebug+
+        internal/goexperiment                                        from runtime+
+        internal/goos                                                from crypto/x509+
+        internal/itoa                                                from internal/poll+
+        internal/msan                                                from syscall+
+        internal/nettrace                                            from net+
+        internal/oserror                                             from io/fs+
+        internal/poll                                                from net+
+        internal/profile                                             from net/http/pprof
+        internal/profilerecord                                       from runtime+
+        internal/race                                                from internal/poll+
+        internal/reflectlite                                         from context+
+        internal/runtime/atomic                                      from internal/runtime/exithook+
+        internal/runtime/exithook                                    from runtime
+        internal/runtime/maps                                        from reflect+
+        internal/runtime/math                                        from internal/runtime/maps+
+        internal/runtime/sys                                         from crypto/subtle+
+   L    internal/runtime/syscall                                     from runtime+
+        internal/singleflight                                        from net
+        internal/stringslite                                         from embed+
+        internal/sync                                                from sync+
+        internal/syscall/execenv                                     from os+
+  LD    internal/syscall/unix                                        from crypto/internal/sysrand+
+   W    internal/syscall/windows                                     from crypto/internal/sysrand+
+   W    internal/syscall/windows/registry                            from mime+
+   W    internal/syscall/windows/sysdll                              from internal/syscall/windows+
+        internal/testlog                                             from os
+        internal/unsafeheader                                        from internal/reflectlite+
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
-        io/ioutil                                                    from github.com/mitchellh/go-ps+
+   L    io/ioutil                                                    from github.com/mitchellh/go-ps+
         iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
@@ -274,7 +359,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from internal/concurrent+
+        math/rand/v2                                                 from crypto/ecdsa+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -282,11 +367,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
+        net/http/internal/ascii                                      from net/http
         net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
-        os                                                           from crypto/rand+
+        os                                                           from crypto/internal/sysrand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
    W    os/user                                                      from tailscale.com/util/winutil+
@@ -295,6 +381,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         reflect                                                      from crypto/x509+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
+        runtime                                                      from crypto/internal/fips140+
         runtime/debug                                                from github.com/prometheus/client_golang/prometheus+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof
@@ -305,7 +392,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         strings                                                      from bufio+
         sync                                                         from compress/flate+
         sync/atomic                                                  from context+
-        syscall                                                      from crypto/rand+
+        syscall                                                      from crypto/internal/sysrand+
         text/tabwriter                                               from runtime/pprof
         text/template                                                from html/template
         text/template/parse                                          from html/template+
@@ -314,3 +401,5 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
         unique                                                       from net/netip
+        unsafe                                                       from bytes+
+        weak                                                         from unique

cmd/derper/derper.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -36,6 +37,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/tailscale/setec/client/setec"
 	"golang.org/x/time/rate"
 	"tailscale.com/atomicfile"
 	"tailscale.com/derp"
@@ -61,15 +63,22 @@ var (
 	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks")
 	runSTUN     = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
 	runDERP     = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
+	flagHome    = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to")
 
 	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
-	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
+	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.")
+	secretsURL      = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key")
+	secretPrefix    = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key")
+	secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)")
 	bootstrapDNS    = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
 	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
+
 	verifyClients   = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 	verifyClientURL = flag.String("verify-client-url", "", "if non-empty, an admission controller URL for permitting client connections; see tailcfg.DERPAdmitClientRequest")
 	verifyFailOpen  = flag.Bool("verify-client-url-fail-open", true, "whether we fail open if --verify-client-url is unreachable")
 
+	socket = flag.String("socket", "", "optional alternate path to tailscaled socket (only relevant when using --verify-clients)")
+
 	acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection")
 	acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection")
 
@@ -77,13 +86,21 @@ var (
 	tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time")
 	// tcpUserTimeout is intentionally short, so that hung connections are cleaned up promptly. DERPs should be nearby users.
 	tcpUserTimeout = flag.Duration("tcp-user-timeout", 15*time.Second, "TCP user timeout")
+	// tcpWriteTimeout is the timeout for writing to client TCP connections. It does not apply to mesh connections.
+	tcpWriteTimeout = flag.Duration("tcp-write-timeout", derp.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes")
 )
 
 var (
 	tlsRequestVersion = &metrics.LabelMap{Label: "version"}
 	tlsActiveVersion  = &metrics.LabelMap{Label: "version"}
+
+	// Exactly 64 hexadecimal lowercase digits.
+	validMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`)
 )
 
+const setecMeshKeyName = "meshkey"
+const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY"
+
 func init() {
 	expvar.Publish("derper_tls_request_version", tlsRequestVersion)
 	expvar.Publish("gauge_derper_tls_active_version", tlsActiveVersion)
@@ -139,6 +156,14 @@ func writeNewConfig() config {
 	return cfg
 }
 
+func checkMeshKey(key string) (string, error) {
+	key = strings.TrimSpace(key)
+	if !validMeshKey.MatchString(key) {
+		return "", errors.New("key must contain exactly 64 hex digits")
+	}
+	return key, nil
+}
+
 func main() {
 	flag.Parse()
 	if *versionFlag {
@@ -171,26 +196,70 @@ func main() {
 
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)
+	s.SetTailscaledSocketPath(*socket)
 	s.SetVerifyClientURL(*verifyClientURL)
 	s.SetVerifyClientURLFailOpen(*verifyFailOpen)
+	s.SetTCPWriteTimeout(*tcpWriteTimeout)
 
-	if *meshPSKFile != "" {
-		b, err := os.ReadFile(*meshPSKFile)
+	var meshKey string
+	if *dev {
+		meshKey = os.Getenv(meshKeyEnvVar)
+		if meshKey == "" {
+			log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar)
+		} else {
+			log.Printf("Set mesh key from %s\n", meshKeyEnvVar)
+		}
+	} else if *secretsURL != "" {
+		meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName)
+		fc, err := setec.NewFileCache(*secretsCacheDir)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("NewFileCache: %v", err)
 		}
-		key := strings.TrimSpace(string(b))
-		if matched, _ := regexp.MatchString(`(?i)^[0-9a-f]{64,}$`, key); !matched {
-			log.Fatalf("key in %s must contain 64+ hex digits", *meshPSKFile)
+		log.Printf("Setting up setec store from %q", *secretsURL)
+		st, err := setec.NewStore(ctx,
+			setec.StoreConfig{
+				Client: setec.Client{Server: *secretsURL},
+				Secrets: []string{
+					meshKeySecret,
+				},
+				Cache: fc,
+			})
+		if err != nil {
+			log.Fatalf("NewStore: %v", err)
 		}
-		s.SetMeshKey(key)
-		log.Printf("DERP mesh key configured")
+		meshKey = st.Secret(meshKeySecret).GetString()
+		log.Println("Got mesh key from setec store")
+		st.Close()
+	} else if *meshPSKFile != "" {
+		b, err := setec.StaticFile(*meshPSKFile)
+		if err != nil {
+			log.Fatalf("StaticFile failed to get key: %v", err)
+		}
+		log.Println("Got mesh key from static file")
+		meshKey = b.GetString()
 	}
+
+	if meshKey == "" && *dev {
+		log.Printf("No mesh key configured for --dev mode")
+	} else if meshKey == "" {
+		log.Printf("No mesh key configured")
+	} else if key, err := checkMeshKey(meshKey); err != nil {
+		log.Fatalf("invalid mesh key: %v", err)
+	} else {
+		s.SetMeshKey(key)
+		log.Println("DERP mesh key configured")
+	}
+
 	if err := startMesh(s); err != nil {
 		log.Fatalf("startMesh: %v", err)
 	}
 	expvar.Publish("derp", s.ExpVar())
 
+	handleHome, ok := getHomeHandler(*flagHome)
+	if !ok {
+		log.Fatalf("unknown --home value %q", *flagHome)
+	}
+
 	mux := http.NewServeMux()
 	if *runDERP {
 		derpHandler := derphttp.Handler(s)
@@ -211,19 +280,7 @@ func main() {
 	mux.HandleFunc("/bootstrap-dns", tsweb.BrowserHeaderHandlerFunc(handleBootstrapDNS))
 	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tsweb.AddBrowserHeaders(w)
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		w.WriteHeader(200)
-		err := homePageTemplate.Execute(w, templateData{
-			ShowAbuseInfo: validProdHostname.MatchString(*hostname),
-			Disabled:      !*runDERP,
-			AllowDebug:    tsweb.AllowDebugAccess(r),
-		})
-		if err != nil {
-			if r.Context().Err() == nil {
-				log.Printf("homePageTemplate.Execute: %v", err)
-			}
-			return
-		}
+		handleHome.ServeHTTP(w, r)
 	}))
 	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tsweb.AddBrowserHeaders(w)
@@ -265,6 +322,9 @@ func main() {
 		Control:   ktimeout.UserTimeout(*tcpUserTimeout),
 		KeepAlive: *tcpKeepAlive,
 	}
+	// As of 2025-02-19, MPTCP does not support TCP_USER_TIMEOUT socket option
+	// set in ktimeout.UserTimeout above.
+	lc.SetMultipathTCP(false)
 
 	quietLogger := log.New(logger.HTTPServerLogFilter{Inner: log.Printf}, "", 0)
 	httpsrv := &http.Server{
@@ -379,6 +439,10 @@ func prodAutocertHostPolicy(_ context.Context, host string) error {
 	return errors.New("invalid hostname")
 }
 
+func defaultSetecCacheDir() string {
+	return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets")
+}
+
 func defaultMeshPSKFile() string {
 	try := []string{
 		"/home/derp/keys/derp-mesh.key",
@@ -509,3 +573,35 @@ var homePageTemplate = template.Must(template.New("home").Parse(`<html><body>
 </body>
 </html>
 `))
+
+// getHomeHandler returns a handler for the home page based on a flag string
+// as documented on the --home flag.
+func getHomeHandler(val string) (_ http.Handler, ok bool) {
+	if val == "" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.WriteHeader(200)
+			err := homePageTemplate.Execute(w, templateData{
+				ShowAbuseInfo: validProdHostname.MatchString(*hostname),
+				Disabled:      !*runDERP,
+				AllowDebug:    tsweb.AllowDebugAccess(r),
+			})
+			if err != nil {
+				if r.Context().Err() == nil {
+					log.Printf("homePageTemplate.Execute: %v", err)
+				}
+				return
+			}
+		}), true
+	}
+	if val == "blank" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.WriteHeader(200)
+		}), true
+	}
+	if strings.HasPrefix(val, "http://") || strings.HasPrefix(val, "https://") {
+		return http.RedirectHandler(val, http.StatusFound), true
+	}
+	return nil, false
+}

cmd/derper/derper_test.go

@@ -138,3 +138,46 @@ func TestTemplate(t *testing.T) {
 		t.Error("Output is missing debug info")
 	}
 }
+
+func TestCheckMeshKey(t *testing.T) {
+	testCases := []struct {
+		name    string
+		input   string
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "KeyOkay",
+			input:   "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			wantErr: false,
+		},
+		{
+			name:    "TrimKeyOkay",
+			input:   "  f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6  ",
+			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			wantErr: false,
+		},
+		{
+			name:    "NotAKey",
+			input:   "zzthisisnotakey",
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			k, err := checkMeshKey(tt.input)
+			if err != nil && !tt.wantErr {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if k != tt.want && err == nil {
+				t.Errorf("want: %s doesn't match expected: %s", tt.want, k)
+			}
+
+		})
+	}
+
+}

cmd/derper/mesh.go

@@ -10,7 +10,6 @@ import (
 	"log"
 	"net"
 	"strings"
-	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -25,15 +24,28 @@ func startMesh(s *derp.Server) error {
 	if !s.HasMeshKey() {
 		return errors.New("--mesh-with requires --mesh-psk-file")
 	}
-	for _, host := range strings.Split(*meshWith, ",") {
-		if err := startMeshWithHost(s, host); err != nil {
+	for _, hostTuple := range strings.Split(*meshWith, ",") {
+		if err := startMeshWithHost(s, hostTuple); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func startMeshWithHost(s *derp.Server, host string) error {
+func startMeshWithHost(s *derp.Server, hostTuple string) error {
+	var host string
+	var dialHost string
+	hostParts := strings.Split(hostTuple, "/")
+	if len(hostParts) > 2 {
+		return fmt.Errorf("too many components in host tuple %q", hostTuple)
+	}
+	host = hostParts[0]
+	if len(hostParts) == 2 {
+		dialHost = hostParts[1]
+	} else {
+		dialHost = hostParts[0]
+	}
+
 	logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
 	netMon := netmon.NewStatic() // good enough for cmd/derper; no need for netns fanciness
 	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf, netMon)
@@ -43,31 +55,20 @@ func startMeshWithHost(s *derp.Server, host string) error {
 	c.MeshKey = s.MeshKey()
 	c.WatchConnectionChanges = true
 
-	// For meshed peers within a region, connect via VPC addresses.
-	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
-		host, port, err := net.SplitHostPort(addr)
-		if err != nil {
-			return nil, err
-		}
+	logf("will dial %q for %q", dialHost, host)
+	if dialHost != host {
 		var d net.Dialer
-		var r net.Resolver
-		if base, ok := strings.CutSuffix(host, ".tailscale.com"); ok && port == "443" {
-			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
-			defer cancel()
-			vpcHost := base + "-vpc.tailscale.com"
-			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
-			if len(ips) > 0 {
-				vpcAddr := net.JoinHostPort(ips[0].String(), port)
-				c, err := d.DialContext(subCtx, network, vpcAddr)
-				if err == nil {
-					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
-					return c, nil
-				}
-				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+		c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
+			_, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				logf("failed to split %q: %v", addr, err)
+				return nil, err
 			}
-		}
-		return d.DialContext(ctx, network, addr)
-	})
+			dialAddr := net.JoinHostPort(dialHost, port)
+			logf("dialing %q instead of %q", dialAddr, addr)
+			return d.DialContext(ctx, network, dialAddr)
+		})
+	}
 
 	add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) }
 	remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) }

cmd/derpprobe/derpprobe.go

@@ -18,18 +18,21 @@ import (
 )
 
 var (
-	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
-	versionFlag  = flag.Bool("version", false, "print version and exit")
-	listen       = flag.String("listen", ":8030", "HTTP listen address")
-	probeOnce    = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
-	spread       = flag.Bool("spread", true, "whether to spread probing over time")
-	interval     = flag.Duration("interval", 15*time.Second, "probe interval")
-	meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
-	stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
-	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
-	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
-	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
-	regionCode   = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
+	derpMapURL         = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
+	versionFlag        = flag.Bool("version", false, "print version and exit")
+	listen             = flag.String("listen", ":8030", "HTTP listen address")
+	probeOnce          = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
+	spread             = flag.Bool("spread", true, "whether to spread probing over time")
+	interval           = flag.Duration("interval", 15*time.Second, "probe interval")
+	meshInterval       = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
+	stunInterval       = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
+	tlsInterval        = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
+	bwInterval         = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
+	bwSize             = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	bwTUNIPv4Address   = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP; we will use a /30 subnet including this IP address")
+	qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second")
+	qdPacketTimeout    = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings")
+	regionCodeOrID     = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -44,12 +47,13 @@ func main() {
 		prober.WithMeshProbing(*meshInterval),
 		prober.WithSTUNProbing(*stunInterval),
 		prober.WithTLSProbing(*tlsInterval),
+		prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout),
 	}
 	if *bwInterval > 0 {
-		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
+		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address))
 	}
-	if *regionCode != "" {
-		opts = append(opts, prober.WithRegion(*regionCode))
+	if *regionCodeOrID != "" {
+		opts = append(opts, prober.WithRegionCodeOrID(*regionCodeOrID))
 	}
 	dp, err := prober.DERP(p, *derpMapURL, opts...)
 	if err != nil {
@@ -106,7 +110,7 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 			// Do not show probes that have not finished yet.
 			continue
 		}
-		if i.Result {
+		if i.Status == prober.ProbeStatusSucceeded {
 			o.addGoodf("%s: %s", p, i.Latency)
 		} else {
 			o.addBadf("%s: %s", p, i.Error)

cmd/get-authkey/main.go

@@ -16,14 +16,10 @@ import (
 	"strings"
 
 	"golang.org/x/oauth2/clientcredentials"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/internal/client/tailscale"
 )
 
 func main() {
-	// Required to use our client API. We're fine with the instability since the
-	// client lives in the same repo as this code.
-	tailscale.I_Acknowledge_This_API_Is_Unstable = true
-
 	reusable := flag.Bool("reusable", false, "allocate a reusable authkey")
 	ephemeral := flag.Bool("ephemeral", false, "allocate an ephemeral authkey")
 	preauth := flag.Bool("preauth", true, "set the authkey as pre-authorized")

cmd/gitops-pusher/gitops-pusher.go

@@ -13,6 +13,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -405,7 +406,8 @@ func getACLETag(ctx context.Context, client *http.Client, tailnet, apiKey string
 	got := resp.StatusCode
 	want := http.StatusOK
 	if got != want {
-		return "", fmt.Errorf("wanted HTTP status code %d but got %d", want, got)
+		errorDetails, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("wanted HTTP status code %d but got %d: %#q", want, got, string(errorDetails))
 	}
 
 	return Shuck(resp.Header.Get("ETag")), nil

cmd/hello/hello.go

@@ -18,8 +18,9 @@ import (
 	"strings"
 	"time"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/tailcfg"
 )
 
 var (
@@ -31,7 +32,7 @@ var (
 //go:embed hello.tmpl.html
 var embeddedTemplate string
 
-var localClient tailscale.LocalClient
+var localClient local.Client
 
 func main() {
 	flag.Parse()
@@ -134,6 +135,10 @@ func tailscaleIP(who *apitype.WhoIsResponse) string {
 	if who == nil {
 		return ""
 	}
+	vals, err := tailcfg.UnmarshalNodeCapJSON[string](who.Node.CapMap, tailcfg.NodeAttrNativeIPV4)
+	if err == nil && len(vals) > 0 {
+		return vals[0]
+	}
 	for _, nodeIP := range who.Node.Addresses {
 		if nodeIP.Addr().Is4() && nodeIP.IsSingleIP() {
 			return nodeIP.Addr().String()

cmd/k8s-operator/connector_test.go

@@ -79,8 +79,8 @@ func TestConnector(t *testing.T) {
 		subnetRoutes: "10.40.0.0/14",
 		app:          kubetypes.AppConnector,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Connector status should get updated with the IP/hostname info when available.
 	const hostname = "foo.tailnetxyz.ts.net"
@@ -106,7 +106,7 @@ func TestConnector(t *testing.T) {
 	opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20"
 	expectReconciled(t, cr, "", "test")
 
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Remove a route.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -114,7 +114,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.subnetRoutes = "10.44.0.0/20"
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Remove the subnet router.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -122,7 +122,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.subnetRoutes = ""
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Re-add the subnet router.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -132,7 +132,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.subnetRoutes = "10.44.0.0/20"
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Delete the Connector.
 	if err = fc.Delete(context.Background(), cn); err != nil {
@@ -175,8 +175,8 @@ func TestConnector(t *testing.T) {
 		hostname:     "test-connector",
 		app:          kubetypes.AppConnector,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Add an exit node.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -184,7 +184,7 @@ func TestConnector(t *testing.T) {
 	})
 	opts.isExitNode = true
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// Delete the Connector.
 	if err = fc.Delete(context.Background(), cn); err != nil {
@@ -203,7 +203,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
 		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
+			Labels:      tsapi.Labels{"foo": "bar"},
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
@@ -261,8 +261,8 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		subnetRoutes: "10.40.0.0/14",
 		app:          kubetypes.AppConnector,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet
 	// ready, so its configuration is NOT applied to the Connector
@@ -271,7 +271,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		conn.Spec.ProxyClass = "custom-metadata"
 	})
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 3. ProxyClass is set to Ready by proxy-class reconciler. Connector
 	// get reconciled and configuration from the ProxyClass is applied to
@@ -286,7 +286,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	})
 	opts.proxyClass = pc.Name
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 4. Connector.spec.proxyClass field is unset, Connector gets
 	// reconciled and configuration from the ProxyClass is removed from the
@@ -296,7 +296,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	})
 	opts.proxyClass = ""
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 }
 
 func TestConnectorWithAppConnector(t *testing.T) {
@@ -351,8 +351,8 @@ func TestConnectorWithAppConnector(t *testing.T) {
 		app:            kubetypes.AppConnector,
 		isAppConnector: true,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 	// Connector's ready condition should be set to true
 
 	cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
@@ -364,7 +364,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
 		Reason:             reasonConnectorCreated,
 		Message:            reasonConnectorCreated,
 	}}
-	expectEqual(t, fc, cn, nil)
+	expectEqual(t, fc, cn)
 
 	// 2. Connector with invalid app connector routes has status set to invalid
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -379,7 +379,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
 		Reason:             reasonConnectorInvalid,
 		Message:            "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
 	}}
-	expectEqual(t, fc, cn, nil)
+	expectEqual(t, fc, cn)
 
 	// 3. Connector with valid app connnector routes becomes ready
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {

cmd/k8s-operator/depaware.txt

@@ -9,7 +9,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/aws/defaults                    from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/aws/middleware                  from github.com/aws/aws-sdk-go-v2/aws/retry+
-   L    github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics  from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/query              from github.com/aws/aws-sdk-go-v2/service/sts
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/restjson           from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/xml                from github.com/aws/aws-sdk-go-v2/service/sts
@@ -31,10 +30,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/aws-sdk-go-v2/internal/auth                   from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
    L    github.com/aws/aws-sdk-go-v2/internal/auth/smithy            from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/internal/configsources          from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/context                from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/aws-sdk-go-v2/internal/endpoints              from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn   from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2           from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
    L    github.com/aws/aws-sdk-go-v2/internal/ini                    from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/internal/middleware             from github.com/aws/aws-sdk-go-v2/service/sso+
    L    github.com/aws/aws-sdk-go-v2/internal/rand                   from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdk                    from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdkio                  from github.com/aws/aws-sdk-go-v2/credentials/processcreds
@@ -69,16 +70,17 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/smithy-go/internal/sync/singleflight          from github.com/aws/smithy-go/auth/bearer
    L    github.com/aws/smithy-go/io                                  from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+
    L    github.com/aws/smithy-go/logging                             from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/smithy-go/metrics                             from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/smithy-go/middleware                          from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/smithy-go/private/requestcompression          from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/smithy-go/ptr                                 from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/smithy-go/rand                                from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/time                                from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/smithy-go/tracing                             from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/transport/http                      from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
-        github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
@@ -94,8 +96,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/evanphx/json-patch/v5                             from sigs.k8s.io/controller-runtime/pkg/client
         github.com/evanphx/json-patch/v5/internal/json               from github.com/evanphx/json-patch/v5
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
-        github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
+        github.com/fxamacker/cbor/v2                                 from tailscale.com/tka+
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
+        github.com/gaissmai/bart/internal/bitset                     from github.com/gaissmai/bart+
+        github.com/gaissmai/bart/internal/sparse                     from github.com/gaissmai/bart
         github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
@@ -110,11 +114,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/go-openapi/jsonpointer                            from github.com/go-openapi/jsonreference
         github.com/go-openapi/jsonreference                          from k8s.io/kube-openapi/pkg/internal+
         github.com/go-openapi/jsonreference/internal                 from github.com/go-openapi/jsonreference
-        github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
+     💣 github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
      💣 github.com/gogo/protobuf/proto                               from k8s.io/api/admission/v1+
         github.com/gogo/protobuf/sortkeys                            from k8s.io/api/admission/v1+
-        github.com/golang/groupcache/lru                             from k8s.io/client-go/tools/record+
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/golang/protobuf/proto                             from k8s.io/client-go/discovery+
         github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header+
         github.com/google/gnostic-models/compiler                    from github.com/google/gnostic-models/openapiv2+
@@ -139,15 +143,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
-        github.com/imdario/mergo                                     from k8s.io/client-go/tools/clientcmd
-   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
+   L 💣 github.com/illarion/gonotify/v3                              from tailscale.com/net/dns
+   L    github.com/illarion/gonotify/v3/syscallf                     from github.com/illarion/gonotify/v3
+   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/feature/tap
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/interfaces                      from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/rfc1035label                    from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/jmespath/go-jmespath                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/josharian/intern                                  from github.com/mailru/easyjson/jlexer
-   L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
      💣 github.com/json-iterator/go                                  from sigs.k8s.io/structured-merge-diff/v4/fieldpath+
@@ -158,7 +161,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
-        github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
+        github.com/kortschak/wol                                     from tailscale.com/feature/wakeonlan
         github.com/mailru/easyjson/buffer                            from github.com/mailru/easyjson/jwriter
      💣 github.com/mailru/easyjson/jlexer                            from github.com/go-openapi/swag
         github.com/mailru/easyjson/jwriter                           from github.com/go-openapi/swag
@@ -172,7 +175,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
         github.com/modern-go/concurrent                              from github.com/json-iterator/go
      💣 github.com/modern-go/reflect2                                from github.com/json-iterator/go
-        github.com/munnerz/goautoneg                                 from k8s.io/kube-openapi/pkg/handler3
+        github.com/munnerz/goautoneg                                 from k8s.io/kube-openapi/pkg/handler3+
         github.com/opencontainers/go-digest                          from github.com/distribution/reference
    L    github.com/pierrec/lz4/v4                                    from github.com/u-root/uio/uio
    L    github.com/pierrec/lz4/v4/internal/lz4block                  from github.com/pierrec/lz4/v4+
@@ -187,7 +190,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/prometheus/client_golang/prometheus/promhttp      from sigs.k8s.io/controller-runtime/pkg/metrics/server+
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
         github.com/prometheus/common/expfmt                          from github.com/prometheus/client_golang/prometheus+
-        github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
         github.com/prometheus/common/model                           from github.com/prometheus/client_golang/prometheus+
   LD    github.com/prometheus/procfs                                 from github.com/prometheus/client_golang/prometheus
   LD    github.com/prometheus/procfs/internal/fs                     from github.com/prometheus/procfs
@@ -200,10 +202,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    W 💣 github.com/tailscale/go-winio/internal/socket                from github.com/tailscale/go-winio
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
-        github.com/tailscale/golang-x-crypto/acme                    from tailscale.com/ipn/ipnlocal
-  LD    github.com/tailscale/golang-x-crypto/internal/poly1305       from github.com/tailscale/golang-x-crypto/ssh
-  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal
-  LD    github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh
         github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2+
         github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
         github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
@@ -225,7 +223,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
         github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device
      💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
-        github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
@@ -240,7 +237,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         go.uber.org/zap/internal/pool                                from go.uber.org/zap+
         go.uber.org/zap/internal/stacktrace                          from go.uber.org/zap
         go.uber.org/zap/zapcore                                      from github.com/go-logr/zapr+
-     💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
+     💣 go4.org/mem                                                  from tailscale.com/client/local+
         go4.org/netipx                                               from tailscale.com/ipn/ipnlocal+
    W 💣 golang.zx2c4.com/wintun                                      from github.com/tailscale/wireguard-go/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/dns+
@@ -252,6 +249,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         google.golang.org/protobuf/internal/descopts                 from google.golang.org/protobuf/internal/filedesc+
         google.golang.org/protobuf/internal/detrand                  from google.golang.org/protobuf/internal/descfmt+
         google.golang.org/protobuf/internal/editiondefaults          from google.golang.org/protobuf/internal/filedesc+
+        google.golang.org/protobuf/internal/editionssupport          from google.golang.org/protobuf/reflect/protodesc
         google.golang.org/protobuf/internal/encoding/defval          from google.golang.org/protobuf/internal/encoding/tag+
         google.golang.org/protobuf/internal/encoding/messageset      from google.golang.org/protobuf/encoding/prototext+
         google.golang.org/protobuf/internal/encoding/tag             from google.golang.org/protobuf/internal/impl
@@ -277,8 +275,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         google.golang.org/protobuf/types/gofeaturespb                from google.golang.org/protobuf/reflect/protodesc
         google.golang.org/protobuf/types/known/anypb                 from github.com/google/gnostic-models/compiler+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
+        gopkg.in/evanphx/json-patch.v4                               from k8s.io/client-go/testing
         gopkg.in/inf.v0                                              from k8s.io/apimachinery/pkg/api/resource
-        gopkg.in/yaml.v2                                             from k8s.io/kube-openapi/pkg/util/proto+
         gopkg.in/yaml.v3                                             from github.com/go-openapi/swag+
         gvisor.dev/gvisor/pkg/atomicbitops                           from gvisor.dev/gvisor/pkg/buffer+
         gvisor.dev/gvisor/pkg/bits                                   from gvisor.dev/gvisor/pkg/buffer
@@ -300,12 +298,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/hash/jenkins                     from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
         gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
         gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
         gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/feature/tap+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
@@ -347,6 +345,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/api/certificates/v1alpha1                             from k8s.io/client-go/applyconfigurations/certificates/v1alpha1+
         k8s.io/api/certificates/v1beta1                              from k8s.io/client-go/applyconfigurations/certificates/v1beta1+
         k8s.io/api/coordination/v1                                   from k8s.io/client-go/applyconfigurations/coordination/v1+
+        k8s.io/api/coordination/v1alpha2                             from k8s.io/client-go/applyconfigurations/coordination/v1alpha2+
         k8s.io/api/coordination/v1beta1                              from k8s.io/client-go/applyconfigurations/coordination/v1beta1+
         k8s.io/api/core/v1                                           from k8s.io/api/apps/v1+
         k8s.io/api/discovery/v1                                      from k8s.io/client-go/applyconfigurations/discovery/v1+
@@ -369,7 +368,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/api/rbac/v1                                           from k8s.io/client-go/applyconfigurations/rbac/v1+
         k8s.io/api/rbac/v1alpha1                                     from k8s.io/client-go/applyconfigurations/rbac/v1alpha1+
         k8s.io/api/rbac/v1beta1                                      from k8s.io/client-go/applyconfigurations/rbac/v1beta1+
-        k8s.io/api/resource/v1alpha2                                 from k8s.io/client-go/applyconfigurations/resource/v1alpha2+
+        k8s.io/api/resource/v1alpha3                                 from k8s.io/client-go/applyconfigurations/resource/v1alpha3+
+        k8s.io/api/resource/v1beta1                                  from k8s.io/client-go/applyconfigurations/resource/v1beta1+
         k8s.io/api/scheduling/v1                                     from k8s.io/client-go/applyconfigurations/scheduling/v1+
         k8s.io/api/scheduling/v1alpha1                               from k8s.io/client-go/applyconfigurations/scheduling/v1alpha1+
         k8s.io/api/scheduling/v1beta1                                from k8s.io/client-go/applyconfigurations/scheduling/v1beta1+
@@ -382,10 +382,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/api/equality                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
         k8s.io/apimachinery/pkg/api/errors                           from k8s.io/apimachinery/pkg/util/managedfields/internal+
         k8s.io/apimachinery/pkg/api/meta                             from k8s.io/apimachinery/pkg/api/validation+
+        k8s.io/apimachinery/pkg/api/meta/testrestmapper              from k8s.io/client-go/testing
         k8s.io/apimachinery/pkg/api/resource                         from k8s.io/api/autoscaling/v1+
         k8s.io/apimachinery/pkg/api/validation                       from k8s.io/apimachinery/pkg/util/managedfields/internal+
      💣 k8s.io/apimachinery/pkg/apis/meta/internalversion            from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
         k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme     from k8s.io/client-go/metadata
+        k8s.io/apimachinery/pkg/apis/meta/internalversion/validation from k8s.io/client-go/util/watchlist
      💣 k8s.io/apimachinery/pkg/apis/meta/v1                         from k8s.io/api/admission/v1+
         k8s.io/apimachinery/pkg/apis/meta/v1/unstructured            from k8s.io/apimachinery/pkg/runtime/serializer/versioning+
         k8s.io/apimachinery/pkg/apis/meta/v1/validation              from k8s.io/apimachinery/pkg/api/validation+
@@ -397,6 +399,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/runtime                              from k8s.io/api/admission/v1+
         k8s.io/apimachinery/pkg/runtime/schema                       from k8s.io/api/admission/v1+
         k8s.io/apimachinery/pkg/runtime/serializer                   from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
+        k8s.io/apimachinery/pkg/runtime/serializer/cbor              from k8s.io/client-go/dynamic+
+        k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct       from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
+        k8s.io/apimachinery/pkg/runtime/serializer/cbor/internal/modes from k8s.io/apimachinery/pkg/runtime/serializer/cbor+
         k8s.io/apimachinery/pkg/runtime/serializer/json              from k8s.io/apimachinery/pkg/runtime/serializer+
         k8s.io/apimachinery/pkg/runtime/serializer/protobuf          from k8s.io/apimachinery/pkg/runtime/serializer
         k8s.io/apimachinery/pkg/runtime/serializer/recognizer        from k8s.io/apimachinery/pkg/runtime/serializer+
@@ -448,6 +453,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/applyconfigurations/certificates/v1alpha1   from k8s.io/client-go/kubernetes/typed/certificates/v1alpha1
         k8s.io/client-go/applyconfigurations/certificates/v1beta1    from k8s.io/client-go/kubernetes/typed/certificates/v1beta1
         k8s.io/client-go/applyconfigurations/coordination/v1         from k8s.io/client-go/kubernetes/typed/coordination/v1
+        k8s.io/client-go/applyconfigurations/coordination/v1alpha2   from k8s.io/client-go/kubernetes/typed/coordination/v1alpha2
         k8s.io/client-go/applyconfigurations/coordination/v1beta1    from k8s.io/client-go/kubernetes/typed/coordination/v1beta1
         k8s.io/client-go/applyconfigurations/core/v1                 from k8s.io/client-go/applyconfigurations/apps/v1+
         k8s.io/client-go/applyconfigurations/discovery/v1            from k8s.io/client-go/kubernetes/typed/discovery/v1
@@ -472,7 +478,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/applyconfigurations/rbac/v1                 from k8s.io/client-go/kubernetes/typed/rbac/v1
         k8s.io/client-go/applyconfigurations/rbac/v1alpha1           from k8s.io/client-go/kubernetes/typed/rbac/v1alpha1
         k8s.io/client-go/applyconfigurations/rbac/v1beta1            from k8s.io/client-go/kubernetes/typed/rbac/v1beta1
-        k8s.io/client-go/applyconfigurations/resource/v1alpha2       from k8s.io/client-go/kubernetes/typed/resource/v1alpha2
+        k8s.io/client-go/applyconfigurations/resource/v1alpha3       from k8s.io/client-go/kubernetes/typed/resource/v1alpha3
+        k8s.io/client-go/applyconfigurations/resource/v1beta1        from k8s.io/client-go/kubernetes/typed/resource/v1beta1
         k8s.io/client-go/applyconfigurations/scheduling/v1           from k8s.io/client-go/kubernetes/typed/scheduling/v1
         k8s.io/client-go/applyconfigurations/scheduling/v1alpha1     from k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1
         k8s.io/client-go/applyconfigurations/scheduling/v1beta1      from k8s.io/client-go/kubernetes/typed/scheduling/v1beta1
@@ -482,8 +489,80 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1
         k8s.io/client-go/discovery                                   from k8s.io/client-go/applyconfigurations/meta/v1+
         k8s.io/client-go/dynamic                                     from sigs.k8s.io/controller-runtime/pkg/cache/internal+
-        k8s.io/client-go/features                                    from k8s.io/client-go/tools/cache
-        k8s.io/client-go/kubernetes                                  from k8s.io/client-go/tools/leaderelection/resourcelock
+        k8s.io/client-go/features                                    from k8s.io/client-go/tools/cache+
+        k8s.io/client-go/gentype                                     from k8s.io/client-go/kubernetes/typed/admissionregistration/v1+
+        k8s.io/client-go/informers                                   from k8s.io/client-go/tools/leaderelection
+        k8s.io/client-go/informers/admissionregistration             from k8s.io/client-go/informers
+        k8s.io/client-go/informers/admissionregistration/v1          from k8s.io/client-go/informers/admissionregistration
+        k8s.io/client-go/informers/admissionregistration/v1alpha1    from k8s.io/client-go/informers/admissionregistration
+        k8s.io/client-go/informers/admissionregistration/v1beta1     from k8s.io/client-go/informers/admissionregistration
+        k8s.io/client-go/informers/apiserverinternal                 from k8s.io/client-go/informers
+        k8s.io/client-go/informers/apiserverinternal/v1alpha1        from k8s.io/client-go/informers/apiserverinternal
+        k8s.io/client-go/informers/apps                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/apps/v1                           from k8s.io/client-go/informers/apps
+        k8s.io/client-go/informers/apps/v1beta1                      from k8s.io/client-go/informers/apps
+        k8s.io/client-go/informers/apps/v1beta2                      from k8s.io/client-go/informers/apps
+        k8s.io/client-go/informers/autoscaling                       from k8s.io/client-go/informers
+        k8s.io/client-go/informers/autoscaling/v1                    from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/autoscaling/v2                    from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/autoscaling/v2beta1               from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/autoscaling/v2beta2               from k8s.io/client-go/informers/autoscaling
+        k8s.io/client-go/informers/batch                             from k8s.io/client-go/informers
+        k8s.io/client-go/informers/batch/v1                          from k8s.io/client-go/informers/batch
+        k8s.io/client-go/informers/batch/v1beta1                     from k8s.io/client-go/informers/batch
+        k8s.io/client-go/informers/certificates                      from k8s.io/client-go/informers
+        k8s.io/client-go/informers/certificates/v1                   from k8s.io/client-go/informers/certificates
+        k8s.io/client-go/informers/certificates/v1alpha1             from k8s.io/client-go/informers/certificates
+        k8s.io/client-go/informers/certificates/v1beta1              from k8s.io/client-go/informers/certificates
+        k8s.io/client-go/informers/coordination                      from k8s.io/client-go/informers
+        k8s.io/client-go/informers/coordination/v1                   from k8s.io/client-go/informers/coordination
+        k8s.io/client-go/informers/coordination/v1alpha2             from k8s.io/client-go/informers/coordination
+        k8s.io/client-go/informers/coordination/v1beta1              from k8s.io/client-go/informers/coordination
+        k8s.io/client-go/informers/core                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/core/v1                           from k8s.io/client-go/informers/core
+        k8s.io/client-go/informers/discovery                         from k8s.io/client-go/informers
+        k8s.io/client-go/informers/discovery/v1                      from k8s.io/client-go/informers/discovery
+        k8s.io/client-go/informers/discovery/v1beta1                 from k8s.io/client-go/informers/discovery
+        k8s.io/client-go/informers/events                            from k8s.io/client-go/informers
+        k8s.io/client-go/informers/events/v1                         from k8s.io/client-go/informers/events
+        k8s.io/client-go/informers/events/v1beta1                    from k8s.io/client-go/informers/events
+        k8s.io/client-go/informers/extensions                        from k8s.io/client-go/informers
+        k8s.io/client-go/informers/extensions/v1beta1                from k8s.io/client-go/informers/extensions
+        k8s.io/client-go/informers/flowcontrol                       from k8s.io/client-go/informers
+        k8s.io/client-go/informers/flowcontrol/v1                    from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/flowcontrol/v1beta1               from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/flowcontrol/v1beta2               from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/flowcontrol/v1beta3               from k8s.io/client-go/informers/flowcontrol
+        k8s.io/client-go/informers/internalinterfaces                from k8s.io/client-go/informers+
+        k8s.io/client-go/informers/networking                        from k8s.io/client-go/informers
+        k8s.io/client-go/informers/networking/v1                     from k8s.io/client-go/informers/networking
+        k8s.io/client-go/informers/networking/v1alpha1               from k8s.io/client-go/informers/networking
+        k8s.io/client-go/informers/networking/v1beta1                from k8s.io/client-go/informers/networking
+        k8s.io/client-go/informers/node                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/node/v1                           from k8s.io/client-go/informers/node
+        k8s.io/client-go/informers/node/v1alpha1                     from k8s.io/client-go/informers/node
+        k8s.io/client-go/informers/node/v1beta1                      from k8s.io/client-go/informers/node
+        k8s.io/client-go/informers/policy                            from k8s.io/client-go/informers
+        k8s.io/client-go/informers/policy/v1                         from k8s.io/client-go/informers/policy
+        k8s.io/client-go/informers/policy/v1beta1                    from k8s.io/client-go/informers/policy
+        k8s.io/client-go/informers/rbac                              from k8s.io/client-go/informers
+        k8s.io/client-go/informers/rbac/v1                           from k8s.io/client-go/informers/rbac
+        k8s.io/client-go/informers/rbac/v1alpha1                     from k8s.io/client-go/informers/rbac
+        k8s.io/client-go/informers/rbac/v1beta1                      from k8s.io/client-go/informers/rbac
+        k8s.io/client-go/informers/resource                          from k8s.io/client-go/informers
+        k8s.io/client-go/informers/resource/v1alpha3                 from k8s.io/client-go/informers/resource
+        k8s.io/client-go/informers/resource/v1beta1                  from k8s.io/client-go/informers/resource
+        k8s.io/client-go/informers/scheduling                        from k8s.io/client-go/informers
+        k8s.io/client-go/informers/scheduling/v1                     from k8s.io/client-go/informers/scheduling
+        k8s.io/client-go/informers/scheduling/v1alpha1               from k8s.io/client-go/informers/scheduling
+        k8s.io/client-go/informers/scheduling/v1beta1                from k8s.io/client-go/informers/scheduling
+        k8s.io/client-go/informers/storage                           from k8s.io/client-go/informers
+        k8s.io/client-go/informers/storage/v1                        from k8s.io/client-go/informers/storage
+        k8s.io/client-go/informers/storage/v1alpha1                  from k8s.io/client-go/informers/storage
+        k8s.io/client-go/informers/storage/v1beta1                   from k8s.io/client-go/informers/storage
+        k8s.io/client-go/informers/storagemigration                  from k8s.io/client-go/informers
+        k8s.io/client-go/informers/storagemigration/v1alpha1         from k8s.io/client-go/informers/storagemigration
+        k8s.io/client-go/kubernetes                                  from k8s.io/client-go/tools/leaderelection/resourcelock+
         k8s.io/client-go/kubernetes/scheme                           from k8s.io/client-go/discovery+
         k8s.io/client-go/kubernetes/typed/admissionregistration/v1   from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes
@@ -507,6 +586,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/kubernetes/typed/certificates/v1alpha1      from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/certificates/v1beta1       from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/coordination/v1            from k8s.io/client-go/kubernetes+
+        k8s.io/client-go/kubernetes/typed/coordination/v1alpha2      from k8s.io/client-go/kubernetes+
         k8s.io/client-go/kubernetes/typed/coordination/v1beta1       from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/core/v1                    from k8s.io/client-go/kubernetes+
         k8s.io/client-go/kubernetes/typed/discovery/v1               from k8s.io/client-go/kubernetes
@@ -529,7 +609,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/kubernetes/typed/rbac/v1                    from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/rbac/v1alpha1              from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/rbac/v1beta1               from k8s.io/client-go/kubernetes
-        k8s.io/client-go/kubernetes/typed/resource/v1alpha2          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/resource/v1alpha3          from k8s.io/client-go/kubernetes
+        k8s.io/client-go/kubernetes/typed/resource/v1beta1           from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/scheduling/v1              from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1        from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/scheduling/v1beta1         from k8s.io/client-go/kubernetes
@@ -537,6 +618,56 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/kubernetes/typed/storage/v1alpha1           from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/storage/v1beta1            from k8s.io/client-go/kubernetes
         k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1  from k8s.io/client-go/kubernetes
+        k8s.io/client-go/listers                                     from k8s.io/client-go/listers/admissionregistration/v1+
+        k8s.io/client-go/listers/admissionregistration/v1            from k8s.io/client-go/informers/admissionregistration/v1
+        k8s.io/client-go/listers/admissionregistration/v1alpha1      from k8s.io/client-go/informers/admissionregistration/v1alpha1
+        k8s.io/client-go/listers/admissionregistration/v1beta1       from k8s.io/client-go/informers/admissionregistration/v1beta1
+        k8s.io/client-go/listers/apiserverinternal/v1alpha1          from k8s.io/client-go/informers/apiserverinternal/v1alpha1
+        k8s.io/client-go/listers/apps/v1                             from k8s.io/client-go/informers/apps/v1
+        k8s.io/client-go/listers/apps/v1beta1                        from k8s.io/client-go/informers/apps/v1beta1
+        k8s.io/client-go/listers/apps/v1beta2                        from k8s.io/client-go/informers/apps/v1beta2
+        k8s.io/client-go/listers/autoscaling/v1                      from k8s.io/client-go/informers/autoscaling/v1
+        k8s.io/client-go/listers/autoscaling/v2                      from k8s.io/client-go/informers/autoscaling/v2
+        k8s.io/client-go/listers/autoscaling/v2beta1                 from k8s.io/client-go/informers/autoscaling/v2beta1
+        k8s.io/client-go/listers/autoscaling/v2beta2                 from k8s.io/client-go/informers/autoscaling/v2beta2
+        k8s.io/client-go/listers/batch/v1                            from k8s.io/client-go/informers/batch/v1
+        k8s.io/client-go/listers/batch/v1beta1                       from k8s.io/client-go/informers/batch/v1beta1
+        k8s.io/client-go/listers/certificates/v1                     from k8s.io/client-go/informers/certificates/v1
+        k8s.io/client-go/listers/certificates/v1alpha1               from k8s.io/client-go/informers/certificates/v1alpha1
+        k8s.io/client-go/listers/certificates/v1beta1                from k8s.io/client-go/informers/certificates/v1beta1
+        k8s.io/client-go/listers/coordination/v1                     from k8s.io/client-go/informers/coordination/v1
+        k8s.io/client-go/listers/coordination/v1alpha2               from k8s.io/client-go/informers/coordination/v1alpha2
+        k8s.io/client-go/listers/coordination/v1beta1                from k8s.io/client-go/informers/coordination/v1beta1
+        k8s.io/client-go/listers/core/v1                             from k8s.io/client-go/informers/core/v1
+        k8s.io/client-go/listers/discovery/v1                        from k8s.io/client-go/informers/discovery/v1
+        k8s.io/client-go/listers/discovery/v1beta1                   from k8s.io/client-go/informers/discovery/v1beta1
+        k8s.io/client-go/listers/events/v1                           from k8s.io/client-go/informers/events/v1
+        k8s.io/client-go/listers/events/v1beta1                      from k8s.io/client-go/informers/events/v1beta1
+        k8s.io/client-go/listers/extensions/v1beta1                  from k8s.io/client-go/informers/extensions/v1beta1
+        k8s.io/client-go/listers/flowcontrol/v1                      from k8s.io/client-go/informers/flowcontrol/v1
+        k8s.io/client-go/listers/flowcontrol/v1beta1                 from k8s.io/client-go/informers/flowcontrol/v1beta1
+        k8s.io/client-go/listers/flowcontrol/v1beta2                 from k8s.io/client-go/informers/flowcontrol/v1beta2
+        k8s.io/client-go/listers/flowcontrol/v1beta3                 from k8s.io/client-go/informers/flowcontrol/v1beta3
+        k8s.io/client-go/listers/networking/v1                       from k8s.io/client-go/informers/networking/v1
+        k8s.io/client-go/listers/networking/v1alpha1                 from k8s.io/client-go/informers/networking/v1alpha1
+        k8s.io/client-go/listers/networking/v1beta1                  from k8s.io/client-go/informers/networking/v1beta1
+        k8s.io/client-go/listers/node/v1                             from k8s.io/client-go/informers/node/v1
+        k8s.io/client-go/listers/node/v1alpha1                       from k8s.io/client-go/informers/node/v1alpha1
+        k8s.io/client-go/listers/node/v1beta1                        from k8s.io/client-go/informers/node/v1beta1
+        k8s.io/client-go/listers/policy/v1                           from k8s.io/client-go/informers/policy/v1
+        k8s.io/client-go/listers/policy/v1beta1                      from k8s.io/client-go/informers/policy/v1beta1
+        k8s.io/client-go/listers/rbac/v1                             from k8s.io/client-go/informers/rbac/v1
+        k8s.io/client-go/listers/rbac/v1alpha1                       from k8s.io/client-go/informers/rbac/v1alpha1
+        k8s.io/client-go/listers/rbac/v1beta1                        from k8s.io/client-go/informers/rbac/v1beta1
+        k8s.io/client-go/listers/resource/v1alpha3                   from k8s.io/client-go/informers/resource/v1alpha3
+        k8s.io/client-go/listers/resource/v1beta1                    from k8s.io/client-go/informers/resource/v1beta1
+        k8s.io/client-go/listers/scheduling/v1                       from k8s.io/client-go/informers/scheduling/v1
+        k8s.io/client-go/listers/scheduling/v1alpha1                 from k8s.io/client-go/informers/scheduling/v1alpha1
+        k8s.io/client-go/listers/scheduling/v1beta1                  from k8s.io/client-go/informers/scheduling/v1beta1
+        k8s.io/client-go/listers/storage/v1                          from k8s.io/client-go/informers/storage/v1
+        k8s.io/client-go/listers/storage/v1alpha1                    from k8s.io/client-go/informers/storage/v1alpha1
+        k8s.io/client-go/listers/storage/v1beta1                     from k8s.io/client-go/informers/storage/v1beta1
+        k8s.io/client-go/listers/storagemigration/v1alpha1           from k8s.io/client-go/informers/storagemigration/v1alpha1
         k8s.io/client-go/metadata                                    from sigs.k8s.io/controller-runtime/pkg/cache/internal+
         k8s.io/client-go/openapi                                     from k8s.io/client-go/discovery
         k8s.io/client-go/pkg/apis/clientauthentication               from k8s.io/client-go/pkg/apis/clientauthentication/install+
@@ -548,6 +679,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/rest                                        from k8s.io/client-go/discovery+
         k8s.io/client-go/rest/watch                                  from k8s.io/client-go/rest
         k8s.io/client-go/restmapper                                  from sigs.k8s.io/controller-runtime/pkg/client/apiutil
+        k8s.io/client-go/testing                                     from k8s.io/client-go/gentype
         k8s.io/client-go/tools/auth                                  from k8s.io/client-go/tools/clientcmd
         k8s.io/client-go/tools/cache                                 from sigs.k8s.io/controller-runtime/pkg/cache+
         k8s.io/client-go/tools/cache/synctrack                       from k8s.io/client-go/tools/cache
@@ -564,11 +696,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/client-go/tools/record/util                           from k8s.io/client-go/tools/record
         k8s.io/client-go/tools/reference                             from k8s.io/client-go/kubernetes/typed/core/v1+
         k8s.io/client-go/transport                                   from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/util/apply                                  from k8s.io/client-go/dynamic+
         k8s.io/client-go/util/cert                                   from k8s.io/client-go/rest+
         k8s.io/client-go/util/connrotation                           from k8s.io/client-go/plugin/pkg/client/auth/exec+
+        k8s.io/client-go/util/consistencydetector                    from k8s.io/client-go/dynamic+
         k8s.io/client-go/util/flowcontrol                            from k8s.io/client-go/kubernetes+
         k8s.io/client-go/util/homedir                                from k8s.io/client-go/tools/clientcmd
         k8s.io/client-go/util/keyutil                                from k8s.io/client-go/util/cert
+        k8s.io/client-go/util/watchlist                              from k8s.io/client-go/dynamic+
         k8s.io/client-go/util/workqueue                              from k8s.io/client-go/transport+
         k8s.io/klog/v2                                               from k8s.io/apimachinery/pkg/api/meta+
         k8s.io/klog/v2/internal/buffer                               from k8s.io/klog/v2
@@ -589,11 +724,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/utils/buffer                                          from k8s.io/client-go/tools/cache
         k8s.io/utils/clock                                           from k8s.io/apimachinery/pkg/util/cache+
         k8s.io/utils/clock/testing                                   from k8s.io/client-go/util/flowcontrol
+        k8s.io/utils/internal/third_party/forked/golang/golang-lru   from k8s.io/utils/lru
         k8s.io/utils/internal/third_party/forked/golang/net          from k8s.io/utils/net
+        k8s.io/utils/lru                                             from k8s.io/client-go/tools/record
         k8s.io/utils/net                                             from k8s.io/apimachinery/pkg/util/net+
         k8s.io/utils/pointer                                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
         k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
-        k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
         k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
         sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
         sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
@@ -626,12 +762,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         sigs.k8s.io/controller-runtime/pkg/metrics                   from sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics+
         sigs.k8s.io/controller-runtime/pkg/metrics/server            from sigs.k8s.io/controller-runtime/pkg/manager
         sigs.k8s.io/controller-runtime/pkg/predicate                 from sigs.k8s.io/controller-runtime/pkg/builder+
-        sigs.k8s.io/controller-runtime/pkg/ratelimiter               from sigs.k8s.io/controller-runtime/pkg/controller+
         sigs.k8s.io/controller-runtime/pkg/reconcile                 from sigs.k8s.io/controller-runtime/pkg/builder+
         sigs.k8s.io/controller-runtime/pkg/recorder                  from sigs.k8s.io/controller-runtime/pkg/leaderelection+
         sigs.k8s.io/controller-runtime/pkg/source                    from sigs.k8s.io/controller-runtime/pkg/builder+
         sigs.k8s.io/controller-runtime/pkg/webhook                   from sigs.k8s.io/controller-runtime/pkg/manager
         sigs.k8s.io/controller-runtime/pkg/webhook/admission         from sigs.k8s.io/controller-runtime/pkg/builder+
+        sigs.k8s.io/controller-runtime/pkg/webhook/admission/metrics from sigs.k8s.io/controller-runtime/pkg/webhook/admission
         sigs.k8s.io/controller-runtime/pkg/webhook/conversion        from sigs.k8s.io/controller-runtime/pkg/builder
         sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics  from sigs.k8s.io/controller-runtime/pkg/webhook+
         sigs.k8s.io/json                                             from k8s.io/apimachinery/pkg/runtime/serializer/json+
@@ -642,11 +778,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         sigs.k8s.io/structured-merge-diff/v4/typed                   from k8s.io/apimachinery/pkg/util/managedfields+
         sigs.k8s.io/structured-merge-diff/v4/value                   from k8s.io/apimachinery/pkg/runtime+
         sigs.k8s.io/yaml                                             from k8s.io/apimachinery/pkg/runtime/serializer/json+
-        sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml
+        sigs.k8s.io/yaml/goyaml.v2                                   from sigs.k8s.io/yaml+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
-        tailscale.com/atomicfile                                     from tailscale.com/ipn+
-        tailscale.com/client/tailscale                               from tailscale.com/client/web+
+     💣 tailscale.com/atomicfile                                     from tailscale.com/ipn+
+        tailscale.com/client/local                                   from tailscale.com/client/tailscale+
+        tailscale.com/client/tailscale                               from tailscale.com/cmd/k8s-operator+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
@@ -663,19 +800,27 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/ethtool                                 from tailscale.com/ipn/ipnlocal
      💣 tailscale.com/doctor/permissions                             from tailscale.com/ipn/ipnlocal
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
-        tailscale.com/drive                                          from tailscale.com/client/tailscale+
-        tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/drive                                          from tailscale.com/client/local+
+        tailscale.com/envknob                                        from tailscale.com/client/local+
         tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
+        tailscale.com/feature                                        from tailscale.com/feature/wakeonlan+
+        tailscale.com/feature/capture                                from tailscale.com/feature/condregister
+        tailscale.com/feature/condregister                           from tailscale.com/tsnet
+   L    tailscale.com/feature/tap                                    from tailscale.com/feature/condregister
+        tailscale.com/feature/wakeonlan                              from tailscale.com/feature/condregister
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
+        tailscale.com/internal/client/tailscale                      from tailscale.com/cmd/k8s-operator
         tailscale.com/internal/noiseconn                             from tailscale.com/control/controlclient
-        tailscale.com/ipn                                            from tailscale.com/client/tailscale+
+        tailscale.com/ipn                                            from tailscale.com/client/local+
+        tailscale.com/ipn/auditlog                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn/conffile                                   from tailscale.com/ipn/ipnlocal+
+     💣 tailscale.com/ipn/desktop                                    from tailscale.com/ipn/ipnlocal+
      💣 tailscale.com/ipn/ipnauth                                    from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/localapi+
-        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
-        tailscale.com/ipn/localapi                                   from tailscale.com/tsnet
+        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/local+
+        tailscale.com/ipn/localapi                                   from tailscale.com/tsnet+
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/ipn/store                                      from tailscale.com/ipn/ipnlocal+
    L    tailscale.com/ipn/store/awsstore                             from tailscale.com/ipn/store
@@ -700,6 +845,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
+        tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial+
         tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
@@ -720,7 +866,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/net/netmon                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
    W 💣 tailscale.com/net/netstat                                    from tailscale.com/portlist
-        tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
+        tailscale.com/net/netutil                                    from tailscale.com/client/local+
         tailscale.com/net/packet                                     from tailscale.com/net/connstats+
         tailscale.com/net/packet/checksum                            from tailscale.com/net/tstun
         tailscale.com/net/ping                                       from tailscale.com/net/netcheck+
@@ -738,17 +884,19 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/tsd+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
-        tailscale.com/paths                                          from tailscale.com/client/tailscale+
+        tailscale.com/paths                                          from tailscale.com/client/local+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
-     💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
+     💣 tailscale.com/safesocket                                     from tailscale.com/client/local+
         tailscale.com/sessionrecording                               from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
-        tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
+        tailscale.com/tailcfg                                        from tailscale.com/client/local+
         tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
+        tailscale.com/tempfork/acme                                  from tailscale.com/ipn/ipnlocal
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
-        tailscale.com/tka                                            from tailscale.com/client/tailscale+
+        tailscale.com/tempfork/httprec                               from tailscale.com/control/controlclient
+        tailscale.com/tka                                            from tailscale.com/client/local+
         tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
         tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
@@ -757,10 +905,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
         tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
+        tailscale.com/types/bools                                    from tailscale.com/tsnet
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
-        tailscale.com/types/key                                      from tailscale.com/client/tailscale+
+        tailscale.com/types/key                                      from tailscale.com/client/local+
         tailscale.com/types/lazy                                     from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/logger                                   from tailscale.com/appc+
         tailscale.com/types/logid                                    from tailscale.com/ipn/ipnlocal+
@@ -773,7 +922,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
         tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
-        tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
+        tailscale.com/types/tkatype                                  from tailscale.com/client/local+
         tailscale.com/types/views                                    from tailscale.com/appc+
         tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/cmd/k8s-operator+
@@ -818,7 +967,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
-        tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/usermetric                                from tailscale.com/health+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
@@ -830,7 +978,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
         tailscale.com/wgengine                                       from tailscale.com/ipn/ipnlocal+
-        tailscale.com/wgengine/capture                               from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap+
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
@@ -846,18 +993,21 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
-  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf
-        golang.org/x/crypto/chacha20                                 from github.com/tailscale/golang-x-crypto/ssh+
+  LD    golang.org/x/crypto/blowfish                                 from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf
+        golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/ssh+
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
-        golang.org/x/crypto/curve25519                               from github.com/tailscale/golang-x-crypto/ssh+
-        golang.org/x/crypto/hkdf                                     from crypto/tls+
+        golang.org/x/crypto/curve25519                               from golang.org/x/crypto/ssh+
+        golang.org/x/crypto/hkdf                                     from tailscale.com/control/controlbase
+        golang.org/x/crypto/internal/alias                           from golang.org/x/crypto/chacha20+
+        golang.org/x/crypto/internal/poly1305                        from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
+  LD    golang.org/x/crypto/ssh                                      from tailscale.com/ipn/ipnlocal
+  LD    golang.org/x/crypto/ssh/internal/bcrypt_pbkdf                from golang.org/x/crypto/ssh
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
@@ -870,6 +1020,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/net/http2/hpack                                 from golang.org/x/net/http2+
         golang.org/x/net/icmp                                        from github.com/prometheus-community/pro-bing+
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
+        golang.org/x/net/internal/httpcommon                         from golang.org/x/net/http2
+        golang.org/x/net/internal/iana                               from golang.org/x/net/icmp+
+        golang.org/x/net/internal/socket                             from golang.org/x/net/icmp+
+        golang.org/x/net/internal/socks                              from golang.org/x/net/proxy
         golang.org/x/net/ipv4                                        from github.com/miekg/dns+
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
@@ -879,7 +1033,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
         golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+
         golang.org/x/sync/errgroup                                   from github.com/mdlayher/socket+
-        golang.org/x/sys/cpu                                         from github.com/josharian/native+
+        golang.org/x/sys/cpu                                         from github.com/tailscale/certstore+
   LD    golang.org/x/sys/unix                                        from github.com/fsnotify/fsnotify+
    W    golang.org/x/sys/windows                                     from github.com/dblohm7/wingoes+
    W    golang.org/x/sys/windows/registry                            from github.com/dblohm7/wingoes+
@@ -902,7 +1056,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
         crypto                                                       from crypto/ecdh+
-        crypto/aes                                                   from crypto/ecdsa+
+        crypto/aes                                                   from crypto/internal/hpke+
         crypto/cipher                                                from crypto/aes+
         crypto/des                                                   from crypto/tls+
         crypto/dsa                                                   from crypto/x509+
@@ -911,22 +1065,62 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         crypto/ed25519                                               from crypto/tls+
         crypto/elliptic                                              from crypto/ecdsa+
         crypto/hmac                                                  from crypto/tls+
+        crypto/internal/boring                                       from crypto/aes+
+        crypto/internal/boring/bbig                                  from crypto/ecdsa+
+        crypto/internal/boring/sig                                   from crypto/internal/boring
+        crypto/internal/entropy                                      from crypto/internal/fips140/drbg
+        crypto/internal/fips140                                      from crypto/internal/fips140/aes+
+        crypto/internal/fips140/aes                                  from crypto/aes+
+        crypto/internal/fips140/aes/gcm                              from crypto/cipher+
+        crypto/internal/fips140/alias                                from crypto/cipher+
+        crypto/internal/fips140/bigmod                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/check                                from crypto/internal/fips140/aes+
+        crypto/internal/fips140/drbg                                 from crypto/internal/fips140/aes/gcm+
+        crypto/internal/fips140/ecdh                                 from crypto/ecdh
+        crypto/internal/fips140/ecdsa                                from crypto/ecdsa
+        crypto/internal/fips140/ed25519                              from crypto/ed25519
+        crypto/internal/fips140/edwards25519                         from crypto/internal/fips140/ed25519
+        crypto/internal/fips140/edwards25519/field                   from crypto/ecdh+
+        crypto/internal/fips140/hkdf                                 from crypto/internal/fips140/tls13+
+        crypto/internal/fips140/hmac                                 from crypto/hmac+
+        crypto/internal/fips140/mlkem                                from crypto/tls
+        crypto/internal/fips140/nistec                               from crypto/elliptic+
+        crypto/internal/fips140/nistec/fiat                          from crypto/internal/fips140/nistec
+        crypto/internal/fips140/rsa                                  from crypto/rsa
+        crypto/internal/fips140/sha256                               from crypto/internal/fips140/check+
+        crypto/internal/fips140/sha3                                 from crypto/internal/fips140/hmac+
+        crypto/internal/fips140/sha512                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/subtle                               from crypto/internal/fips140/aes+
+        crypto/internal/fips140/tls12                                from crypto/tls
+        crypto/internal/fips140/tls13                                from crypto/tls
+        crypto/internal/fips140deps/byteorder                        from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/cpu                              from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/godebug                          from crypto/internal/fips140+
+        crypto/internal/fips140hash                                  from crypto/ecdsa+
+        crypto/internal/fips140only                                  from crypto/cipher+
+        crypto/internal/hpke                                         from crypto/tls
+        crypto/internal/impl                                         from crypto/internal/fips140/aes+
+        crypto/internal/randutil                                     from crypto/dsa+
+        crypto/internal/sysrand                                      from crypto/internal/entropy+
         crypto/md5                                                   from crypto/tls+
         crypto/rand                                                  from crypto/ed25519+
         crypto/rc4                                                   from crypto/tls+
         crypto/rsa                                                   from crypto/tls+
         crypto/sha1                                                  from crypto/tls+
         crypto/sha256                                                from crypto/tls+
+        crypto/sha3                                                  from crypto/internal/fips140hash
         crypto/sha512                                                from crypto/ecdsa+
-        crypto/subtle                                                from crypto/aes+
+        crypto/subtle                                                from crypto/cipher+
         crypto/tls                                                   from github.com/aws/aws-sdk-go-v2/aws/transport/http+
+        crypto/tls/internal/fips140tls                               from crypto/tls
         crypto/x509                                                  from crypto/tls+
+   D    crypto/x509/internal/macos                                   from crypto/x509
         crypto/x509/pkix                                             from crypto/x509+
         database/sql                                                 from github.com/prometheus/client_golang/prometheus/collectors
         database/sql/driver                                          from database/sql+
    W    debug/dwarf                                                  from debug/pe
    W    debug/pe                                                     from github.com/dblohm7/wingoes/pe
-        embed                                                        from crypto/internal/nistec+
+        embed                                                        from github.com/tailscale/web-client-prebuilt+
         encoding                                                     from encoding/gob+
         encoding/asn1                                                from crypto/x509+
         encoding/base32                                              from github.com/fxamacker/cbor/v2+
@@ -956,6 +1150,48 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         hash/maphash                                                 from go4.org/mem
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
+        internal/abi                                                 from crypto/x509/internal/macos+
+        internal/asan                                                from syscall+
+        internal/bisect                                              from internal/godebug
+        internal/bytealg                                             from bytes+
+        internal/byteorder                                           from crypto/cipher+
+        internal/chacha8rand                                         from math/rand/v2+
+        internal/coverage/rtcov                                      from runtime
+        internal/cpu                                                 from crypto/internal/fips140deps/cpu+
+        internal/filepathlite                                        from os+
+        internal/fmtsort                                             from fmt+
+        internal/goarch                                              from crypto/internal/fips140deps/cpu+
+        internal/godebug                                             from archive/tar+
+        internal/godebugs                                            from internal/godebug+
+        internal/goexperiment                                        from runtime+
+        internal/goos                                                from crypto/x509+
+        internal/itoa                                                from internal/poll+
+        internal/lazyregexp                                          from go/doc
+        internal/msan                                                from syscall+
+        internal/nettrace                                            from net+
+        internal/oserror                                             from io/fs+
+        internal/poll                                                from net+
+        internal/profile                                             from net/http/pprof
+        internal/profilerecord                                       from runtime+
+        internal/race                                                from internal/poll+
+        internal/reflectlite                                         from context+
+        internal/runtime/atomic                                      from internal/runtime/exithook+
+        internal/runtime/exithook                                    from runtime
+        internal/runtime/maps                                        from reflect+
+        internal/runtime/math                                        from internal/runtime/maps+
+        internal/runtime/sys                                         from crypto/subtle+
+   L    internal/runtime/syscall                                     from runtime+
+        internal/saferio                                             from debug/pe+
+        internal/singleflight                                        from net
+        internal/stringslite                                         from embed+
+        internal/sync                                                from sync+
+        internal/syscall/execenv                                     from os+
+  LD    internal/syscall/unix                                        from crypto/internal/sysrand+
+   W    internal/syscall/windows                                     from crypto/internal/sysrand+
+   W    internal/syscall/windows/registry                            from mime+
+   W    internal/syscall/windows/sysdll                              from internal/syscall/windows+
+        internal/testlog                                             from os
+        internal/unsafeheader                                        from internal/reflectlite+
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
@@ -964,6 +1200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         log/internal                                                 from log+
         log/slog                                                     from github.com/go-logr/logr+
         log/slog/internal                                            from log/slog
+        log/slog/internal/buffer                                     from log/slog
         maps                                                         from sigs.k8s.io/controller-runtime/pkg/predicate+
         math                                                         from archive/tar+
         math/big                                                     from crypto/dsa+
@@ -975,15 +1212,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         mime/quotedprintable                                         from mime/multipart
         net                                                          from crypto/tls+
         net/http                                                     from expvar+
-        net/http/httptest                                            from tailscale.com/control/controlclient
         net/http/httptrace                                           from github.com/prometheus-community/pro-bing+
         net/http/httputil                                            from github.com/aws/smithy-go/transport/http+
         net/http/internal                                            from net/http+
+        net/http/internal/ascii                                      from net/http+
         net/http/pprof                                               from sigs.k8s.io/controller-runtime/pkg/manager+
         net/netip                                                    from github.com/gaissmai/bart+
         net/textproto                                                from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
         net/url                                                      from crypto/x509+
-        os                                                           from crypto/rand+
+        os                                                           from crypto/internal/sysrand+
         os/exec                                                      from github.com/aws/aws-sdk-go-v2/credentials/processcreds+
         os/signal                                                    from sigs.k8s.io/controller-runtime/pkg/manager/signals
         os/user                                                      from archive/tar+
@@ -992,6 +1229,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints+
         regexp/syntax                                                from regexp
+        runtime                                                      from archive/tar+
         runtime/debug                                                from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof+
@@ -1011,3 +1249,5 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
         unique                                                       from net/netip
+        unsafe                                                       from bytes+
+        weak                                                         from unique

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -63,7 +63,10 @@ rules:
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["get","list","watch"]
+  verbs: ["get","list","watch", "update"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["update"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: connectors.tailscale.com
 spec:
   group: tailscale.com

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: dnsconfigs.tailscale.com
 spec:
   group: tailscale.com

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: proxyclasses.tailscale.com
 spec:
   group: tailscale.com
@@ -99,6 +99,16 @@ spec:
                         enable:
                           description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
                           type: boolean
+                        labels:
+                          description: |-
+                            Labels to add to the ServiceMonitor.
+                            Labels must be valid Kubernetes labels.
+                            https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+                          type: object
+                          additionalProperties:
+                            type: string
+                            maxLength: 63
+                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                   x-kubernetes-validations:
                     - rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable  && !self.enable)'
                       message: ServiceMonitor can only be enabled if metrics are enabled
@@ -133,6 +143,8 @@ spec:
                       type: object
                       additionalProperties:
                         type: string
+                        maxLength: 63
+                        pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                     pod:
                       description: Configuration for the proxy Pod.
                       type: object
@@ -416,7 +428,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                               Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -431,7 +443,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                               Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -588,7 +600,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                           Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -603,7 +615,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                           Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -761,7 +773,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                               Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -776,7 +788,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                               Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -933,7 +945,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                           Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -948,7 +960,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                           Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -1062,6 +1074,8 @@ spec:
                           type: object
                           additionalProperties:
                             type: string
+                            maxLength: 63
+                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                         nodeName:
                           description: |-
                             Proxy Pod's node name.
@@ -1160,6 +1174,32 @@ spec:
                                 Note that this field cannot be set when spec.os.name is windows.
                               type: integer
                               format: int64
+                            seLinuxChangePolicy:
+                              description: |-
+                                seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                                It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                                Valid values are "MountOption" and "Recursive".
+
+                                "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                                This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                                "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                                This requires all Pods that share the same volume to use the same SELinux label.
+                                It is not possible to share the same volume among privileged and unprivileged Pods.
+                                Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                                whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                                CSIDriver instance. Other volumes are always re-labelled recursively.
+                                "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                                If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                                If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                                and "Recursive" for all other volumes.
+
+                                This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                                All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                                Note that this field cannot be set when spec.os.name is windows.
+                              type: string
                             seLinuxOptions:
                               description: |-
                                 The SELinux context to be applied to all containers.
@@ -1208,18 +1248,28 @@ spec:
                                   type: string
                             supplementalGroups:
                               description: |-
-                                A list of groups applied to the first process run in each container, in addition
-                                to the container's primary GID, the fsGroup (if specified), and group memberships
-                                defined in the container image for the uid of the container process. If unspecified,
-                                no additional groups are added to any container. Note that group memberships
-                                defined in the container image for the uid of the container process are still effective,
-                                even if they are not included in this list.
+                                A list of groups applied to the first process run in each container, in
+                                addition to the container's primary GID and fsGroup (if specified).  If
+                                the SupplementalGroupsPolicy feature is enabled, the
+                                supplementalGroupsPolicy field determines whether these are in addition
+                                to or instead of any group memberships defined in the container image.
+                                If unspecified, no additional groups are added, though group memberships
+                                defined in the container image may still be used, depending on the
+                                supplementalGroupsPolicy field.
                                 Note that this field cannot be set when spec.os.name is windows.
                               type: array
                               items:
                                 type: integer
                                 format: int64
                               x-kubernetes-list-type: atomic
+                            supplementalGroupsPolicy:
+                              description: |-
+                                Defines how supplemental groups of the first container processes are calculated.
+                                Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                                (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                                and the container runtime must implement support for this feature.
+                                Note that this field cannot be set when spec.os.name is windows.
+                              type: string
                             sysctls:
                               description: |-
                                 Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -1375,6 +1425,12 @@ spec:
                                           the Pod where this field is used. It makes that resource available
                                           inside a container.
                                         type: string
+                                      request:
+                                        description: |-
+                                          Request is the name chosen for a request in the referenced claim.
+                                          If empty, everything from the claim is made available, otherwise
+                                          only the result of this request.
+                                        type: string
                                   x-kubernetes-list-map-keys:
                                     - name
                                   x-kubernetes-list-type: map
@@ -1479,7 +1535,7 @@ spec:
                                 procMount:
                                   description: |-
                                     procMount denotes the type of proc mount to use for the containers.
-                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                    The default value is Default which uses the container runtime defaults for
                                     readonly paths and masked paths.
                                     This requires the ProcMountType feature flag to be enabled.
                                     Note that this field cannot be set when spec.os.name is windows.
@@ -1699,6 +1755,12 @@ spec:
                                           the Pod where this field is used. It makes that resource available
                                           inside a container.
                                         type: string
+                                      request:
+                                        description: |-
+                                          Request is the name chosen for a request in the referenced claim.
+                                          If empty, everything from the claim is made available, otherwise
+                                          only the result of this request.
+                                        type: string
                                   x-kubernetes-list-map-keys:
                                     - name
                                   x-kubernetes-list-type: map
@@ -1803,7 +1865,7 @@ spec:
                                 procMount:
                                   description: |-
                                     procMount denotes the type of proc mount to use for the containers.
-                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                    The default value is Default which uses the container runtime defaults for
                                     readonly paths and masked paths.
                                     This requires the ProcMountType feature flag to be enabled.
                                     Note that this field cannot be set when spec.os.name is windows.

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: proxygroups.tailscale.com
 spec:
   group: tailscale.com
@@ -20,9 +20,24 @@ spec:
           jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
           name: Status
           type: string
+        - description: ProxyGroup type.
+          jsonPath: .spec.type
+          name: Type
+          type: string
       name: v1alpha1
       schema:
         openAPIV3Schema:
+          description: |-
+            ProxyGroup defines a set of Tailscale devices that will act as proxies.
+            Currently only egress ProxyGroups are supported.
+
+            Use the tailscale.com/proxy-group annotation on a Service to specify that
+            the egress proxy should be implemented by a ProxyGroup instead of a single
+            dedicated proxy. In addition to running a highly available set of proxies,
+            ProxyGroup also allows for serving many annotated Services from a single
+            set of proxies to minimise resource consumption.
+
+            More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
           type: object
           required:
             - spec
@@ -73,6 +88,7 @@ spec:
                     Defaults to 2.
                   type: integer
                   format: int32
+                  minimum: 0
                 tags:
                   description: |-
                     Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
@@ -86,10 +102,16 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
-                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                  description: |-
+                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                    Type is immutable once a ProxyGroup is created.
                   type: string
                   enum:
                     - egress
+                    - ingress
+                  x-kubernetes-validations:
+                    - rule: self == oldSelf
+                      message: ProxyGroup type is immutable
             status:
               description: |-
                 ProxyGroupStatus describes the status of the ProxyGroup resources. This is

cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    controller-gen.kubebuilder.io/version: v0.17.0
   name: recorders.tailscale.com
 spec:
   group: tailscale.com
@@ -27,6 +27,12 @@ spec:
       name: v1alpha1
       schema:
         openAPIV3Schema:
+          description: |-
+            Recorder defines a tsrecorder device for recording SSH sessions. By default,
+            it will store recordings in a local ephemeral volume. If you want to persist
+            recordings, you can configure an S3-compatible API for storage.
+
+            More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
           type: object
           required:
             - spec
@@ -366,7 +372,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                               Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -381,7 +387,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                               Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -538,7 +544,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                           Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -553,7 +559,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                           Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -711,7 +717,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                               Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -726,7 +732,7 @@ spec:
                                               pod labels will be ignored. The default value is empty.
                                               The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                               Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                              This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                              This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                             type: array
                                             items:
                                               type: string
@@ -883,7 +889,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                           Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -898,7 +904,7 @@ spec:
                                           pod labels will be ignored. The default value is empty.
                                           The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                           Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                          This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                          This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                         type: array
                                         items:
                                           type: string
@@ -1060,6 +1066,12 @@ spec:
                                           the Pod where this field is used. It makes that resource available
                                           inside a container.
                                         type: string
+                                      request:
+                                        description: |-
+                                          Request is the name chosen for a request in the referenced claim.
+                                          If empty, everything from the claim is made available, otherwise
+                                          only the result of this request.
+                                        type: string
                                   x-kubernetes-list-map-keys:
                                     - name
                                   x-kubernetes-list-type: map
@@ -1159,7 +1171,7 @@ spec:
                                 procMount:
                                   description: |-
                                     procMount denotes the type of proc mount to use for the containers.
-                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                    The default value is Default which uses the container runtime defaults for
                                     readonly paths and masked paths.
                                     This requires the ProcMountType feature flag to be enabled.
                                     Note that this field cannot be set when spec.os.name is windows.
@@ -1395,6 +1407,32 @@ spec:
                                 Note that this field cannot be set when spec.os.name is windows.
                               type: integer
                               format: int64
+                            seLinuxChangePolicy:
+                              description: |-
+                                seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                                It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                                Valid values are "MountOption" and "Recursive".
+
+                                "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                                This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                                "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                                This requires all Pods that share the same volume to use the same SELinux label.
+                                It is not possible to share the same volume among privileged and unprivileged Pods.
+                                Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                                whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                                CSIDriver instance. Other volumes are always re-labelled recursively.
+                                "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                                If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                                If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                                and "Recursive" for all other volumes.
+
+                                This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                                All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                                Note that this field cannot be set when spec.os.name is windows.
+                              type: string
                             seLinuxOptions:
                               description: |-
                                 The SELinux context to be applied to all containers.
@@ -1443,18 +1481,28 @@ spec:
                                   type: string
                             supplementalGroups:
                               description: |-
-                                A list of groups applied to the first process run in each container, in addition
-                                to the container's primary GID, the fsGroup (if specified), and group memberships
-                                defined in the container image for the uid of the container process. If unspecified,
-                                no additional groups are added to any container. Note that group memberships
-                                defined in the container image for the uid of the container process are still effective,
-                                even if they are not included in this list.
+                                A list of groups applied to the first process run in each container, in
+                                addition to the container's primary GID and fsGroup (if specified).  If
+                                the SupplementalGroupsPolicy feature is enabled, the
+                                supplementalGroupsPolicy field determines whether these are in addition
+                                to or instead of any group memberships defined in the container image.
+                                If unspecified, no additional groups are added, though group memberships
+                                defined in the container image may still be used, depending on the
+                                supplementalGroupsPolicy field.
                                 Note that this field cannot be set when spec.os.name is windows.
                               type: array
                               items:
                                 type: integer
                                 format: int64
                               x-kubernetes-list-type: atomic
+                            supplementalGroupsPolicy:
+                              description: |-
+                                Defines how supplemental groups of the first container processes are calculated.
+                                Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                                (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                                and the container runtime must implement support for this feature.
+                                Note that this field cannot be set when spec.os.name is windows.
+                              type: string
                             sysctls:
                               description: |-
                                 Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -31,7 +31,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
     annotations:
-        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+        controller-gen.kubebuilder.io/version: v0.17.0
     name: connectors.tailscale.com
 spec:
     group: tailscale.com
@@ -294,7 +294,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
     annotations:
-        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+        controller-gen.kubebuilder.io/version: v0.17.0
     name: dnsconfigs.tailscale.com
 spec:
     group: tailscale.com
@@ -476,7 +476,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
     annotations:
-        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+        controller-gen.kubebuilder.io/version: v0.17.0
     name: proxyclasses.tailscale.com
 spec:
     group: tailscale.com
@@ -563,6 +563,16 @@ spec:
                                             enable:
                                                 description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
                                                 type: boolean
+                                            labels:
+                                                additionalProperties:
+                                                    maxLength: 63
+                                                    pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
+                                                    type: string
+                                                description: |-
+                                                    Labels to add to the ServiceMonitor.
+                                                    Labels must be valid Kubernetes labels.
+                                                    https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+                                                type: object
                                         required:
                                             - enable
                                         type: object
@@ -592,6 +602,8 @@ spec:
                                         type: object
                                     labels:
                                         additionalProperties:
+                                            maxLength: 63
+                                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                                             type: string
                                         description: |-
                                             Labels that will be added to the StatefulSet created for the proxy.
@@ -874,7 +886,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                         Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -889,7 +901,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                         Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -1050,7 +1062,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                 Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -1065,7 +1077,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                 Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -1219,7 +1231,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                         Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -1234,7 +1246,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                         Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -1395,7 +1407,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                 Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -1410,7 +1422,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                 Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -1522,6 +1534,8 @@ spec:
                                                 type: array
                                             labels:
                                                 additionalProperties:
+                                                    maxLength: 63
+                                                    pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                                                     type: string
                                                 description: |-
                                                     Labels that will be added to the proxy Pod.
@@ -1627,6 +1641,32 @@ spec:
                                                             Note that this field cannot be set when spec.os.name is windows.
                                                         format: int64
                                                         type: integer
+                                                    seLinuxChangePolicy:
+                                                        description: |-
+                                                            seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                                                            It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                                                            Valid values are "MountOption" and "Recursive".
+
+                                                            "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                                                            This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                                                            "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                                                            This requires all Pods that share the same volume to use the same SELinux label.
+                                                            It is not possible to share the same volume among privileged and unprivileged Pods.
+                                                            Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                                                            whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                                                            CSIDriver instance. Other volumes are always re-labelled recursively.
+                                                            "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                                                            If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                                                            If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                                                            and "Recursive" for all other volumes.
+
+                                                            This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                                                            All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                                                            Note that this field cannot be set when spec.os.name is windows.
+                                                        type: string
                                                     seLinuxOptions:
                                                         description: |-
                                                             The SELinux context to be applied to all containers.
@@ -1675,18 +1715,28 @@ spec:
                                                         type: object
                                                     supplementalGroups:
                                                         description: |-
-                                                            A list of groups applied to the first process run in each container, in addition
-                                                            to the container's primary GID, the fsGroup (if specified), and group memberships
-                                                            defined in the container image for the uid of the container process. If unspecified,
-                                                            no additional groups are added to any container. Note that group memberships
-                                                            defined in the container image for the uid of the container process are still effective,
-                                                            even if they are not included in this list.
+                                                            A list of groups applied to the first process run in each container, in
+                                                            addition to the container's primary GID and fsGroup (if specified).  If
+                                                            the SupplementalGroupsPolicy feature is enabled, the
+                                                            supplementalGroupsPolicy field determines whether these are in addition
+                                                            to or instead of any group memberships defined in the container image.
+                                                            If unspecified, no additional groups are added, though group memberships
+                                                            defined in the container image may still be used, depending on the
+                                                            supplementalGroupsPolicy field.
                                                             Note that this field cannot be set when spec.os.name is windows.
                                                         items:
                                                             format: int64
                                                             type: integer
                                                         type: array
                                                         x-kubernetes-list-type: atomic
+                                                    supplementalGroupsPolicy:
+                                                        description: |-
+                                                            Defines how supplemental groups of the first container processes are calculated.
+                                                            Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                                                            (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                                                            and the container runtime must implement support for this feature.
+                                                            Note that this field cannot be set when spec.os.name is windows.
+                                                        type: string
                                                     sysctls:
                                                         description: |-
                                                             Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -1837,6 +1887,12 @@ spec:
                                                                                 the Pod where this field is used. It makes that resource available
                                                                                 inside a container.
                                                                             type: string
+                                                                        request:
+                                                                            description: |-
+                                                                                Request is the name chosen for a request in the referenced claim.
+                                                                                If empty, everything from the claim is made available, otherwise
+                                                                                only the result of this request.
+                                                                            type: string
                                                                     required:
                                                                         - name
                                                                     type: object
@@ -1945,7 +2001,7 @@ spec:
                                                             procMount:
                                                                 description: |-
                                                                     procMount denotes the type of proc mount to use for the containers.
-                                                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                                                    The default value is Default which uses the container runtime defaults for
                                                                     readonly paths and masked paths.
                                                                     This requires the ProcMountType feature flag to be enabled.
                                                                     Note that this field cannot be set when spec.os.name is windows.
@@ -2161,6 +2217,12 @@ spec:
                                                                                 the Pod where this field is used. It makes that resource available
                                                                                 inside a container.
                                                                             type: string
+                                                                        request:
+                                                                            description: |-
+                                                                                Request is the name chosen for a request in the referenced claim.
+                                                                                If empty, everything from the claim is made available, otherwise
+                                                                                only the result of this request.
+                                                                            type: string
                                                                     required:
                                                                         - name
                                                                     type: object
@@ -2269,7 +2331,7 @@ spec:
                                                             procMount:
                                                                 description: |-
                                                                     procMount denotes the type of proc mount to use for the containers.
-                                                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                                                    The default value is Default which uses the container runtime defaults for
                                                                     readonly paths and masked paths.
                                                                     This requires the ProcMountType feature flag to be enabled.
                                                                     Note that this field cannot be set when spec.os.name is windows.
@@ -2703,7 +2765,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
     annotations:
-        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+        controller-gen.kubebuilder.io/version: v0.17.0
     name: proxygroups.tailscale.com
 spec:
     group: tailscale.com
@@ -2721,9 +2783,24 @@ spec:
               jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
               name: Status
               type: string
+            - description: ProxyGroup type.
+              jsonPath: .spec.type
+              name: Type
+              type: string
           name: v1alpha1
           schema:
             openAPIV3Schema:
+                description: |-
+                    ProxyGroup defines a set of Tailscale devices that will act as proxies.
+                    Currently only egress ProxyGroups are supported.
+
+                    Use the tailscale.com/proxy-group annotation on a Service to specify that
+                    the egress proxy should be implemented by a ProxyGroup instead of a single
+                    dedicated proxy. In addition to running a highly available set of proxies,
+                    ProxyGroup also allows for serving many annotated Services from a single
+                    set of proxies to minimise resource consumption.
+
+                    More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
                 properties:
                     apiVersion:
                         description: |-
@@ -2767,6 +2844,7 @@ spec:
                                     Replicas specifies how many replicas to create the StatefulSet with.
                                     Defaults to 2.
                                 format: int32
+                                minimum: 0
                                 type: integer
                             tags:
                                 description: |-
@@ -2781,10 +2859,16 @@ spec:
                                     type: string
                                 type: array
                             type:
-                                description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                                description: |-
+                                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                                    Type is immutable once a ProxyGroup is created.
                                 enum:
                                     - egress
+                                    - ingress
                                 type: string
+                                x-kubernetes-validations:
+                                    - message: ProxyGroup type is immutable
+                                      rule: self == oldSelf
                         required:
                             - type
                         type: object
@@ -2891,7 +2975,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
     annotations:
-        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+        controller-gen.kubebuilder.io/version: v0.17.0
     name: recorders.tailscale.com
 spec:
     group: tailscale.com
@@ -2916,6 +3000,12 @@ spec:
           name: v1alpha1
           schema:
             openAPIV3Schema:
+                description: |-
+                    Recorder defines a tsrecorder device for recording SSH sessions. By default,
+                    it will store recordings in a local ephemeral volume. If you want to persist
+                    recordings, you can configure an S3-compatible API for storage.
+
+                    More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
                 properties:
                     apiVersion:
                         description: |-
@@ -3239,7 +3329,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                         Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -3254,7 +3344,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                         Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -3415,7 +3505,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                 Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -3430,7 +3520,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                 Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -3584,7 +3674,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                         Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -3599,7 +3689,7 @@ spec:
                                                                                         pod labels will be ignored. The default value is empty.
                                                                                         The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                         Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                        This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                        This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                                     items:
                                                                                         type: string
                                                                                     type: array
@@ -3760,7 +3850,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both matchLabelKeys and labelSelector.
                                                                                 Also, matchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -3775,7 +3865,7 @@ spec:
                                                                                 pod labels will be ignored. The default value is empty.
                                                                                 The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
                                                                                 Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
-                                                                                This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                                                This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
                                                                             items:
                                                                                 type: string
                                                                             type: array
@@ -3937,6 +4027,12 @@ spec:
                                                                                 the Pod where this field is used. It makes that resource available
                                                                                 inside a container.
                                                                             type: string
+                                                                        request:
+                                                                            description: |-
+                                                                                Request is the name chosen for a request in the referenced claim.
+                                                                                If empty, everything from the claim is made available, otherwise
+                                                                                only the result of this request.
+                                                                            type: string
                                                                     required:
                                                                         - name
                                                                     type: object
@@ -4040,7 +4136,7 @@ spec:
                                                             procMount:
                                                                 description: |-
                                                                     procMount denotes the type of proc mount to use for the containers.
-                                                                    The default is DefaultProcMount which uses the container runtime defaults for
+                                                                    The default value is Default which uses the container runtime defaults for
                                                                     readonly paths and masked paths.
                                                                     This requires the ProcMountType feature flag to be enabled.
                                                                     Note that this field cannot be set when spec.os.name is windows.
@@ -4277,6 +4373,32 @@ spec:
                                                             Note that this field cannot be set when spec.os.name is windows.
                                                         format: int64
                                                         type: integer
+                                                    seLinuxChangePolicy:
+                                                        description: |-
+                                                            seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
+                                                            It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
+                                                            Valid values are "MountOption" and "Recursive".
+
+                                                            "Recursive" means relabeling of all files on all Pod volumes by the container runtime.
+                                                            This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
+
+                                                            "MountOption" mounts all eligible Pod volumes with `-o context` mount option.
+                                                            This requires all Pods that share the same volume to use the same SELinux label.
+                                                            It is not possible to share the same volume among privileged and unprivileged Pods.
+                                                            Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
+                                                            whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
+                                                            CSIDriver instance. Other volumes are always re-labelled recursively.
+                                                            "MountOption" value is allowed only when SELinuxMount feature gate is enabled.
+
+                                                            If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
+                                                            If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
+                                                            and "Recursive" for all other volumes.
+
+                                                            This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
+
+                                                            All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
+                                                            Note that this field cannot be set when spec.os.name is windows.
+                                                        type: string
                                                     seLinuxOptions:
                                                         description: |-
                                                             The SELinux context to be applied to all containers.
@@ -4325,18 +4447,28 @@ spec:
                                                         type: object
                                                     supplementalGroups:
                                                         description: |-
-                                                            A list of groups applied to the first process run in each container, in addition
-                                                            to the container's primary GID, the fsGroup (if specified), and group memberships
-                                                            defined in the container image for the uid of the container process. If unspecified,
-                                                            no additional groups are added to any container. Note that group memberships
-                                                            defined in the container image for the uid of the container process are still effective,
-                                                            even if they are not included in this list.
+                                                            A list of groups applied to the first process run in each container, in
+                                                            addition to the container's primary GID and fsGroup (if specified).  If
+                                                            the SupplementalGroupsPolicy feature is enabled, the
+                                                            supplementalGroupsPolicy field determines whether these are in addition
+                                                            to or instead of any group memberships defined in the container image.
+                                                            If unspecified, no additional groups are added, though group memberships
+                                                            defined in the container image may still be used, depending on the
+                                                            supplementalGroupsPolicy field.
                                                             Note that this field cannot be set when spec.os.name is windows.
                                                         items:
                                                             format: int64
                                                             type: integer
                                                         type: array
                                                         x-kubernetes-list-type: atomic
+                                                    supplementalGroupsPolicy:
+                                                        description: |-
+                                                            Defines how supplemental groups of the first container processes are calculated.
+                                                            Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
+                                                            (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
+                                                            and the container runtime must implement support for this feature.
+                                                            Note that this field cannot be set when spec.os.name is windows.
+                                                        type: string
                                                     sysctls:
                                                         description: |-
                                                             Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -4722,6 +4854,13 @@ rules:
         - get
         - list
         - watch
+        - update
+    - apiGroups:
+        - ""
+      resources:
+        - pods/status
+      verbs:
+        - update
     - apiGroups:
         - apps
       resources:

cmd/k8s-operator/e2e/main_test.go

@@ -21,7 +21,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/internal/client/tailscale"
 )
 
 const (
@@ -64,7 +64,6 @@ func TestMain(m *testing.M) {
 func runTests(m *testing.M) (int, error) {
 	zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar()
 	logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
-	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" {
 		cleanup, err := setupClientAndACLs()

cmd/k8s-operator/egress-eps.go

@@ -20,7 +20,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
 	"tailscale.com/kube/egressservices"
 	"tailscale.com/types/ptr"
 )
@@ -71,25 +70,27 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	if err != nil {
 		return res, fmt.Errorf("error retrieving ExternalName Service: %w", err)
 	}
-	if !tsoperator.EgressServiceIsValidAndConfigured(svc) {
-		l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name)
-		return res, nil
-	}
 
 	// TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is
 	// wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to
 	// determine if a reconcile is needed.
 
 	oldEps := eps.DeepCopy()
-	proxyGroupName := eps.Labels[labelProxyGroup]
 	tailnetSvc := tailnetSvcName(svc)
 	l = l.With("tailnet-service-name", tailnetSvc)
 
 	// Retrieve the desired tailnet service configuration from the ConfigMap.
+	proxyGroupName := eps.Labels[labelProxyGroup]
 	_, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace)
 	if err != nil {
 		return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err)
 	}
+	if cfgs == nil {
+		// TODO(irbekrm): this path would be hit if egress service was once exposed on a ProxyGroup that later
+		// got deleted. Probably the EndpointSlices then need to be deleted too- need to rethink this flow.
+		l.Debugf("No egress config found, likely because ProxyGroup has not been created")
+		return res, nil
+	}
 	cfg, ok := (*cfgs)[tailnetSvc]
 	if !ok {
 		l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc)

cmd/k8s-operator/egress-eps_test.go

@@ -112,7 +112,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
 				Terminating: pointer.ToBool(false),
 			},
 		})
-		expectEqual(t, fc, eps, nil)
+		expectEqual(t, fc, eps)
 	})
 	t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
 		_, stateS := podAndSecretForProxyGroup("foo")           // replica Pod has IP 10.0.0.1
@@ -122,7 +122,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
 		})
 		expectReconciled(t, er, "operator-ns", "foo")
 		eps.Endpoints = []discoveryv1.Endpoint{}
-		expectEqual(t, fc, eps, nil)
+		expectEqual(t, fc, eps)
 	})
 }
 

cmd/k8s-operator/egress-pod-readiness.go

@@ -0,0 +1,274 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/tstime"
+	"tailscale.com/util/httpm"
+)
+
+const tsEgressReadinessGate = "tailscale.com/egress-services"
+
+// egressPodsReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods.
+// The condition is used as a readiness gate for the Pod, meaning that kubelet will not mark the Pod as ready before the
+// condition is set. The ProxyGroup StatefulSet updates are rolled out in such a way that no Pod is restarted, before
+// the previous Pod is marked as ready, so ensuring that the Pod does not get marked as ready when it is not yet able to
+// route traffic for egress service prevents downtime during restarts caused by no available endpoints left because
+// every Pod has been recreated and is not yet added to endpoints.
+// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate
+type egressPodsReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	tsNamespace string
+	clock       tstime.Clock
+	httpClient  doer          // http client that can be set to a mock client in tests
+	maxBackoff  time.Duration // max backoff period between health check calls
+}
+
+// Reconcile reconciles an egress ProxyGroup Pods on changes to those Pods and ProxyGroup EndpointSlices. It ensures
+// that for each Pod who is ready to route traffic to all egress services for the ProxyGroup, the Pod has a
+// tailscale.com/egress-services condition to set, so that kubelet will mark the Pod as ready.
+//
+// For the Pod to be ready
+// to route traffic to the egress service, the kube proxy needs to have set up the Pod's IP as an endpoint for the
+// ClusterIP Service corresponding to the egress service.
+//
+// Note that the endpoints for the ClusterIP Service are configured by the operator itself using custom
+// EndpointSlices(egress-eps-reconciler), so the routing is not blocked on Pod's readiness.
+//
+// Each egress service has a corresponding ClusterIP Service, that exposes all user configured
+// tailnet ports, as well as a health check port for the proxy.
+//
+// The reconciler calls the health check endpoint of each Service up to N number of times, where N is the number of
+// replicas for the ProxyGroup x 3, and checks if the received response is healthy response from the Pod being reconciled.
+//
+// The health check response contains a header with the
+// Pod's IP address- this is used to determine whether the response is received from this Pod.
+//
+// If the Pod does not appear to be serving the health check endpoint (pre-v1.80 proxies), the reconciler just sets the
+// readiness condition for backwards compatibility reasons.
+func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := er.logger.With("Pod", req.NamespacedName)
+	l.Debugf("starting reconcile")
+	defer l.Debugf("reconcile finished")
+
+	pod := new(corev1.Pod)
+	err = er.Get(ctx, req.NamespacedName, pod)
+	if apierrors.IsNotFound(err) {
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get Pod: %w", err)
+	}
+	if !pod.DeletionTimestamp.IsZero() {
+		l.Debugf("Pod is being deleted, do nothing")
+		return res, nil
+	}
+	if pod.Labels[LabelParentType] != proxyTypeProxyGroup {
+		l.Infof("[unexpected] reconciler called for a Pod that is not a ProxyGroup Pod")
+		return res, nil
+	}
+
+	// If the Pod does not have the readiness gate set, there is no need to add the readiness condition. In practice
+	// this will happen if the user has configured custom TS_LOCAL_ADDR_PORT, thus disabling the graceful failover.
+	if !slices.ContainsFunc(pod.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool {
+		return r.ConditionType == tsEgressReadinessGate
+	}) {
+		l.Debug("Pod does not have egress readiness gate set, skipping")
+		return res, nil
+	}
+
+	proxyGroupName := pod.Labels[LabelParentName]
+	pg := new(tsapi.ProxyGroup)
+	if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil {
+		return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err)
+	}
+	if pg.Spec.Type != typeEgress {
+		l.Infof("[unexpected] reconciler called for %q ProxyGroup Pod", pg.Spec.Type)
+		return res, nil
+	}
+	// Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup.
+	lbls := map[string]string{
+		LabelManaged:    "true",
+		labelProxyGroup: proxyGroupName,
+		labelSvcType:    typeEgress,
+	}
+	svcs := &corev1.ServiceList{}
+	if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil {
+		return res, fmt.Errorf("error listing ClusterIP Services")
+	}
+
+	idx := xslices.IndexFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
+		return c.Type == tsEgressReadinessGate
+	})
+	if idx != -1 {
+		l.Debugf("Pod is already ready, do nothing")
+		return res, nil
+	}
+
+	var routesMissing atomic.Bool
+	errChan := make(chan error, len(svcs.Items))
+	for _, svc := range svcs.Items {
+		s := svc
+		go func() {
+			ll := l.With("service_name", s.Name)
+			d := retrieveClusterDomain(er.tsNamespace, ll)
+			healthCheckAddr := healthCheckForSvc(&s, d)
+			if healthCheckAddr == "" {
+				ll.Debugf("ClusterIP Service does not expose a health check endpoint, unable to verify if routing is set up")
+				errChan <- nil
+				return
+			}
+
+			var routesSetup bool
+			bo := backoff.NewBackoff(s.Name, ll.Infof, er.maxBackoff)
+			for range numCalls(pgReplicas(pg)) {
+				if ctx.Err() != nil {
+					errChan <- nil
+					return
+				}
+				state, err := er.lookupPodRouteViaSvc(ctx, pod, healthCheckAddr, ll)
+				if err != nil {
+					errChan <- fmt.Errorf("error validating if routing has been set up for Pod: %w", err)
+					return
+				}
+				if state == healthy || state == cannotVerify {
+					routesSetup = true
+					break
+				}
+				if state == unreachable || state == unhealthy || state == podNotReady {
+					bo.BackOff(ctx, errors.New("backoff"))
+				}
+			}
+			if !routesSetup {
+				ll.Debugf("Pod is not yet configured as Service endpoint")
+				routesMissing.Store(true)
+			}
+			errChan <- nil
+		}()
+	}
+	for range len(svcs.Items) {
+		e := <-errChan
+		err = errors.Join(err, e)
+	}
+	if err != nil {
+		return res, fmt.Errorf("error verifying conectivity: %w", err)
+	}
+	if rm := routesMissing.Load(); rm {
+		l.Info("Pod is not yet added as an endpoint for all egress targets, waiting...")
+		return reconcile.Result{RequeueAfter: shortRequeue}, nil
+	}
+	if err := er.setPodReady(ctx, pod, l); err != nil {
+		return res, fmt.Errorf("error setting Pod as ready: %w", err)
+	}
+	return res, nil
+}
+
+func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, l *zap.SugaredLogger) error {
+	if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
+		return c.Type == tsEgressReadinessGate
+	}) {
+		return nil
+	}
+	l.Infof("Pod is ready to route traffic to all egress targets")
+	pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
+		Type:               tsEgressReadinessGate,
+		Status:             corev1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: er.clock.Now()},
+	})
+	return er.Status().Update(ctx, pod)
+}
+
+// healthCheckState is the result of a single request to an egress Service health check endpoint with a goal to hit a
+// specific backend Pod.
+type healthCheckState int8
+
+const (
+	cannotVerify healthCheckState = iota // not verifiable for this setup (i.e earlier proxy version)
+	unreachable                          // no backends or another network error
+	notFound                             // hit another backend
+	unhealthy                            // not 200
+	podNotReady                          // Pod is not ready, i.e does not have an IP address yet
+	healthy                              // 200
+)
+
+// lookupPodRouteViaSvc attempts to reach a Pod using a health check endpoint served by a Service and returns the state of the health check.
+func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, l *zap.SugaredLogger) (healthCheckState, error) {
+	if !slices.ContainsFunc(pod.Spec.Containers[0].Env, func(e corev1.EnvVar) bool {
+		return e.Name == "TS_ENABLE_HEALTH_CHECK" && e.Value == "true"
+	}) {
+		l.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service")
+		return cannotVerify, nil
+	}
+	wantsIP, err := podIPv4(pod)
+	if err != nil {
+		return -1, fmt.Errorf("error determining Pod's IP address: %w", err)
+	}
+	if wantsIP == "" {
+		return podNotReady, nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, time.Second*3)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, healthCheckAddr, nil)
+	if err != nil {
+		return -1, fmt.Errorf("error creating new HTTP request: %w", err)
+	}
+	// Do not re-use the same connection for the next request so to maximize the chance of hitting all backends equally.
+	req.Close = true
+	resp, err := er.httpClient.Do(req)
+	if err != nil {
+		// This is most likely because this is the first Pod and is not yet added to Service endoints. Other
+		// error types are possible, but checking for those would likely make the system too fragile.
+		return unreachable, nil
+	}
+	defer resp.Body.Close()
+	gotIP := resp.Header.Get(kubetypes.PodIPv4Header)
+	if gotIP == "" {
+		l.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service")
+		return cannotVerify, nil
+	}
+	if !strings.EqualFold(wantsIP, gotIP) {
+		return notFound, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return unhealthy, nil
+	}
+	return healthy, nil
+}
+
+// numCalls return the number of times an endpoint on a ProxyGroup Service should be called till it can be safely
+// assumed that, if none of the responses came back from a specific Pod then traffic for the Service is currently not
+// being routed to that Pod. This assumes that traffic for the Service is routed via round robin, so
+// InternalTrafficPolicy must be 'Cluster' and session affinity must be None.
+func numCalls(replicas int32) int32 {
+	return replicas * 3
+}
+
+// doer is an interface for HTTP client that can be set to a mock client in tests.
+type doer interface {
+	Do(*http.Request) (*http.Response, error)
+}

cmd/k8s-operator/egress-pod-readiness_test.go

@@ -0,0 +1,525 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
+)
+
+func TestEgressPodReadiness(t *testing.T) {
+	// We need to pass a Pod object to WithStatusSubresource because of some quirks in how the fake client
+	// works. Without this code we would not be able to update Pod's status further down.
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithStatusSubresource(&corev1.Pod{}).
+		Build()
+	zl, _ := zap.NewDevelopment()
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	rec := &egressPodsReconciler{
+		tsNamespace: "operator-ns",
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       cl,
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "dev",
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type:     "egress",
+			Replicas: ptr.To(int32(3)),
+		},
+	}
+	mustCreate(t, fc, pg)
+	podIP := "10.0.0.2"
+	podTemplate := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "operator-ns",
+			Name:      "pod",
+			Labels: map[string]string{
+				LabelParentType: "proxygroup",
+				LabelParentName: "dev",
+			},
+		},
+		Spec: corev1.PodSpec{
+			ReadinessGates: []corev1.PodReadinessGate{{
+				ConditionType: tsEgressReadinessGate,
+			}},
+			Containers: []corev1.Container{{
+				Name: "tailscale",
+				Env: []corev1.EnvVar{{
+					Name:  "TS_ENABLE_HEALTH_CHECK",
+					Value: "true",
+				}},
+			}},
+		},
+		Status: corev1.PodStatus{
+			PodIPs: []corev1.PodIP{{IP: podIP}},
+		},
+	}
+
+	t.Run("no_egress_services", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		mustCreate(t, fc, pod)
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod)
+	})
+	t.Run("one_svc_already_routed_to", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		resp := readyResps(podIP, 1)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resp},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+
+		// A subsequent reconcile should not change the Pod.
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+		expectEqual(t, fc, pod)
+
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_many_backends_eventually_routed_to", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only
+		// return with the right Pod IP.
+		resps := append(readyResps("10.0.0.3", 4), append(readyResps("10.0.0.4", 4), readyResps(podIP, 1)...)...)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resps},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_eventually_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only
+		// return with 200 status code.
+		resps := append(unreadyResps(podIP, 8), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resps},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_never_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times and Pod should be
+		// requeued if neither of those succeed.
+		resps := readyResps("10.0.0.3", 9)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resps},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_many_backends_already_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		resps := readyResps(podIP, 1)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps,
+				hep3: resps,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_eventually_routable_and_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...)
+		resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...)
+		resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_never_routable_and_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
+		// will be requeued if neither succeeds.
+		resps := readyResps("10.0.0.3", 9)
+		resps2 := append(readyResps("10.0.0.3", 5), readyResps("10.0.0.4", 4)...)
+		resps3 := unreadyResps(podIP, 9)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_one_never_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
+		// will be requeued if any one never succeeds.
+		resps := readyResps(podIP, 9)
+		resps2 := readyResps(podIP, 9)
+		resps3 := append(readyResps("10.0.0.3", 5), readyResps("10.0.0.4", 4)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_one_never_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
+		// will be requeued if any one never succeeds.
+		resps := readyResps(podIP, 9)
+		resps2 := unreadyResps(podIP, 9)
+		resps3 := readyResps(podIP, 9)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_different_ports_eventually_healthy_and_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9003)
+		svc2, hep2 := newSvc("svc-2", 9004)
+		svc3, hep3 := newSvc("svc-3", 9010)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried up to 9 times and
+		// marked as success as soon as one try succeeds.
+		resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...)
+		resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...)
+		resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	// Proxies of 1.78 and earlier did not set the Pod IP header.
+	t.Run("pod_does_not_return_ip_header", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		pod.Name = "foo-bar"
+
+		svc, hep := newSvc("foo-bar", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// If a response does not contain Pod IP header, we assume that this is an earlier proxy version,
+		// readiness cannot be verified so the readiness gate is just set to true.
+		resps := unreadyResps("", 1)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep: resps,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_eventually_healthy_and_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// If a response errors, it is probably because the Pod is not yet properly running, so retry.
+		resps := append(erroredResps(8), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep: resps,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_svc_does_not_have_health_port", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		// If a Service does not have health port set, we assume that it is not possible to determine Pod's
+		// readiness and set it to ready.
+		svc, _ := newSvc("svc", -1)
+		mustCreateAll(t, fc, svc, pod)
+		rec.httpClient = nil
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("error_setting_up_healthcheck", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		// This is not a realistic reason for error, but we are just testing the behaviour of a healthcheck
+		// lookup failing.
+		pod.Status.PodIPs = []corev1.PodIP{{IP: "not-an-ip"}}
+
+		svc, _ := newSvc("svc", 9002)
+		svc2, _ := newSvc("svc-2", 9002)
+		svc3, _ := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		rec.httpClient = nil
+		expectError(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("pod_does_not_have_an_ip_address", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		pod.Status.PodIPs = nil
+
+		svc, _ := newSvc("svc", 9002)
+		svc2, _ := newSvc("svc-2", 9002)
+		svc3, _ := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		rec.httpClient = nil
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+}
+
+func readyResps(ip string, num int) (resps []fakeResponse) {
+	for range num {
+		resps = append(resps, fakeResponse{statusCode: 200, podIP: ip})
+	}
+	return resps
+}
+
+func unreadyResps(ip string, num int) (resps []fakeResponse) {
+	for range num {
+		resps = append(resps, fakeResponse{statusCode: 503, podIP: ip})
+	}
+	return resps
+}
+
+func erroredResps(num int) (resps []fakeResponse) {
+	for range num {
+		resps = append(resps, fakeResponse{err: errors.New("timeout")})
+	}
+	return resps
+}
+
+func newSvc(name string, port int32) (*corev1.Service, string) {
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "operator-ns",
+			Name:      name,
+			Labels: map[string]string{
+				LabelManaged:    "true",
+				labelProxyGroup: "dev",
+				labelSvcType:    typeEgress,
+			},
+		},
+		Spec: corev1.ServiceSpec{},
+	}
+	if port != -1 {
+		svc.Spec.Ports = []corev1.ServicePort{
+			{
+				Name:       tsHealthCheckPortName,
+				Port:       port,
+				TargetPort: intstr.FromInt(9002),
+				Protocol:   "TCP",
+			},
+		}
+	}
+	return svc, fmt.Sprintf("http://%s.operator-ns.svc.cluster.local:%d/healthz", name, port)
+}
+
+func podSetReady(pod *corev1.Pod, cl *tstest.Clock) {
+	pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
+		Type:               tsEgressReadinessGate,
+		Status:             corev1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+	})
+}
+
+// fakeHTTPClient is a mock HTTP client with a preset map of request URLs to list of responses. When it receives a
+// request for a specific URL, it returns the preset response for that URL. It errors if an unexpected request is
+// received.
+type fakeHTTPClient struct {
+	t     *testing.T
+	mu    sync.Mutex // protects following
+	state map[string][]fakeResponse
+}
+
+func (f *fakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	f.mu.Lock()
+	resps := f.state[req.URL.String()]
+	if len(resps) == 0 {
+		f.mu.Unlock()
+		log.Printf("\n\n\nURL %q\n\n\n", req.URL)
+		f.t.Fatalf("fakeHTTPClient received an unexpected request for %q", req.URL)
+	}
+	defer func() {
+		if len(resps) == 1 {
+			delete(f.state, req.URL.String())
+			f.mu.Unlock()
+			return
+		}
+		f.state[req.URL.String()] = f.state[req.URL.String()][1:]
+		f.mu.Unlock()
+	}()
+
+	resp := resps[0]
+	if resp.err != nil {
+		return nil, resp.err
+	}
+	r := http.Response{
+		StatusCode: resp.statusCode,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(bytes.NewReader([]byte{})),
+	}
+	r.Header.Add(kubetypes.PodIPv4Header, resp.podIP)
+	return &r, nil
+}
+
+type fakeResponse struct {
+	err        error
+	statusCode int
+	podIP      string // for the Pod IP header
+}

cmd/k8s-operator/egress-services-readiness.go

@@ -48,11 +48,12 @@ type egressSvcsReadinessReconciler struct {
 // service to determine how many replicas are currently able to route traffic.
 func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	l := esrr.logger.With("Service", req.NamespacedName)
-	defer l.Info("reconcile finished")
+	l.Debugf("starting reconcile")
+	defer l.Debugf("reconcile finished")
 
 	svc := new(corev1.Service)
 	if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
-		l.Info("Service not found")
+		l.Debugf("Service not found")
 		return res, nil
 	} else if err != nil {
 		return res, fmt.Errorf("failed to get Service: %w", err)
@@ -127,16 +128,16 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
 			return res, err
 		}
 		if pod == nil {
-			l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
+			l.Warnf("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
 			reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
 			return res, nil
 		}
-		l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
+		l.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs)
 		ready := false
 		for _, ep := range eps.Endpoints {
-			l.Infof("looking at endpoint with addresses %v", ep.Addresses)
+			l.Debugf("looking at endpoint with addresses %v", ep.Addresses)
 			if endpointReadyForPod(&ep, pod, l) {
-				l.Infof("endpoint is ready for Pod")
+				l.Debugf("endpoint is ready for Pod")
 				ready = true
 				break
 			}
@@ -165,7 +166,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
 func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
 	podIP, err := podIPv4(pod)
 	if err != nil {
-		l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
+		l.Warnf("[unexpected] error retrieving Pod's IPv4 address: %v", err)
 		return false
 	}
 	// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.

cmd/k8s-operator/egress-services-readiness_test.go

@@ -67,24 +67,24 @@ func TestEgressServiceReadiness(t *testing.T) {
 	setClusterNotReady(egressSvc, cl, zl.Sugar())
 	t.Run("endpointslice_does_not_exist", func(t *testing.T) {
 		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // not ready
+		expectEqual(t, fc, egressSvc) // not ready
 	})
 	t.Run("proxy_group_does_not_exist", func(t *testing.T) {
 		mustCreate(t, fc, eps)
 		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // still not ready
+		expectEqual(t, fc, egressSvc) // still not ready
 	})
 	t.Run("proxy_group_not_ready", func(t *testing.T) {
 		mustCreate(t, fc, pg)
 		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // still not ready
+		expectEqual(t, fc, egressSvc) // still not ready
 	})
 	t.Run("no_ready_replicas", func(t *testing.T) {
 		setPGReady(pg, cl, zl.Sugar())
 		mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
 			p.Status = pg.Status
 		})
-		expectEqual(t, fc, pg, nil)
+		expectEqual(t, fc, pg)
 		for i := range pgReplicas(pg) {
 			p := pod(pg, i)
 			mustCreate(t, fc, p)
@@ -94,7 +94,7 @@ func TestEgressServiceReadiness(t *testing.T) {
 		}
 		expectReconciled(t, rec, "dev", "my-app")
 		setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
-		expectEqual(t, fc, egressSvc, nil) // still not ready
+		expectEqual(t, fc, egressSvc) // still not ready
 	})
 	t.Run("one_ready_replica", func(t *testing.T) {
 		setEndpointForReplica(pg, 0, eps)
@@ -103,7 +103,7 @@ func TestEgressServiceReadiness(t *testing.T) {
 		})
 		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
 		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // partially ready
+		expectEqual(t, fc, egressSvc) // partially ready
 	})
 	t.Run("all_replicas_ready", func(t *testing.T) {
 		for i := range pgReplicas(pg) {
@@ -114,7 +114,7 @@ func TestEgressServiceReadiness(t *testing.T) {
 		})
 		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
 		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // ready
+		expectEqual(t, fc, egressSvc) // ready
 	})
 }
 

cmd/k8s-operator/egress-services.go

@@ -59,6 +59,8 @@ const (
 	maxPorts = 1000
 
 	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
+
+	tsHealthCheckPortName = "tailscale-health-check"
 )
 
 var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
@@ -229,15 +231,16 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 		found := false
 		for _, wantsPM := range svc.Spec.Ports {
 			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
-				// We don't use the port name to distinguish this port internally, but Kubernetes
-				// require that, for Service ports with more than one name each port is uniquely named.
-				// So we can always pick the port name from the ExternalName Service as at this point we
-				// know that those are valid names because Kuberentes already validated it once. Note
-				// that users could have changed an unnamed port to a named port and might have changed
-				// port names- this should still work.
+				// We want to both preserve the user set port names for ease of debugging, but also
+				// ensure that we name all unnamed ports as the ClusterIP Service that we create will
+				// always have at least two ports.
 				// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
 				// See also https://github.com/tailscale/tailscale/issues/13406#issuecomment-2507230388
-				clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
+				if wantsPM.Name != "" {
+					clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
+				} else {
+					clusterIPSvc.Spec.Ports[i].Name = "tailscale-unnamed"
+				}
 				found = true
 				break
 			}
@@ -252,6 +255,12 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 	// ClusterIP Service produce new target port and add a portmapping to
 	// the ClusterIP Service.
 	for _, wantsPM := range svc.Spec.Ports {
+		// Because we add a healthcheck port of our own, we will always have at least two ports. That
+		// means that we cannot have ports with name not set.
+		// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
+		if wantsPM.Name == "" {
+			wantsPM.Name = "tailscale-unnamed"
+		}
 		found := false
 		for _, gotPM := range clusterIPSvc.Spec.Ports {
 			if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) {
@@ -278,6 +287,25 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 			})
 		}
 	}
+	var healthCheckPort int32 = defaultLocalAddrPort
+
+	for {
+		if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool {
+			return p.Port == healthCheckPort
+		}) {
+			break
+		}
+		healthCheckPort++
+		if healthCheckPort > 10002 {
+			return nil, false, fmt.Errorf("unable to find a free port for internal health check in range [9002, 10002]")
+		}
+	}
+	clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
+		Name:       tsHealthCheckPortName,
+		Port:       healthCheckPort,
+		TargetPort: intstr.FromInt(defaultLocalAddrPort),
+		Protocol:   "TCP",
+	})
 	if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
 		if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
 			svc.Labels = clusterIPSvc.Labels
@@ -320,7 +348,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 	}
 	tailnetSvc := tailnetSvcName(svc)
 	gotCfg := (*cfgs)[tailnetSvc]
-	wantsCfg := egressSvcCfg(svc, clusterIPSvc)
+	wantsCfg := egressSvcCfg(svc, clusterIPSvc, esr.tsNamespace, l)
 	if !reflect.DeepEqual(gotCfg, wantsCfg) {
 		l.Debugf("updating egress services ConfigMap %s", cm.Name)
 		mak.Set(cfgs, tailnetSvc, wantsCfg)
@@ -495,13 +523,6 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, err
 	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-
 	if violations := validateEgressService(svc, pg); len(violations) > 0 {
 		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
 		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
@@ -510,11 +531,34 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+	}
+
 	l.Debugf("egress service is valid")
 	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
 	return true, nil
 }
 
+func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, l *zap.SugaredLogger) egressservices.Config {
+	d := retrieveClusterDomain(ns, l)
+	tt := tailnetTargetFromSvc(externalNameSvc)
+	hep := healthCheckForSvc(clusterIPSvc, d)
+	cfg := egressservices.Config{
+		TailnetTarget:       tt,
+		HealthCheckEndpoint: hep,
+	}
+	for _, svcPort := range clusterIPSvc.Spec.Ports {
+		if svcPort.Name == tsHealthCheckPortName {
+			continue // exclude healthcheck from egress svcs configs
+		}
+		pm := portMap(svcPort)
+		mak.Set(&cfg.Ports, pm, struct{}{})
+	}
+	return cfg
+}
+
 func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
 	violations := validateService(svc)
 
@@ -584,19 +628,13 @@ func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
 	}
 }
 
-func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config {
-	tt := tailnetTargetFromSvc(externalNameSvc)
-	cfg := egressservices.Config{TailnetTarget: tt}
-	for _, svcPort := range clusterIPSvc.Spec.Ports {
-		pm := portMap(svcPort)
-		mak.Set(&cfg.Ports, pm, struct{}{})
-	}
-	return cfg
-}
-
 func portMap(p corev1.ServicePort) egressservices.PortMap {
 	// TODO (irbekrm): out of bounds check?
-	return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)}
+	return egressservices.PortMap{
+		Protocol:   string(p.Protocol),
+		MatchPort:  uint16(p.TargetPort.IntVal),
+		TargetPort: uint16(p.Port),
+	}
 }
 
 func isEgressSvcForProxyGroup(obj client.Object) bool {
@@ -618,7 +656,11 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 			Namespace: tsNamespace,
 		},
 	}
-	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
+	err = cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)
+	if apierrors.IsNotFound(err) { // ProxyGroup resources have not been created (yet)
+		return nil, nil, nil
+	}
+	if err != nil {
 		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
 	}
 	cfgs = &egressservices.Configs{}
@@ -740,3 +782,17 @@ func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.
 	svc.Status = *st
 	return err
 }
+
+// healthCheckForSvc return the URL of the containerboot's health check endpoint served by this Service or empty string.
+func healthCheckForSvc(svc *corev1.Service, clusterDomain string) string {
+	// This version of the operator always sets health check port on the egress Services. However, it is possible
+	// that this reconcile loops runs during a proxy upgrade from a version that did not set the health check port
+	// and parses a Service that does not have the port set yet.
+	i := slices.IndexFunc(svc.Spec.Ports, func(port corev1.ServicePort) bool {
+		return port.Name == tsHealthCheckPortName
+	})
+	if i == -1 {
+		return ""
+	}
+	return fmt.Sprintf("http://%s.%s.svc.%s:%d/healthz", svc.Name, svc.Namespace, clusterDomain, svc.Spec.Ports[i].Port)
+}

cmd/k8s-operator/egress-services_test.go

@@ -18,6 +18,7 @@ import (
 	discoveryv1 "k8s.io/api/discovery/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
@@ -78,42 +79,16 @@ func TestTailscaleEgressServices(t *testing.T) {
 			Selector:     nil,
 			Ports: []corev1.ServicePort{
 				{
-					Name:     "http",
 					Protocol: "TCP",
 					Port:     80,
 				},
-				{
-					Name:     "https",
-					Protocol: "TCP",
-					Port:     443,
-				},
 			},
 		},
 	}
 
-	t.Run("proxy_group_not_ready", func(t *testing.T) {
+	t.Run("service_one_unnamed_port", func(t *testing.T) {
 		mustCreate(t, fc, svc)
 		expectReconciled(t, esr, "default", "test")
-		// Service should have EgressSvcValid condition set to Unknown.
-		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
-		expectEqual(t, fc, svc, nil)
-	})
-
-	t.Run("proxy_group_ready", func(t *testing.T) {
-		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
-			pg.Status.Conditions = []metav1.Condition{
-				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
-			}
-		})
-		expectReconciled(t, esr, "default", "test")
-		validateReadyService(t, fc, esr, svc, clock, zl, cm)
-	})
-	t.Run("service_retain_one_unnamed_port", func(t *testing.T) {
-		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80}}
-		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
-			s.Spec.Ports = svc.Spec.Ports
-		})
-		expectReconciled(t, esr, "default", "test")
 		validateReadyService(t, fc, esr, svc, clock, zl, cm)
 	})
 	t.Run("service_add_two_named_ports", func(t *testing.T) {
@@ -162,9 +137,9 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco
 	expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
 	clusterSvc := mustGetClusterIPSvc(t, fc, name)
 	// Verify that an EndpointSlice has been created.
-	expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
+	expectEqual(t, fc, endpointSlice(name, svc, clusterSvc))
 	// Verify that ConfigMap contains configuration for the new egress service.
-	mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
+	mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl)
 	r := svcConfiguredReason(svc, true, zl.Sugar())
 	// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
 	// CluterIP Service.
@@ -174,7 +149,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco
 	}
 	svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
 	svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
-	expectEqual(t, fc, svc, nil)
+	expectEqual(t, fc, svc)
 
 }
 
@@ -203,6 +178,23 @@ func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *c
 
 func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
 	labels := egressSvcChildResourceLabels(extNSvc)
+	ports := make([]corev1.ServicePort, len(extNSvc.Spec.Ports))
+	for i, port := range extNSvc.Spec.Ports {
+		ports[i] = corev1.ServicePort{ // Copy the port to avoid modifying the original.
+			Name:     port.Name,
+			Port:     port.Port,
+			Protocol: port.Protocol,
+		}
+		if port.Name == "" {
+			ports[i].Name = "tailscale-unnamed"
+		}
+	}
+	ports = append(ports, corev1.ServicePort{
+		Name:       "tailscale-health-check",
+		Port:       9002,
+		TargetPort: intstr.FromInt(9002),
+		Protocol:   "TCP",
+	})
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:         name,
@@ -212,7 +204,7 @@ func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
 		},
 		Spec: corev1.ServiceSpec{
 			Type:  corev1.ServiceTypeClusterIP,
-			Ports: extNSvc.Spec.Ports,
+			Ports: ports,
 		},
 	}
 }
@@ -257,9 +249,9 @@ func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
 	return ports
 }
 
-func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
+func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap, l *zap.Logger) {
 	t.Helper()
-	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
+	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc, clusterIPSvc.Namespace, l.Sugar())
 	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
 		t.Fatalf("Error retrieving ConfigMap: %v", err)
 	}

cmd/k8s-operator/ingress-for-pg.go

@@ -0,0 +1,813 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"reflect"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"math/rand/v2"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/internal/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
+)
+
+const (
+	serveConfigKey = "serve-config.json"
+	VIPSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s"
+	// FinalizerNamePG is the finalizer used by the IngressPGReconciler
+	FinalizerNamePG = "tailscale.com/ingress-pg-finalizer"
+
+	indexIngressProxyGroup = ".metadata.annotations.ingress-proxy-group"
+	// annotationHTTPEndpoint can be used to configure the Ingress to expose an HTTP endpoint to tailnet (as
+	// well as the default HTTPS endpoint).
+	annotationHTTPEndpoint = "tailscale.com/http-endpoint"
+)
+
+var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
+
+// HAIngressReconciler is a controller that reconciles Tailscale Ingresses
+// should be exposed on an ingress ProxyGroup (in HA mode).
+type HAIngressReconciler struct {
+	client.Client
+
+	recorder    record.EventRecorder
+	logger      *zap.SugaredLogger
+	tsClient    tsClient
+	tsnetServer tsnetServer
+	tsNamespace string
+	lc          localClient
+	defaultTags []string
+	operatorID  string // stableID of the operator's Tailscale device
+
+	mu sync.Mutex // protects following
+	// managedIngresses is a set of all ingress resources that we're currently
+	// managing. This is only used for metrics.
+	managedIngresses set.Slice[types.UID]
+}
+
+// Reconcile reconciles Ingresses that should be exposed over Tailscale in HA
+// mode (on a ProxyGroup). It looks at all Ingresses with
+// tailscale.com/proxy-group annotation. For each such Ingress, it ensures that
+// a VIPService named after the hostname of the Ingress exists and is up to
+// date. It also ensures that the serve config for the ingress ProxyGroup is
+// updated to route traffic for the VIPService to the Ingress's backend
+// Services.  Ingress hostname change also results in the VIPService for the
+// previous hostname being cleaned up and a new VIPService being created for the
+// new hostname.
+// HA Ingresses support multi-cluster Ingress setup.
+// Each VIPService contains a list of owner references that uniquely identify
+// the Ingress resource and the operator.  When an Ingress that acts as a
+// backend is being deleted, the corresponding VIPService is only deleted if the
+// only owner reference that it contains is for this Ingress. If other owner
+// references are found, then cleanup operation only removes this Ingress' owner
+// reference.
+func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	logger := r.logger.With("Ingress", req.NamespacedName)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	ing := new(networkingv1.Ingress)
+	err = r.Get(ctx, req.NamespacedName, ing)
+	if apierrors.IsNotFound(err) {
+		// Request object not found, could have been deleted after reconcile request.
+		logger.Debugf("Ingress not found, assuming it was deleted")
+		return res, nil
+	} else if err != nil {
+		return res, fmt.Errorf("failed to get Ingress: %w", err)
+	}
+
+	// hostname is the name of the VIPService that will be created for this Ingress as well as the first label in
+	// the MagicDNS name of the Ingress.
+	hostname := hostnameForIngress(ing)
+	logger = logger.With("hostname", hostname)
+
+	// needsRequeue is set to true if the underlying VIPService has changed as a result of this reconcile. If that
+	// is the case, we reconcile the Ingress one more time to ensure that concurrent updates to the VIPService in a
+	// multi-cluster Ingress setup have not resulted in another actor overwriting our VIPService update.
+	needsRequeue := false
+	if !ing.DeletionTimestamp.IsZero() || !r.shouldExpose(ing) {
+		needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger)
+	} else {
+		needsRequeue, err = r.maybeProvision(ctx, hostname, ing, logger)
+	}
+	if err != nil {
+		return res, err
+	}
+	if needsRequeue {
+		res = reconcile.Result{RequeueAfter: requeueInterval()}
+	}
+	return res, nil
+}
+
+// maybeProvision ensures that a VIPService for this Ingress exists and is up to date and that the serve config for the
+// corresponding ProxyGroup contains the Ingress backend's definition.
+// If a VIPService does not exist, it will be created.
+// If a VIPService exists, but only with owner references from other operator instances, an owner reference for this
+// operator instance is added.
+// If a VIPService exists, but does not have an owner reference from any operator, we error
+// out assuming that this is an owner reference created by an unknown actor.
+// Returns true if the operation resulted in a VIPService update.
+func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) (svcsChanged bool, err error) {
+	if err := validateIngressClass(ctx, r.Client); err != nil {
+		logger.Infof("error validating tailscale IngressClass: %v.", err)
+		return false, nil
+	}
+	// Get and validate ProxyGroup readiness
+	pgName := ing.Annotations[AnnotationProxyGroup]
+	if pgName == "" {
+		logger.Infof("[unexpected] no ProxyGroup annotation, skipping VIPService provisioning")
+		return false, nil
+	}
+	logger = logger.With("ProxyGroup", pgName)
+
+	pg := &tsapi.ProxyGroup{}
+	if err := r.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.Infof("ProxyGroup %q does not exist", pgName)
+			return false, nil
+		}
+		return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
+	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		logger.Infof("ProxyGroup %q is not (yet) ready", pgName)
+		return false, nil
+	}
+
+	// Validate Ingress configuration
+	if err := r.validateIngress(ctx, ing, pg); err != nil {
+		logger.Infof("invalid Ingress configuration: %v", err)
+		r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error())
+		return false, nil
+	}
+
+	if !IsHTTPSEnabledOnTailnet(r.tsnetServer) {
+		r.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
+	}
+
+	logger = logger.With("proxy-group", pg.Name)
+
+	if !slices.Contains(ing.Finalizers, FinalizerNamePG) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to tell the operator that the high level,
+		// multi-reconcile operation is underway.
+		logger.Infof("exposing Ingress over tailscale")
+		ing.Finalizers = append(ing.Finalizers, FinalizerNamePG)
+		if err := r.Update(ctx, ing); err != nil {
+			return false, fmt.Errorf("failed to add finalizer: %w", err)
+		}
+		r.mu.Lock()
+		r.managedIngresses.Add(ing.UID)
+		gaugePGIngressResources.Set(int64(r.managedIngresses.Len()))
+		r.mu.Unlock()
+	}
+
+	// 1. Ensure that if Ingress' hostname has changed, any VIPService
+	// resources corresponding to the old hostname are cleaned up.
+	// In practice, this function will ensure that any VIPServices that are
+	// associated with the provided ProxyGroup and no longer owned by an
+	// Ingress are cleaned up. This is fine- it is not expensive and ensures
+	// that in edge cases (a single update changed both hostname and removed
+	// ProxyGroup annotation) the VIPService is more likely to be
+	// (eventually) removed.
+	svcsChanged, err = r.maybeCleanupProxyGroup(ctx, pgName, logger)
+	if err != nil {
+		return false, fmt.Errorf("failed to cleanup VIPService resources for ProxyGroup: %w", err)
+	}
+
+	// 2. Ensure that there isn't a VIPService with the same hostname
+	// already created and not owned by this Ingress.
+	// TODO(irbekrm): perhaps in future we could have record names being
+	// stored on VIPServices. I am not certain if there might not be edge
+	// cases (custom domains, etc?) where attempting to determine the DNS
+	// name of the VIPService in this way won't be incorrect.
+	tcd, err := r.tailnetCertDomain(ctx)
+	if err != nil {
+		return false, fmt.Errorf("error determining DNS name base: %w", err)
+	}
+	dnsName := hostname + "." + tcd
+	serviceName := tailcfg.ServiceName("svc:" + hostname)
+	existingVIPSvc, err := r.tsClient.GetVIPService(ctx, serviceName)
+	// TODO(irbekrm): here and when creating the VIPService, verify if the
+	// error is not terminal (and therefore should not be reconciled). For
+	// example, if the hostname is already a hostname of a Tailscale node,
+	// the GET here will fail.
+	if err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound {
+			return false, fmt.Errorf("error getting VIPService %q: %w", hostname, err)
+		}
+	}
+	// Generate the VIPService comment for new or existing VIPService. This
+	// checks and ensures that VIPService's owner references are updated for
+	// this Ingress and errors if that is not possible (i.e. because it
+	// appears that the VIPService has been created by a non-operator
+	// actor).
+	svcComment, err := r.ownerRefsComment(existingVIPSvc)
+	if err != nil {
+		const instr = "To proceed, you can either manually delete the existing VIPService or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition"
+		msg := fmt.Sprintf("error ensuring ownership of VIPService %s: %v. %s", hostname, err, instr)
+		logger.Warn(msg)
+		r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidVIPService", msg)
+		return false, nil
+	}
+
+	// 3. Ensure that the serve config for the ProxyGroup contains the VIPService.
+	cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
+	if err != nil {
+		return false, fmt.Errorf("error getting Ingress serve config: %w", err)
+	}
+	if cm == nil {
+		logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.")
+		return svcsChanged, nil
+	}
+	ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName))
+	handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, dnsName, logger)
+	if err != nil {
+		return false, fmt.Errorf("failed to get handlers for Ingress: %w", err)
+	}
+	ingCfg := &ipn.ServiceConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{
+			443: {
+				HTTPS: true,
+			},
+		},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{
+			ep: {
+				Handlers: handlers,
+			},
+		},
+	}
+
+	// Add HTTP endpoint if configured.
+	if isHTTPEndpointEnabled(ing) {
+		logger.Infof("exposing Ingress over HTTP")
+		epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", dnsName))
+		ingCfg.TCP[80] = &ipn.TCPPortHandler{
+			HTTP: true,
+		}
+		ingCfg.Web[epHTTP] = &ipn.WebServerConfig{
+			Handlers: handlers,
+		}
+	}
+
+	var gotCfg *ipn.ServiceConfig
+	if cfg != nil && cfg.Services != nil {
+		gotCfg = cfg.Services[serviceName]
+	}
+	if !reflect.DeepEqual(gotCfg, ingCfg) {
+		logger.Infof("Updating serve config")
+		mak.Set(&cfg.Services, serviceName, ingCfg)
+		cfgBytes, err := json.Marshal(cfg)
+		if err != nil {
+			return false, fmt.Errorf("error marshaling serve config: %w", err)
+		}
+		mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
+		if err := r.Update(ctx, cm); err != nil {
+			return false, fmt.Errorf("error updating serve config: %w", err)
+		}
+	}
+
+	// 4. Ensure that the VIPService exists and is up to date.
+	tags := r.defaultTags
+	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
+		tags = strings.Split(tstr, ",")
+	}
+
+	vipPorts := []string{"443"} // always 443 for Ingress
+	if isHTTPEndpointEnabled(ing) {
+		vipPorts = append(vipPorts, "80")
+	}
+
+	vipSvc := &tailscale.VIPService{
+		Name:    serviceName,
+		Tags:    tags,
+		Ports:   vipPorts,
+		Comment: svcComment,
+	}
+	if existingVIPSvc != nil {
+		vipSvc.Addrs = existingVIPSvc.Addrs
+	}
+	// TODO(irbekrm): right now if two Ingress resources attempt to apply different VIPService configs (different
+	// tags, or HTTP endpoint settings) we can end up reconciling those in a loop. We should detect when an Ingress
+	// with the same generation number has been reconciled ~more than N times and stop attempting to apply updates.
+	if existingVIPSvc == nil ||
+		!reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) ||
+		!reflect.DeepEqual(vipSvc.Ports, existingVIPSvc.Ports) ||
+		!strings.EqualFold(vipSvc.Comment, existingVIPSvc.Comment) {
+		logger.Infof("Ensuring VIPService %q exists and is up to date", hostname)
+		if err := r.tsClient.CreateOrUpdateVIPService(ctx, vipSvc); err != nil {
+			return false, fmt.Errorf("error creating VIPService: %w", err)
+		}
+	}
+
+	// 5. Update tailscaled's AdvertiseServices config, which should add the VIPService
+	// IPs to the ProxyGroup Pods' AllowedIPs in the next netmap update if approved.
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, true, logger); err != nil {
+		return false, fmt.Errorf("failed to update tailscaled config: %w", err)
+	}
+
+	// TODO(irbekrm): check that the replicas are ready to route traffic for the VIPService before updating Ingress
+	// status.
+	// 6. Update Ingress status
+	oldStatus := ing.Status.DeepCopy()
+	ports := []networkingv1.IngressPortStatus{
+		{
+			Protocol: "TCP",
+			Port:     443,
+		},
+	}
+	if isHTTPEndpointEnabled(ing) {
+		ports = append(ports, networkingv1.IngressPortStatus{
+			Protocol: "TCP",
+			Port:     80,
+		})
+	}
+	ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
+		{
+			Hostname: dnsName,
+			Ports:    ports,
+		},
+	}
+	if apiequality.Semantic.DeepEqual(oldStatus, ing.Status) {
+		return svcsChanged, nil
+	}
+	if err := r.Status().Update(ctx, ing); err != nil {
+		return false, fmt.Errorf("failed to update Ingress status: %w", err)
+	}
+	return svcsChanged, nil
+}
+
+// VIPServices that are associated with the provided ProxyGroup and no longer managed this operator's instance are deleted, if not owned by other operator instances, else the owner reference is cleaned up.
+// Returns true if the operation resulted in existing VIPService updates (owner reference removal).
+func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) (svcsChanged bool, err error) {
+	// Get serve config for the ProxyGroup
+	cm, cfg, err := r.proxyGroupServeConfig(ctx, proxyGroupName)
+	if err != nil {
+		return false, fmt.Errorf("getting serve config: %w", err)
+	}
+	if cfg == nil {
+		return false, nil // ProxyGroup does not have any VIPServices
+	}
+
+	ingList := &networkingv1.IngressList{}
+	if err := r.List(ctx, ingList); err != nil {
+		return false, fmt.Errorf("listing Ingresses: %w", err)
+	}
+	serveConfigChanged := false
+	// For each VIPService in serve config...
+	for vipServiceName := range cfg.Services {
+		// ...check if there is currently an Ingress with this hostname
+		found := false
+		for _, i := range ingList.Items {
+			ingressHostname := hostnameForIngress(&i)
+			if ingressHostname == vipServiceName.WithoutPrefix() {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipServiceName)
+
+			// Delete the VIPService from control if necessary.
+			svc, _ := r.tsClient.GetVIPService(ctx, vipServiceName)
+			if svc != nil && isVIPServiceForAnyIngress(svc) {
+				logger.Infof("cleaning up orphaned VIPService %q", vipServiceName)
+				svcsChanged, err = r.cleanupVIPService(ctx, vipServiceName, logger)
+				if err != nil {
+					errResp := &tailscale.ErrResponse{}
+					if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
+						return false, fmt.Errorf("deleting VIPService %q: %w", vipServiceName, err)
+					}
+				}
+			}
+
+			// Make sure the VIPService is not advertised in tailscaled or serve config.
+			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
+				return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
+			}
+			delete(cfg.Services, vipServiceName)
+			serveConfigChanged = true
+		}
+	}
+
+	if serveConfigChanged {
+		cfgBytes, err := json.Marshal(cfg)
+		if err != nil {
+			return false, fmt.Errorf("marshaling serve config: %w", err)
+		}
+		mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
+		if err := r.Update(ctx, cm); err != nil {
+			return false, fmt.Errorf("updating serve config: %w", err)
+		}
+	}
+	return svcsChanged, nil
+}
+
+// maybeCleanup ensures that any resources, such as a VIPService created for this Ingress, are cleaned up when the
+// Ingress is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the VIPService is only
+// deleted if it does not contain any other owner references. If it does the cleanup only removes the owner reference
+// corresponding to this Ingress.
+func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) (svcChanged bool, err error) {
+	logger.Debugf("Ensuring any resources for Ingress are cleaned up")
+	ix := slices.Index(ing.Finalizers, FinalizerNamePG)
+	if ix < 0 {
+		logger.Debugf("no finalizer, nothing to do")
+		return false, nil
+	}
+	logger.Infof("Ensuring that VIPService %q configuration is cleaned up", hostname)
+
+	// Ensure that if cleanup succeeded Ingress finalizers are removed.
+	defer func() {
+		if err != nil {
+			return
+		}
+		if e := r.deleteFinalizer(ctx, ing, logger); err != nil {
+			err = errors.Join(err, e)
+		}
+	}()
+
+	// 1. Check if there is a VIPService associated with this Ingress.
+	pg := ing.Annotations[AnnotationProxyGroup]
+	cm, cfg, err := r.proxyGroupServeConfig(ctx, pg)
+	if err != nil {
+		return false, fmt.Errorf("error getting ProxyGroup serve config: %w", err)
+	}
+	serviceName := tailcfg.ServiceName("svc:" + hostname)
+
+	// VIPService is always first added to serve config and only then created in the Tailscale API, so if it is not
+	// found in the serve config, we can assume that there is no VIPService. (If the serve config does not exist at
+	// all, it is possible that the ProxyGroup has been deleted before cleaning up the Ingress, so carry on with
+	// cleanup).
+	if cfg != nil && cfg.Services != nil && cfg.Services[serviceName] == nil {
+		return false, nil
+	}
+
+	// 2. Clean up the VIPService resources.
+	svcChanged, err = r.cleanupVIPService(ctx, serviceName, logger)
+	if err != nil {
+		return false, fmt.Errorf("error deleting VIPService: %w", err)
+	}
+	if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup
+		return svcChanged, nil
+	}
+
+	// 3. Unadvertise the VIPService in tailscaled config.
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
+		return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
+	}
+
+	// 4. Remove the VIPService from the serve config for the ProxyGroup.
+	logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
+	delete(cfg.Services, serviceName)
+	cfgBytes, err := json.Marshal(cfg)
+	if err != nil {
+		return false, fmt.Errorf("error marshaling serve config: %w", err)
+	}
+	mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
+	return svcChanged, r.Update(ctx, cm)
+}
+
+func (r *HAIngressReconciler) deleteFinalizer(ctx context.Context, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
+	found := false
+	ing.Finalizers = slices.DeleteFunc(ing.Finalizers, func(f string) bool {
+		found = true
+		return f == FinalizerNamePG
+	})
+	if !found {
+		return nil
+	}
+	logger.Debug("ensure %q finalizer is removed", FinalizerNamePG)
+
+	if err := r.Update(ctx, ing); err != nil {
+		return fmt.Errorf("failed to remove finalizer %q: %w", FinalizerNamePG, err)
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.managedIngresses.Remove(ing.UID)
+	gaugePGIngressResources.Set(int64(r.managedIngresses.Len()))
+	return nil
+}
+
+func pgIngressCMName(pg string) string {
+	return fmt.Sprintf("%s-ingress-config", pg)
+}
+
+func (r *HAIngressReconciler) proxyGroupServeConfig(ctx context.Context, pg string) (cm *corev1.ConfigMap, cfg *ipn.ServeConfig, err error) {
+	name := pgIngressCMName(pg)
+	cm = &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: r.tsNamespace,
+		},
+	}
+	if err := r.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil && !apierrors.IsNotFound(err) {
+		return nil, nil, fmt.Errorf("error retrieving ingress serve config ConfigMap %s: %v", name, err)
+	}
+	if apierrors.IsNotFound(err) {
+		return nil, nil, nil
+	}
+	cfg = &ipn.ServeConfig{}
+	if len(cm.BinaryData[serveConfigKey]) != 0 {
+		if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
+			return nil, nil, fmt.Errorf("error unmarshaling ingress serve config %v: %w", cm.BinaryData[serveConfigKey], err)
+		}
+	}
+	return cm, cfg, nil
+}
+
+type localClient interface {
+	StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error)
+}
+
+// tailnetCertDomain returns the base domain (TCD) of the current tailnet.
+func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, error) {
+	st, err := r.lc.StatusWithoutPeers(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error getting tailscale status: %w", err)
+	}
+	return st.CurrentTailnet.MagicDNSSuffix, nil
+}
+
+// shouldExpose returns true if the Ingress should be exposed over Tailscale in HA mode (on a ProxyGroup).
+func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
+	isTSIngress := ing != nil &&
+		ing.Spec.IngressClassName != nil &&
+		*ing.Spec.IngressClassName == tailscaleIngressClassName
+	pgAnnot := ing.Annotations[AnnotationProxyGroup]
+	return isTSIngress && pgAnnot != ""
+}
+
+func isVIPServiceForAnyIngress(svc *tailscale.VIPService) bool {
+	if svc == nil {
+		return false
+	}
+	return strings.HasPrefix(svc.Comment, "tailscale.com/k8s-operator:owned-by:")
+}
+
+// validateIngress validates that the Ingress is properly configured.
+// Currently validates:
+// - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags
+// - The derived hostname is a valid DNS label
+// - The referenced ProxyGroup exists and is of type 'ingress'
+// - Ingress' TLS block is invalid
+func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networkingv1.Ingress, pg *tsapi.ProxyGroup) error {
+	var errs []error
+
+	// Validate tags if present
+	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
+		tags := strings.Split(tstr, ",")
+		for _, tag := range tags {
+			tag = strings.TrimSpace(tag)
+			if err := tailcfg.CheckTag(tag); err != nil {
+				errs = append(errs, fmt.Errorf("tailscale.com/tags annotation contains invalid tag %q: %w", tag, err))
+			}
+		}
+	}
+
+	// Validate TLS configuration
+	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) {
+		errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS))
+	}
+
+	// Validate that the hostname will be a valid DNS label
+	hostname := hostnameForIngress(ing)
+	if err := dnsname.ValidLabel(hostname); err != nil {
+		errs = append(errs, fmt.Errorf("invalid hostname %q: %w. Ensure that the hostname is a valid DNS label", hostname, err))
+	}
+
+	// Validate ProxyGroup type
+	if pg.Spec.Type != tsapi.ProxyGroupTypeIngress {
+		errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q",
+			pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress))
+	}
+
+	// Validate ProxyGroup readiness
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name))
+	}
+
+	// It is invalid to have multiple Ingress resources for the same VIPService in one cluster.
+	ingList := &networkingv1.IngressList{}
+	if err := r.List(ctx, ingList); err != nil {
+		errs = append(errs, fmt.Errorf("[unexpected] error listing Ingresses: %w", err))
+		return errors.Join(errs...)
+	}
+	for _, i := range ingList.Items {
+		if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.Name != ing.Name {
+			errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", i.Name, hostname))
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// cleanupVIPService deletes any VIPService by the provided name if it is not owned by operator instances other than this one.
+// If a VIPService is found, but contains other owner references, only removes this operator's owner reference.
+// If a VIPService by the given name is not found or does not contain this operator's owner reference, do nothing.
+// It returns true if an existing VIPService was updated to remove owner reference, as well as any error that occurred.
+func (r *HAIngressReconciler) cleanupVIPService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, _ error) {
+	svc, err := r.tsClient.GetVIPService(ctx, name)
+	if err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
+			return false, nil
+		}
+
+		return false, fmt.Errorf("error getting VIPService: %w", err)
+	}
+	if svc == nil {
+		return false, nil
+	}
+	c, err := parseComment(svc)
+	if err != nil {
+		return false, fmt.Errorf("error parsing VIPService comment")
+	}
+	if c == nil || len(c.OwnerRefs) == 0 {
+		return false, nil
+	}
+	// Comparing with the operatorID only means that we will not be able to
+	// clean up VIPServices in cases where the operator was deleted from the
+	// cluster before deleting the Ingress. Perhaps the comparison could be
+	// 'if or.OperatorID === r.operatorID || or.ingressUID == r.ingressUID'.
+	ix := slices.IndexFunc(c.OwnerRefs, func(or OwnerRef) bool {
+		return or.OperatorID == r.operatorID
+	})
+	if ix == -1 {
+		return false, nil
+	}
+	if len(c.OwnerRefs) == 1 {
+		logger.Infof("Deleting VIPService %q", name)
+		return false, r.tsClient.DeleteVIPService(ctx, name)
+	}
+	c.OwnerRefs = slices.Delete(c.OwnerRefs, ix, ix+1)
+	logger.Infof("Deleting VIPService %q", name)
+	json, err := json.Marshal(c)
+	if err != nil {
+		return false, fmt.Errorf("error marshalling updated VIPService owner reference: %w", err)
+	}
+	svc.Comment = string(json)
+	return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc)
+}
+
+// isHTTPEndpointEnabled returns true if the Ingress has been configured to expose an HTTP endpoint to tailnet.
+func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool {
+	if ing == nil {
+		return false
+	}
+	return ing.Annotations[annotationHTTPEndpoint] == "enabled"
+}
+
+func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, shouldBeAdvertised bool, logger *zap.SugaredLogger) (err error) {
+	logger.Debugf("Updating ProxyGroup tailscaled configs to advertise service %q: %v", serviceName, shouldBeAdvertised)
+
+	// Get all config Secrets for this ProxyGroup.
+	secrets := &corev1.SecretList{}
+	if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil {
+		return fmt.Errorf("failed to list config Secrets: %w", err)
+	}
+
+	for _, secret := range secrets.Items {
+		var updated bool
+		for fileName, confB := range secret.Data {
+			var conf ipn.ConfigVAlpha
+			if err := json.Unmarshal(confB, &conf); err != nil {
+				return fmt.Errorf("error unmarshalling ProxyGroup config: %w", err)
+			}
+
+			// Update the services to advertise if required.
+			idx := slices.Index(conf.AdvertiseServices, serviceName.String())
+			isAdvertised := idx >= 0
+			switch {
+			case isAdvertised == shouldBeAdvertised:
+				// Already up to date.
+				continue
+			case isAdvertised:
+				// Needs to be removed.
+				conf.AdvertiseServices = slices.Delete(conf.AdvertiseServices, idx, idx+1)
+			case shouldBeAdvertised:
+				// Needs to be added.
+				conf.AdvertiseServices = append(conf.AdvertiseServices, serviceName.String())
+			}
+
+			// Update the Secret.
+			confB, err := json.Marshal(conf)
+			if err != nil {
+				return fmt.Errorf("error marshalling ProxyGroup config: %w", err)
+			}
+			mak.Set(&secret.Data, fileName, confB)
+			updated = true
+		}
+
+		if updated {
+			if err := a.Update(ctx, &secret); err != nil {
+				return fmt.Errorf("error updating ProxyGroup config Secret: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// OwnerRef is an owner reference that uniquely identifies a Tailscale
+// Kubernetes operator instance.
+type OwnerRef struct {
+	// OperatorID is the stable ID of the operator's Tailscale device.
+	OperatorID string `json:"operatorID,omitempty"`
+}
+
+// comment is the content of the VIPService.Comment field.
+type comment struct {
+	// OwnerRefs is a list of owner references that identify all operator
+	// instances that manage this VIPService.
+	OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
+}
+
+// ownerRefsComment return VIPService Comment that includes owner reference for this
+// operator instance for the provided VIPService. If the VIPService is nil, a
+// new comment with owner ref is returned. If the VIPService is not nil, the
+// existing comment is returned with the owner reference added, if not already
+// present. If the VIPService is not nil, but does not contain a comment we
+// return an error as this likely means that the VIPService was created by
+// somthing other than a Tailscale Kubernetes operator.
+func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (string, error) {
+	ref := OwnerRef{
+		OperatorID: r.operatorID,
+	}
+	if svc == nil {
+		c := &comment{OwnerRefs: []OwnerRef{ref}}
+		json, err := json.Marshal(c)
+		if err != nil {
+			return "", fmt.Errorf("[unexpected] unable to marshal VIPService comment contents: %w, please report this", err)
+		}
+		return string(json), nil
+	}
+	c, err := parseComment(svc)
+	if err != nil {
+		return "", fmt.Errorf("error parsing existing VIPService comment: %w", err)
+	}
+	if c == nil || len(c.OwnerRefs) == 0 {
+		return "", fmt.Errorf("VIPService %s exists, but does not contain Comment field with owner references- not proceeding as this is likely a resource created by something other than a Tailscale Kubernetes Operator", svc.Name)
+	}
+	if slices.Contains(c.OwnerRefs, ref) { // up to date
+		return svc.Comment, nil
+	}
+	c.OwnerRefs = append(c.OwnerRefs, ref)
+	json, err := json.Marshal(c)
+	if err != nil {
+		return "", fmt.Errorf("error marshalling updated owner references: %w", err)
+	}
+	return string(json), nil
+}
+
+// parseComment returns VIPService comment or nil if none found or not matching the expected format.
+func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
+	if vipSvc.Comment == "" {
+		return nil, nil
+	}
+	c := &comment{}
+	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
+		return nil, fmt.Errorf("error parsing VIPService Comment field %q: %w", vipSvc.Comment, err)
+	}
+	return c, nil
+}
+
+// requeueInterval returns a time duration between 5 and 10 minutes, which is
+// the period of time after which an HA Ingress, whose VIPService has been newly
+// created or changed, needs to be requeued. This is to protect against
+// VIPService owner references being overwritten as a result of concurrent
+// updates during multi-clutster Ingress create/update operations.
+func requeueInterval() time.Duration {
+	return time.Duration(rand.N(5)+5) * time.Minute
+}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -0,0 +1,703 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"reflect"
+	"testing"
+
+	"slices"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"tailscale.com/internal/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/ptr"
+)
+
+func TestIngressPGReconciler(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+
+	// Verify initial reconciliation
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
+	})
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+
+	// Verify VIPService uses custom tags
+	vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
+	if err != nil {
+		t.Fatalf("getting VIPService: %v", err)
+	}
+	if vipSvc == nil {
+		t.Fatal("VIPService not created")
+	}
+	wantTags := []string{"tag:custom", "tag:test"} // custom tags only
+	gotTags := slices.Clone(vipSvc.Tags)
+	slices.Sort(gotTags)
+	slices.Sort(wantTags)
+	if !slices.Equal(gotTags, wantTags) {
+		t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags)
+	}
+
+	// Create second Ingress
+	ing2 := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-other-ingress",
+			Namespace: "default",
+			UID:       types.UID("5678-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-other-svc.tailnetxyz.ts.net"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing2)
+
+	// Verify second Ingress reconciliation
+	expectReconciled(t, ingPGR, "default", "my-other-ingress")
+	verifyServeConfig(t, fc, "svc:my-other-svc", false)
+	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
+
+	// Verify first Ingress is still working
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"})
+
+	// Delete second Ingress
+	if err := fc.Delete(context.Background(), ing2); err != nil {
+		t.Fatalf("deleting second Ingress: %v", err)
+	}
+	expectReconciled(t, ingPGR, "default", "my-other-ingress")
+
+	// Verify second Ingress cleanup
+	cm := &corev1.ConfigMap{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-pg-ingress-config",
+		Namespace: "operator-ns",
+	}, cm); err != nil {
+		t.Fatalf("getting ConfigMap: %v", err)
+	}
+
+	cfg := &ipn.ServeConfig{}
+	if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
+		t.Fatalf("unmarshaling serve config: %v", err)
+	}
+
+	// Verify first Ingress is still configured
+	if cfg.Services["svc:my-svc"] == nil {
+		t.Error("first Ingress service config was incorrectly removed")
+	}
+	// Verify second Ingress was cleaned up
+	if cfg.Services["svc:my-other-svc"] != nil {
+		t.Error("second Ingress service config was not cleaned up")
+	}
+
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+
+	// Delete the first Ingress and verify cleanup
+	if err := fc.Delete(context.Background(), ing); err != nil {
+		t.Fatalf("deleting Ingress: %v", err)
+	}
+
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+
+	// Verify the ConfigMap was cleaned up
+	cm = &corev1.ConfigMap{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-pg-ingress-config",
+		Namespace: "operator-ns",
+	}, cm); err != nil {
+		t.Fatalf("getting ConfigMap: %v", err)
+	}
+
+	cfg = &ipn.ServeConfig{}
+	if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
+		t.Fatalf("unmarshaling serve config: %v", err)
+	}
+
+	if len(cfg.Services) > 0 {
+		t.Error("serve config not cleaned up")
+	}
+	verifyTailscaledConfig(t, fc, nil)
+}
+
+func TestValidateIngress(t *testing.T) {
+	baseIngress := &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			Annotations: map[string]string{
+				AnnotationProxyGroup: "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"test"}},
+			},
+		},
+	}
+
+	readyProxyGroup := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-pg",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type: tsapi.ProxyGroupTypeIngress,
+		},
+		Status: tsapi.ProxyGroupStatus{
+			Conditions: []metav1.Condition{
+				{
+					Type:               string(tsapi.ProxyGroupReady),
+					Status:             metav1.ConditionTrue,
+					ObservedGeneration: 1,
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name         string
+		ing          *networkingv1.Ingress
+		pg           *tsapi.ProxyGroup
+		existingIngs []networkingv1.Ingress
+		wantErr      string
+	}{
+		{
+			name: "valid_ingress_with_hostname",
+			ing: &networkingv1.Ingress{
+				ObjectMeta: baseIngress.ObjectMeta,
+				Spec: networkingv1.IngressSpec{
+					TLS: []networkingv1.IngressTLS{
+						{Hosts: []string{"test.example.com"}},
+					},
+				},
+			},
+			pg: readyProxyGroup,
+		},
+		{
+			name: "valid_ingress_with_default_hostname",
+			ing:  baseIngress,
+			pg:   readyProxyGroup,
+		},
+		{
+			name: "invalid_tags",
+			ing: &networkingv1.Ingress{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      baseIngress.Name,
+					Namespace: baseIngress.Namespace,
+					Annotations: map[string]string{
+						AnnotationTags: "tag:invalid!",
+					},
+				},
+			},
+			pg:      readyProxyGroup,
+			wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes",
+		},
+		{
+			name: "multiple_TLS_entries",
+			ing: &networkingv1.Ingress{
+				ObjectMeta: baseIngress.ObjectMeta,
+				Spec: networkingv1.IngressSpec{
+					TLS: []networkingv1.IngressTLS{
+						{Hosts: []string{"test1.example.com"}},
+						{Hosts: []string{"test2.example.com"}},
+					},
+				},
+			},
+			pg:      readyProxyGroup,
+			wantErr: "Ingress contains invalid TLS block [{[test1.example.com] } {[test2.example.com] }]: only a single TLS entry with a single host is allowed",
+		},
+		{
+			name: "multiple_hosts_in_TLS_entry",
+			ing: &networkingv1.Ingress{
+				ObjectMeta: baseIngress.ObjectMeta,
+				Spec: networkingv1.IngressSpec{
+					TLS: []networkingv1.IngressTLS{
+						{Hosts: []string{"test1.example.com", "test2.example.com"}},
+					},
+				},
+			},
+			pg:      readyProxyGroup,
+			wantErr: "Ingress contains invalid TLS block [{[test1.example.com test2.example.com] }]: only a single TLS entry with a single host is allowed",
+		},
+		{
+			name: "wrong_proxy_group_type",
+			ing:  baseIngress,
+			pg: &tsapi.ProxyGroup{
+				ObjectMeta: readyProxyGroup.ObjectMeta,
+				Spec: tsapi.ProxyGroupSpec{
+					Type: tsapi.ProxyGroupType("foo"),
+				},
+				Status: readyProxyGroup.Status,
+			},
+			wantErr: "ProxyGroup \"test-pg\" is of type \"foo\" but must be of type \"ingress\"",
+		},
+		{
+			name: "proxy_group_not_ready",
+			ing:  baseIngress,
+			pg: &tsapi.ProxyGroup{
+				ObjectMeta: readyProxyGroup.ObjectMeta,
+				Spec:       readyProxyGroup.Spec,
+				Status: tsapi.ProxyGroupStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:               string(tsapi.ProxyGroupReady),
+							Status:             metav1.ConditionFalse,
+							ObservedGeneration: 1,
+						},
+					},
+				},
+			},
+			wantErr: "ProxyGroup \"test-pg\" is not ready",
+		},
+		{
+			name: "duplicate_hostname",
+			ing:  baseIngress,
+			pg:   readyProxyGroup,
+			existingIngs: []networkingv1.Ingress{{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "existing-ingress",
+					Namespace: "default",
+					Annotations: map[string]string{
+						AnnotationProxyGroup: "test-pg",
+					},
+				},
+				Spec: networkingv1.IngressSpec{
+					IngressClassName: ptr.To("tailscale"),
+					TLS: []networkingv1.IngressTLS{
+						{Hosts: []string{"test"}},
+					},
+				},
+			}},
+			wantErr: `found duplicate Ingress "existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fc := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme).
+				WithObjects(tt.ing).
+				WithLists(&networkingv1.IngressList{Items: tt.existingIngs}).
+				Build()
+			r := &HAIngressReconciler{Client: fc}
+			err := r.validateIngress(context.Background(), tt.ing, tt.pg)
+			if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) {
+				t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+
+	// Create test Ingress with HTTP endpoint enabled
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group":   "test-pg",
+				"tailscale.com/http-endpoint": "enabled",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc"}},
+			},
+		},
+	}
+	if err := fc.Create(context.Background(), ing); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify initial reconciliation with HTTP enabled
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyVIPService(t, ft, "svc:my-svc", []string{"80", "443"})
+	verifyServeConfig(t, fc, "svc:my-svc", true)
+
+	// Verify Ingress status
+	ing = &networkingv1.Ingress{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-ingress",
+		Namespace: "default",
+	}, ing); err != nil {
+		t.Fatal(err)
+	}
+
+	wantStatus := []networkingv1.IngressPortStatus{
+		{Port: 443, Protocol: "TCP"},
+		{Port: 80, Protocol: "TCP"},
+	}
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
+		t.Errorf("incorrect status ports: got %v, want %v",
+			ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
+	}
+
+	// Remove HTTP endpoint annotation
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		delete(ing.Annotations, "tailscale.com/http-endpoint")
+	})
+
+	// Verify reconciliation after removing HTTP
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+
+	// Verify Ingress status
+	ing = &networkingv1.Ingress{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-ingress",
+		Namespace: "default",
+	}, ing); err != nil {
+		t.Fatal(err)
+	}
+
+	wantStatus = []networkingv1.IngressPortStatus{
+		{Port: 443, Protocol: "TCP"},
+	}
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
+		t.Errorf("incorrect status ports: got %v, want %v",
+			ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
+	}
+}
+
+func verifyVIPService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) {
+	t.Helper()
+	vipSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName))
+	if err != nil {
+		t.Fatalf("getting VIPService %q: %v", serviceName, err)
+	}
+	if vipSvc == nil {
+		t.Fatalf("VIPService %q not created", serviceName)
+	}
+	gotPorts := slices.Clone(vipSvc.Ports)
+	slices.Sort(gotPorts)
+	slices.Sort(wantPorts)
+	if !slices.Equal(gotPorts, wantPorts) {
+		t.Errorf("incorrect ports for VIPService %q: got %v, want %v", serviceName, gotPorts, wantPorts)
+	}
+}
+
+func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantHTTP bool) {
+	t.Helper()
+
+	cm := &corev1.ConfigMap{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-pg-ingress-config",
+		Namespace: "operator-ns",
+	}, cm); err != nil {
+		t.Fatalf("getting ConfigMap: %v", err)
+	}
+
+	cfg := &ipn.ServeConfig{}
+	if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil {
+		t.Fatalf("unmarshaling serve config: %v", err)
+	}
+
+	t.Logf("Looking for service %q in config: %+v", serviceName, cfg)
+
+	svc := cfg.Services[tailcfg.ServiceName(serviceName)]
+	if svc == nil {
+		t.Fatalf("service %q not found in serve config, services: %+v", serviceName, maps.Keys(cfg.Services))
+	}
+
+	wantHandlers := 1
+	if wantHTTP {
+		wantHandlers = 2
+	}
+
+	// Check TCP handlers
+	if len(svc.TCP) != wantHandlers {
+		t.Errorf("incorrect number of TCP handlers for service %q: got %d, want %d", serviceName, len(svc.TCP), wantHandlers)
+	}
+	if wantHTTP {
+		if h, ok := svc.TCP[uint16(80)]; !ok {
+			t.Errorf("HTTP (port 80) handler not found for service %q", serviceName)
+		} else if !h.HTTP {
+			t.Errorf("HTTP not enabled for port 80 handler for service %q", serviceName)
+		}
+	}
+	if h, ok := svc.TCP[uint16(443)]; !ok {
+		t.Errorf("HTTPS (port 443) handler not found for service %q", serviceName)
+	} else if !h.HTTPS {
+		t.Errorf("HTTPS not enabled for port 443 handler for service %q", serviceName)
+	}
+
+	// Check Web handlers
+	if len(svc.Web) != wantHandlers {
+		t.Errorf("incorrect number of Web handlers for service %q: got %d, want %d", serviceName, len(svc.Web), wantHandlers)
+	}
+}
+
+func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) {
+	var expected string
+	if expectedServices != nil {
+		expectedServicesJSON, err := json.Marshal(expectedServices)
+		if err != nil {
+			t.Fatalf("marshaling expected services: %v", err)
+		}
+		expected = fmt.Sprintf(`,"AdvertiseServices":%s`, expectedServicesJSON)
+	}
+	expectEqual(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgConfigSecretName("test-pg", 0),
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "config"),
+		},
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
+		},
+	})
+}
+
+func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) {
+
+	tsIngressClass := &networkingv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
+		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
+	}
+
+	// Pre-create the ProxyGroup
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-pg",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type: tsapi.ProxyGroupTypeIngress,
+		},
+	}
+
+	// Pre-create the ConfigMap for the ProxyGroup
+	pgConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg-ingress-config",
+			Namespace: "operator-ns",
+		},
+		BinaryData: map[string][]byte{
+			"serve-config.json": []byte(`{"Services":{}}`),
+		},
+	}
+
+	// Pre-create a config Secret for the ProxyGroup
+	pgCfgSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgConfigSecretName("test-pg", 0),
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "config"),
+		},
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): []byte("{}"),
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pg, pgCfgSecret, pgConfigMap, tsIngressClass).
+		WithStatusSubresource(pg).
+		Build()
+
+	// Set ProxyGroup status to ready
+	pg.Status.Conditions = []metav1.Condition{
+		{
+			Type:               string(tsapi.ProxyGroupReady),
+			Status:             metav1.ConditionTrue,
+			ObservedGeneration: 1,
+		},
+	}
+	if err := fc.Status().Update(context.Background(), pg); err != nil {
+		t.Fatal(err)
+	}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lc := &fakeLocalClient{
+		status: &ipnstate.Status{
+			CurrentTailnet: &ipnstate.TailnetStatus{
+				MagicDNSSuffix: "ts.net",
+			},
+		},
+	}
+
+	ingPGR := &HAIngressReconciler{
+		Client:      fc,
+		tsClient:    ft,
+		defaultTags: []string{"tag:k8s"},
+		tsNamespace: "operator-ns",
+		tsnetServer: fakeTsnetServer,
+		logger:      zl.Sugar(),
+		recorder:    record.NewFakeRecorder(10),
+		lc:          lc,
+	}
+
+	return ingPGR, fc, ft
+}
+
+func TestIngressPGReconciler_MultiCluster(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+	ingPGR.operatorID = "operator-1"
+
+	// Create initial Ingress
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+
+	// Simulate existing VIPService from another cluster
+	existingVIPSvc := &tailscale.VIPService{
+		Name:    "svc:my-svc",
+		Comment: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
+	}
+	ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{
+		"svc:my-svc": existingVIPSvc,
+	}
+
+	// Verify reconciliation adds our operator reference
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+
+	vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
+	if err != nil {
+		t.Fatalf("getting VIPService: %v", err)
+	}
+	if vipSvc == nil {
+		t.Fatal("VIPService not found")
+	}
+
+	c := &comment{}
+	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
+		t.Fatalf("parsing comment: %v", err)
+	}
+
+	wantOwnerRefs := []OwnerRef{
+		{OperatorID: "operator-2"},
+		{OperatorID: "operator-1"},
+	}
+	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
+	}
+
+	// Delete the Ingress and verify VIPService still exists with one owner ref
+	if err := fc.Delete(context.Background(), ing); err != nil {
+		t.Fatalf("deleting Ingress: %v", err)
+	}
+	expectRequeue(t, ingPGR, "default", "test-ingress")
+
+	vipSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc")
+	if err != nil {
+		t.Fatalf("getting VIPService after deletion: %v", err)
+	}
+	if vipSvc == nil {
+		t.Fatal("VIPService was incorrectly deleted")
+	}
+
+	c = &comment{}
+	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
+		t.Fatalf("parsing comment after deletion: %v", err)
+	}
+
+	wantOwnerRefs = []OwnerRef{
+		{OperatorID: "operator-2"},
+	}
+	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
+	}
+}

cmd/k8s-operator/ingress.go

@@ -26,6 +26,7 @@ import (
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
 )
 
@@ -58,7 +59,7 @@ var (
 )
 
 func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := a.logger.With("ingress-ns", req.Namespace, "ingress-name", req.Name)
+	logger := a.logger.With("Ingress", req.NamespacedName)
 	logger.Debugf("starting reconcile")
 	defer logger.Debugf("reconcile finished")
 
@@ -72,6 +73,7 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get ing: %w", err)
 	}
 	if !ing.DeletionTimestamp.IsZero() || !a.shouldExpose(ing) {
+		// TODO(irbekrm): this message is confusing if the Ingress is an HA Ingress
 		logger.Debugf("ingress is being deleted or should not be exposed, cleaning up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, ing)
 	}
@@ -128,9 +130,8 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 // This function adds a finalizer to ing, ensuring that we can handle orderly
 // deprovisioning later.
 func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
-	if err := a.validateIngressClass(ctx); err != nil {
+	if err := validateIngressClass(ctx, a.Client); err != nil {
 		logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err)
-
 	}
 	if !slices.Contains(ing.Finalizers, FinalizerName) {
 		// This log line is printed exactly once during initial provisioning,
@@ -159,7 +160,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
 	a.mu.Unlock()
 
-	if !a.ssr.IsHTTPSEnabledOnTailnet() {
+	if !IsHTTPSEnabledOnTailnet(a.ssr.tsnetServer) {
 		a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
 	}
 
@@ -185,73 +186,16 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	}
 
 	web := sc.Web[magic443]
-	addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
-		if b == nil {
-			return
-		}
-		if b.Service == nil {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
-			return
-		}
-		var svc corev1.Service
-		if err := a.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
-			return
-		}
-		if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
-			return
-		}
-		var port int32
-		if b.Service.Port.Name != "" {
-			for _, p := range svc.Spec.Ports {
-				if p.Name == b.Service.Port.Name {
-					port = p.Port
-					break
-				}
-			}
-		} else {
-			port = b.Service.Port.Number
-		}
-		if port == 0 {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
-			return
-		}
-		proto := "http://"
-		if port == 443 || b.Service.Port.Name == "https" {
-			proto = "https+insecure://"
-		}
-		web.Handlers[path] = &ipn.HTTPHandler{
-			Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
-		}
-	}
-	addIngressBackend(ing.Spec.DefaultBackend, "/")
 
 	var tlsHost string // hostname or FQDN or empty
 	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
 		tlsHost = ing.Spec.TLS[0].Hosts[0]
 	}
-	for _, rule := range ing.Spec.Rules {
-		// Host is optional, but if it's present it must match the TLS host
-		// otherwise we ignore the rule.
-		if rule.Host != "" && rule.Host != tlsHost {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
-			continue
-		}
-		for _, p := range rule.HTTP.Paths {
-			// Send a warning if folks use Exact path type - to make
-			// it easier for us to support Exact path type matching
-			// in the future if needed.
-			// https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types
-			if *p.PathType == networkingv1.PathTypeExact {
-				msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future."
-				logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg))
-				a.recorder.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg)
-			}
-			addIngressBackend(&p.Backend, p.Path)
-		}
+	handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, tlsHost, logger)
+	if err != nil {
+		return fmt.Errorf("failed to get handlers for ingress: %w", err)
 	}
-
+	web.Handlers = handlers
 	if len(web.Handlers) == 0 {
 		logger.Warn("Ingress contains no valid backends")
 		a.recorder.Eventf(ing, corev1.EventTypeWarning, "NoValidBackends", "no valid backends")
@@ -263,10 +207,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
 		tags = strings.Split(tstr, ",")
 	}
-	hostname := ing.Namespace + "-" + ing.Name + "-ingress"
-	if tlsHost != "" {
-		hostname, _, _ = strings.Cut(tlsHost, ".")
-	}
+	hostname := hostnameForIngress(ing)
 
 	sts := &tailscaleSTSConfig{
 		Hostname:            hostname,
@@ -322,28 +263,106 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
 	return ing != nil &&
 		ing.Spec.IngressClassName != nil &&
-		*ing.Spec.IngressClassName == tailscaleIngressClassName
+		*ing.Spec.IngressClassName == tailscaleIngressClassName &&
+		ing.Annotations[AnnotationProxyGroup] == ""
 }
 
 // validateIngressClass attempts to validate that 'tailscale' IngressClass
 // included in Tailscale installation manifests exists and has not been modified
 // to attempt to enable features that we do not support.
-func (a *IngressReconciler) validateIngressClass(ctx context.Context) error {
+func validateIngressClass(ctx context.Context, cl client.Client) error {
 	ic := &networkingv1.IngressClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: tailscaleIngressClassName,
 		},
 	}
-	if err := a.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) {
-		return errors.New("Tailscale IngressClass not found in cluster. Latest installation manifests include a tailscale IngressClass - please update")
+	if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) {
+		return errors.New("'tailscale' IngressClass not found in cluster.")
 	} else if err != nil {
 		return fmt.Errorf("error retrieving 'tailscale' IngressClass: %w", err)
 	}
 	if ic.Spec.Controller != tailscaleIngressControllerName {
-		return fmt.Errorf("Tailscale Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName)
+		return fmt.Errorf("'tailscale' Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName)
 	}
 	if ic.GetAnnotations()[ingressClassDefaultAnnotation] != "" {
 		return fmt.Errorf("%s annotation is set on 'tailscale' IngressClass, but Tailscale Ingress controller does not support default Ingress class. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ingressClassDefaultAnnotation)
 	}
 	return nil
 }
+
+func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHost string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) {
+	addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
+		if b == nil {
+			return
+		}
+		if b.Service == nil {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
+			return
+		}
+		var svc corev1.Service
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
+			return
+		}
+		if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
+			return
+		}
+		var port int32
+		if b.Service.Port.Name != "" {
+			for _, p := range svc.Spec.Ports {
+				if p.Name == b.Service.Port.Name {
+					port = p.Port
+					break
+				}
+			}
+		} else {
+			port = b.Service.Port.Number
+		}
+		if port == 0 {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
+			return
+		}
+		proto := "http://"
+		if port == 443 || b.Service.Port.Name == "https" {
+			proto = "https+insecure://"
+		}
+		mak.Set(&handlers, path, &ipn.HTTPHandler{
+			Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
+		})
+	}
+	addIngressBackend(ing.Spec.DefaultBackend, "/")
+	for _, rule := range ing.Spec.Rules {
+		// Host is optional, but if it's present it must match the TLS host
+		// otherwise we ignore the rule.
+		if rule.Host != "" && rule.Host != tlsHost {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
+			continue
+		}
+		for _, p := range rule.HTTP.Paths {
+			// Send a warning if folks use Exact path type - to make
+			// it easier for us to support Exact path type matching
+			// in the future if needed.
+			// https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types
+			if *p.PathType == networkingv1.PathTypeExact {
+				msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future."
+				logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg))
+				rec.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg)
+			}
+			addIngressBackend(&p.Backend, p.Path)
+		}
+	}
+	return handlers, nil
+}
+
+// hostnameForIngress returns the hostname for an Ingress resource.
+// If the Ingress has TLS configured with a host, it returns the first component of that host.
+// Otherwise, it returns a hostname derived from the Ingress name and namespace.
+func hostnameForIngress(ing *networkingv1.Ingress) string {
+	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
+		h := ing.Spec.TLS[0].Hosts[0]
+		hostname, _, _ := strings.Cut(h, ".")
+		return hostname
+	}
+	return ing.Namespace + "-" + ing.Name + "-ingress"
+}

cmd/k8s-operator/ingress_test.go

@@ -103,9 +103,9 @@ func TestTailscaleIngress(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Ingress status gets updated with ingress proxy's MagicDNS name
 	// once that becomes available.
@@ -120,7 +120,7 @@ func TestTailscaleIngress(t *testing.T) {
 			{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
 		},
 	}
-	expectEqual(t, fc, ing, nil)
+	expectEqual(t, fc, ing)
 
 	// 3. Resources get created for Ingress that should allow forwarding
 	// cluster traffic
@@ -129,7 +129,7 @@ func TestTailscaleIngress(t *testing.T) {
 	})
 	opts.shouldEnableForwardingClusterTrafficViaIngress = true
 	expectReconciled(t, ingR, "default", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 4. Resources get cleaned up when Ingress class is unset
 	mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
@@ -229,9 +229,9 @@ func TestTailscaleIngressHostname(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set
 	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
@@ -243,7 +243,7 @@ func TestTailscaleIngressHostname(t *testing.T) {
 	expectReconciled(t, ingR, "default", "test")
 	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 
-	expectEqual(t, fc, ing, nil)
+	expectEqual(t, fc, ing)
 
 	// 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint
 	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
@@ -259,7 +259,7 @@ func TestTailscaleIngressHostname(t *testing.T) {
 			{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
 		},
 	}
-	expectEqual(t, fc, ing, nil)
+	expectEqual(t, fc, ing)
 
 	// 4. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint ready
 	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
@@ -271,7 +271,7 @@ func TestTailscaleIngressHostname(t *testing.T) {
 	})
 	expectReconciled(t, ingR, "default", "test")
 	ing.Status.LoadBalancer.Ingress = nil
-	expectEqual(t, fc, ing, nil)
+	expectEqual(t, fc, ing)
 
 	// 5. Ingress proxy's state has https_endpoints set, but its capver is not matching Pod UID (downgrade)
 	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
@@ -287,7 +287,7 @@ func TestTailscaleIngressHostname(t *testing.T) {
 		},
 	}
 	expectReconciled(t, ingR, "default", "test")
-	expectEqual(t, fc, ing, nil)
+	expectEqual(t, fc, ing)
 }
 
 func TestTailscaleIngressWithProxyClass(t *testing.T) {
@@ -295,7 +295,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
 		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
+			Labels:      tsapi.Labels{"foo": "bar"},
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
@@ -383,9 +383,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet
 	// ready, so proxy resource configuration does not change.
@@ -393,7 +393,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata")
 	})
 	expectReconciled(t, ingR, "default", "test")
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get
 	// reconciled and configuration from the ProxyClass is applied to the
@@ -408,7 +408,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, ingR, "default", "test")
 	opts.proxyClass = pc.Name
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 4. tailscale.com/proxy-class label is removed from the Ingress, the
 	// Ingress gets reconciled and the custom ProxyClass configuration is
@@ -418,18 +418,13 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, ingR, "default", "test")
 	opts.proxyClass = ""
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 }
 
 func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
-		Spec: tsapi.ProxyClassSpec{
-			Metrics: &tsapi.Metrics{
-				Enable:         true,
-				ServiceMonitor: &tsapi.ServiceMonitor{Enable: true},
-			},
-		},
+		Spec:       tsapi.ProxyClassSpec{},
 		Status: tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
@@ -437,32 +432,6 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				ObservedGeneration: 1,
 			}}},
 	}
-	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass).
-		WithStatusSubresource(pc).
-		Build()
-	ft := &fakeTSClient{}
-	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	ingR := &IngressReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			tsnetServer:       fakeTsnetServer,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger: zl.Sugar(),
-	}
-	// 1. Enable metrics- expect metrics Service to be created
 	ing := &networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
 		ObjectMeta: metav1.ObjectMeta{
@@ -491,8 +460,7 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 			},
 		},
 	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
+	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -504,11 +472,38 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				Name: "http"},
 			},
 		},
-	})
-
+	}
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pc, tsIngressClass, ing, svc).
+		WithStatusSubresource(pc).
+		Build()
+	ft := &fakeTSClient{}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ingR := &IngressReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			tsnetServer:       fakeTsnetServer,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
 	expectReconciled(t, ingR, "default", "test")
-
 	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
+	serveConfig := &ipn.ServeConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
+	}
 	opts := configOpts{
 		stsName:            shortName,
 		secretName:         fullName,
@@ -517,27 +512,51 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 		parentType:         "ingress",
 		hostname:           "default-test",
 		app:                kubetypes.AppIngressResource,
-		enableMetrics:      true,
 		namespaced:         true,
 		proxyType:          proxyTypeIngressResource,
+		serveConfig:        serveConfig,
+		resourceVersion:    "1",
 	}
-	serveConfig := &ipn.ServeConfig{
-		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
-		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
-	}
-	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
-	expectEqual(t, fc, expectedMetricsService(opts), nil)
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	// 1. Enable metrics- expect metrics Service to be created
+	mustUpdate(t, fc, "", "metrics", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics = &tsapi.Metrics{Enable: true}
+	})
+	opts.enableMetrics = true
+
+	expectReconciled(t, ingR, "default", "test")
+
+	expectEqual(t, fc, expectedMetricsService(opts))
+
 	// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
 	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
-		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
+		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true, Labels: tsapi.Labels{"foo": "bar"}}
 	})
 	expectReconciled(t, ingR, "default", "test")
+	expectEqual(t, fc, expectedMetricsService(opts))
+
 	// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
 	mustCreate(t, fc, crd)
 	expectReconciled(t, ingR, "default", "test")
+	opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
+	expectEqual(t, fc, expectedMetricsService(opts))
 	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 4. Update ServiceMonitor CRD and reconcile- ServiceMonitor should get updated
+	mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics.ServiceMonitor.Labels = nil
+	})
+	expectReconciled(t, ingR, "default", "test")
+	opts.serviceMonitorLabels = nil
+	opts.resourceVersion = "2"
+	expectEqual(t, fc, expectedMetricsService(opts))
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 5. Disable metrics - metrics resources should get deleted.
+	mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics = nil
+	})
+	expectReconciled(t, ingR, "default", "test")
+	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
+	// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
 }

cmd/k8s-operator/metrics_resources.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"reflect"
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -115,15 +116,15 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o
 		return maybeCleanupServiceMonitor(ctx, cl, opts.proxyStsName, opts.tsNamespace)
 	}
 
-	logger.Info("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
-	svcMonitor, err := newServiceMonitor(metricsSvc)
+	logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
+	svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor)
 	if err != nil {
 		return fmt.Errorf("error creating ServiceMonitor: %w", err)
 	}
-	// We don't use createOrUpdate here because that does not work with unstructured types. We also do not update
-	// the ServiceMonitor because it is not expected that any of its fields would change. Currently this is good
-	// enough, but in future we might want to add logic to create-or-update unstructured types.
-	err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), svcMonitor.DeepCopy())
+
+	// We don't use createOrUpdate here because that does not work with unstructured types.
+	existing := svcMonitor.DeepCopy()
+	err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), existing)
 	if apierrors.IsNotFound(err) {
 		if err := cl.Create(ctx, svcMonitor); err != nil {
 			return fmt.Errorf("error creating ServiceMonitor: %w", err)
@@ -133,6 +134,13 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o
 	if err != nil {
 		return fmt.Errorf("error getting ServiceMonitor: %w", err)
 	}
+	// Currently, we only update labels on the ServiceMonitor as those are the only values that can change.
+	if !reflect.DeepEqual(existing.GetLabels(), svcMonitor.GetLabels()) {
+		existing.SetLabels(svcMonitor.GetLabels())
+		if err := cl.Update(ctx, existing); err != nil {
+			return fmt.Errorf("error updating ServiceMonitor: %w", err)
+		}
+	}
 	return nil
 }
 
@@ -165,9 +173,13 @@ func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName,
 // newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that
 // proxy that can be applied to the kube API server.
 // The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema.
-func newServiceMonitor(metricsSvc *corev1.Service) (*unstructured.Unstructured, error) {
+func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) {
 	sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace)
 	sm.ObjectMeta.Labels = metricsSvc.Labels
+	if spec != nil && len(spec.Labels) > 0 {
+		sm.ObjectMeta.Labels = mergeMapKeys(sm.ObjectMeta.Labels, spec.Labels.Parse())
+	}
+
 	sm.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(metricsSvc, corev1.SchemeGroupVersion.WithKind("Service"))}
 	sm.Spec = ServiceMonitorSpec{
 		Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels},
@@ -270,3 +282,14 @@ type metricsOpts struct {
 func isNamespacedProxyType(typ string) bool {
 	return typ == proxyTypeIngressResource || typ == proxyTypeIngressService
 }
+
+func mergeMapKeys(a, b map[string]string) map[string]string {
+	m := make(map[string]string, len(a)+len(b))
+	for key, val := range b {
+		m[key] = val
+	}
+	for key, val := range a {
+		m[key] = val
+	}
+	return m
+}

cmd/k8s-operator/nameserver_test.go

@@ -69,7 +69,7 @@ func TestNameserverReconciler(t *testing.T) {
 	wantsDeploy.Namespace = "tailscale"
 	labels := nameserverResourceLabels("test", "tailscale")
 	wantsDeploy.ObjectMeta.Labels = labels
-	expectEqual(t, fc, wantsDeploy, nil)
+	expectEqual(t, fc, wantsDeploy)
 
 	// Verify that DNSConfig advertizes the nameserver's Service IP address,
 	// has the ready status condition and tailscale finalizer.
@@ -88,7 +88,7 @@ func TestNameserverReconciler(t *testing.T) {
 		Message:            reasonNameserverCreated,
 		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
 	})
-	expectEqual(t, fc, dnsCfg, nil)
+	expectEqual(t, fc, dnsCfg)
 
 	// // Verify that nameserver image gets updated to match DNSConfig spec.
 	mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
@@ -96,7 +96,7 @@ func TestNameserverReconciler(t *testing.T) {
 	})
 	expectReconciled(t, nr, "", "test")
 	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
-	expectEqual(t, fc, wantsDeploy, nil)
+	expectEqual(t, fc, wantsDeploy)
 
 	// Verify that when another actor sets ConfigMap data, it does not get
 	// overwritten by nameserver reconciler.
@@ -114,7 +114,7 @@ func TestNameserverReconciler(t *testing.T) {
 		TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
 		Data:     map[string]string{"records.json": string(bs)},
 	}
-	expectEqual(t, fc, wantCm, nil)
+	expectEqual(t, fc, wantCm)
 
 	// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
 	// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
@@ -123,5 +123,5 @@ func TestNameserverReconciler(t *testing.T) {
 	})
 	expectReconciled(t, nr, "", "test")
 	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
-	expectEqual(t, fc, wantsDeploy, nil)
+	expectEqual(t, fc, wantsDeploy)
 }

cmd/k8s-operator/operator.go

@@ -9,6 +9,8 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"net/http"
 	"os"
 	"regexp"
 	"strconv"
@@ -18,7 +20,6 @@ import (
 	"github.com/go-logr/zapr"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"golang.org/x/oauth2/clientcredentials"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
@@ -39,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -88,6 +90,15 @@ func main() {
 	zlog := kzap.NewRaw(opts...).Sugar()
 	logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
 
+	if tsNamespace == "" {
+		const namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
+		b, err := os.ReadFile(namespaceFile)
+		if err != nil {
+			zlog.Fatalf("Could not get operator namespace from OPERATOR_NAMESPACE environment variable or default projected volume: %v", err)
+		}
+		tsNamespace = strings.TrimSpace(string(b))
+	}
+
 	// The operator can run either as a plain operator or it can
 	// additionally act as api-server proxy
 	// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
@@ -98,14 +109,14 @@ func main() {
 		hostinfo.SetApp(kubetypes.AppAPIServerProxy)
 	}
 
-	s, tsClient := initTSNet(zlog)
+	s, tsc := initTSNet(zlog)
 	defer s.Close()
 	restConfig := config.GetConfigOrDie()
 	maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
 	rOpts := reconcilerOpts{
 		log:                           zlog,
 		tsServer:                      s,
-		tsClient:                      tsClient,
+		tsClient:                      tsc,
 		tailscaleNamespace:            tsNamespace,
 		restConfig:                    restConfig,
 		proxyImage:                    image,
@@ -121,7 +132,7 @@ func main() {
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
 // CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate
 // with Tailscale.
-func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
+func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) {
 	var (
 		clientIDPath     = defaultEnv("CLIENT_ID_FILE", "")
 		clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "")
@@ -133,23 +144,10 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 	if clientIDPath == "" || clientSecretPath == "" {
 		startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set")
 	}
-	clientID, err := os.ReadFile(clientIDPath)
+	tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath)
 	if err != nil {
-		startlog.Fatalf("reading client ID %q: %v", clientIDPath, err)
+		startlog.Fatalf("error creating Tailscale client: %v", err)
 	}
-	clientSecret, err := os.ReadFile(clientSecretPath)
-	if err != nil {
-		startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err)
-	}
-	credentials := clientcredentials.Config{
-		ClientID:     string(clientID),
-		ClientSecret: string(clientSecret),
-		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
-	}
-	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-k8s-operator"
-	tsClient.HTTPClient = credentials.Client(context.Background())
-
 	s := &tsnet.Server{
 		Hostname: hostname,
 		Logf:     zlog.Named("tailscaled").Debugf,
@@ -202,7 +200,7 @@ waitOnline:
 					},
 				},
 			}
-			authkey, _, err := tsClient.CreateKey(ctx, caps)
+			authkey, _, err := tsc.CreateKey(ctx, caps)
 			if err != nil {
 				startlog.Fatalf("creating operator authkey: %v", err)
 			}
@@ -226,7 +224,7 @@ waitOnline:
 		}
 		time.Sleep(time.Second)
 	}
-	return s, tsClient
+	return s, tsc
 }
 
 // runReconcilers starts the controller-runtime manager and registers the
@@ -320,6 +318,7 @@ func runReconcilers(opts reconcilerOpts) {
 	err = builder.
 		ControllerManagedBy(mgr).
 		For(&networkingv1.Ingress{}).
+		Named("ingress-reconciler").
 		Watches(&appsv1.StatefulSet{}, ingressChildFilter).
 		Watches(&corev1.Secret{}, ingressChildFilter).
 		Watches(&corev1.Service{}, svcHandlerForIngress).
@@ -334,6 +333,38 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
 	}
+	lc, err := opts.tsServer.LocalClient()
+	if err != nil {
+		startlog.Fatalf("could not get local client: %v", err)
+	}
+	id, err := id(context.Background(), lc)
+	if err != nil {
+		startlog.Fatalf("error determining stable ID of the operator's Tailscale device: %v", err)
+	}
+	ingressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(ingressesFromIngressProxyGroup(mgr.GetClient(), opts.log))
+	err = builder.
+		ControllerManagedBy(mgr).
+		For(&networkingv1.Ingress{}).
+		Named("ingress-pg-reconciler").
+		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
+		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
+		Complete(&HAIngressReconciler{
+			recorder:    eventRecorder,
+			tsClient:    opts.tsClient,
+			tsnetServer: opts.tsServer,
+			defaultTags: strings.Split(opts.proxyTags, ","),
+			Client:      mgr.GetClient(),
+			logger:      opts.log.Named("ingress-pg-reconciler"),
+			lc:          lc,
+			operatorID:  id,
+			tsNamespace: opts.tailscaleNamespace,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create ingress-pg-reconciler: %v", err)
+	}
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyGroup, indexPGIngresses); err != nil {
+		startlog.Fatalf("failed setting up indexer for HA Ingresses: %v", err)
+	}
 
 	connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector"))
 	// If a ProxyClassChanges, enqueue all Connectors that have
@@ -341,6 +372,7 @@ func runReconcilers(opts reconcilerOpts) {
 	proxyClassFilterForConnector := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForConnector(mgr.GetClient(), startlog))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.Connector{}).
+		Named("connector-reconciler").
 		Watches(&appsv1.StatefulSet{}, connectorFilter).
 		Watches(&corev1.Secret{}, connectorFilter).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForConnector).
@@ -360,6 +392,7 @@ func runReconcilers(opts reconcilerOpts) {
 	nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver"))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.DNSConfig{}).
+		Named("nameserver-reconciler").
 		Watches(&appsv1.Deployment{}, nameserverFilter).
 		Watches(&corev1.ConfigMap{}, nameserverFilter).
 		Watches(&corev1.Service{}, nameserverFilter).
@@ -433,12 +466,31 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
 	}
 
+	podsForEps := handler.EnqueueRequestsFromMapFunc(podsFromEgressEps(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
+	podsER := handler.EnqueueRequestsFromMapFunc(egressPodsHandler)
+	err = builder.
+		ControllerManagedBy(mgr).
+		Named("egress-pods-readiness-reconciler").
+		Watches(&discoveryv1.EndpointSlice{}, podsForEps).
+		Watches(&corev1.Pod{}, podsER).
+		Complete(&egressPodsReconciler{
+			Client:      mgr.GetClient(),
+			tsNamespace: opts.tailscaleNamespace,
+			clock:       tstime.DefaultClock{},
+			logger:      opts.log.Named("egress-pods-readiness-reconciler"),
+			httpClient:  http.DefaultClient,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create egress Pods readiness reconciler: %v", err)
+	}
+
 	// ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that
 	// define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get
 	// reconciled if the CRD is applied at a later point.
 	serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
+		Named("proxyclass-reconciler").
 		Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter).
 		Complete(&ProxyClassReconciler{
 			Client:   mgr.GetClient(),
@@ -482,6 +534,7 @@ func runReconcilers(opts reconcilerOpts) {
 	recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.Recorder{}).
+		Named("recorder-reconciler").
 		Watches(&appsv1.StatefulSet{}, recorderFilter).
 		Watches(&corev1.ServiceAccount{}, recorderFilter).
 		Watches(&corev1.Secret{}, recorderFilter).
@@ -499,12 +552,14 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create Recorder reconciler: %v", err)
 	}
 
-	// Recorder reconciler.
+	// ProxyGroup reconciler.
 	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
 	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyGroup{}).
+		Named("proxygroup-reconciler").
 		Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
+		Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
 		Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
 		Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
 		Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
@@ -536,7 +591,7 @@ func runReconcilers(opts reconcilerOpts) {
 type reconcilerOpts struct {
 	log                *zap.SugaredLogger
 	tsServer           *tsnet.Server
-	tsClient           *tailscale.Client
+	tsClient           tsClient
 	tailscaleNamespace string       // namespace in which operator resources will be deployed
 	restConfig         *rest.Config // config for connecting to the kube API server
 	proxyImage         string       // <proxy-image-repo>:<proxy-image-tag>
@@ -661,12 +716,6 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 	}
 }
 
-type tsClient interface {
-	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
-	Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
-	DeleteDevice(ctx context.Context, nodeStableID string) error
-}
-
 func isManagedResource(o client.Object) bool {
 	ls := o.GetLabels()
 	return ls[LabelManaged] == "true"
@@ -759,7 +808,7 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass,
+// proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass,
 // returns a list of reconcile requests for all Connectors that have
 // .spec.proxyClass set.
 func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -802,6 +851,10 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl
 			if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
 				return nil
 			}
+			if hasProxyGroupAnnotation(&ing) {
+				// We don't want to reconcile backend Services for Ingresses for ProxyGroups.
+				continue
+			}
 			if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
 				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
 			}
@@ -884,6 +937,20 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
 	}
 }
 
+func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
+	if typ := o.GetLabels()[LabelParentType]; typ != proxyTypeProxyGroup {
+		return nil
+	}
+	return []reconcile.Request{
+		{
+			NamespacedName: types.NamespacedName{
+				Namespace: o.GetNamespace(),
+				Name:      o.GetName(),
+			},
+		},
+	}
+}
+
 // egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
@@ -976,7 +1043,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
 // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
 // user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
 		pg, ok := o.(*tsapi.ProxyGroup)
 		if !ok {
 			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
@@ -986,7 +1053,7 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 			return nil
 		}
 		svcList := &corev1.ServiceList{}
-		if err := cl.List(context.Background(), svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
+		if err := cl.List(ctx, svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
 			logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
 			return nil
 		}
@@ -1003,10 +1070,40 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
+// ingressesFromIngressProxyGroup is an event handler for ingress ProxyGroups. It returns reconcile requests for all
+// user-created Ingresses that should be exposed on this ProxyGroup.
+func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		pg, ok := o.(*tsapi.ProxyGroup)
+		if !ok {
+			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
+			return nil
+		}
+		if pg.Spec.Type != tsapi.ProxyGroupTypeIngress {
+			return nil
+		}
+		ingList := &networkingv1.IngressList{}
+		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pg.Name}); err != nil {
+			logger.Infof("error listing Ingresses: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, svc := range ingList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: svc.Namespace,
+					Name:      svc.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
 // should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
 func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
 		svc, ok := o.(*corev1.Service)
 		if !ok {
 			logger.Infof("[unexpected] Service handler triggered for an object that is not a Service")
@@ -1016,7 +1113,7 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns
 			return nil
 		}
 		epsList := &discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
+		if err := cl.List(ctx, epsList, client.InNamespace(ns),
 			client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
 			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
 			return nil
@@ -1034,6 +1131,43 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns
 	}
 }
 
+func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		eps, ok := o.(*discoveryv1.EndpointSlice)
+		if !ok {
+			logger.Infof("[unexpected] EndpointSlice handler triggered for an object that is not a EndpointSlice")
+			return nil
+		}
+		if eps.Labels[labelProxyGroup] == "" {
+			return nil
+		}
+		if eps.Labels[labelSvcType] != "egress" {
+			return nil
+		}
+		podLabels := map[string]string{
+			LabelManaged:    "true",
+			LabelParentType: "proxygroup",
+			LabelParentName: eps.Labels[labelProxyGroup],
+		}
+		podList := &corev1.PodList{}
+		if err := cl.List(ctx, podList, client.InNamespace(ns),
+			client.MatchingLabels(podLabels)); err != nil {
+			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, pod := range podList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: pod.Namespace,
+					Name:      pod.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // proxyClassesWithServiceMonitor returns an event handler that, given that the event is for the Prometheus
 // ServiceMonitor CRD, returns all ProxyClasses that define that a ServiceMonitor should be created.
 func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1085,3 +1219,64 @@ func indexEgressServices(o client.Object) []string {
 	}
 	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
 }
+
+// indexPGIngresses adds a local index to a cached Tailscale Ingresses meant to be exposed on a ProxyGroup. The index is
+// used a list filter.
+func indexPGIngresses(o client.Object) []string {
+	if !hasProxyGroupAnnotation(o) {
+		return nil
+	}
+	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
+}
+
+// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service
+// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation,
+// the associated Ingress gets reconciled.
+func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		ingList := networkingv1.IngressList{}
+		if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil {
+			logger.Debugf("error listing Ingresses: %v", err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, ing := range ingList.Items {
+			if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
+				continue
+			}
+			if !hasProxyGroupAnnotation(&ing) {
+				continue
+			}
+			if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
+				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
+			}
+			for _, rule := range ing.Spec.Rules {
+				if rule.HTTP == nil {
+					continue
+				}
+				for _, path := range rule.HTTP.Paths {
+					if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() {
+						reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
+					}
+				}
+			}
+		}
+		return reqs
+	}
+}
+
+func hasProxyGroupAnnotation(obj client.Object) bool {
+	ing := obj.(*networkingv1.Ingress)
+	return ing.Annotations[AnnotationProxyGroup] != ""
+}
+
+func id(ctx context.Context, lc *local.Client) (string, error) {
+	st, err := lc.StatusWithoutPeers(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error getting tailscale status: %w", err)
+	}
+	if st.Self == nil {
+		return "", fmt.Errorf("unexpected: device's status does not contain node's metadata")
+	}
+	return string(st.Self.ID), nil
+}

cmd/k8s-operator/operator_test.go

@@ -16,6 +16,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -105,7 +106,7 @@ func TestLoadBalancerClass(t *testing.T) {
 			}},
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Delete the misconfiguration so the proxy starts getting created on the
 	// next reconcile.
@@ -127,9 +128,9 @@ func TestLoadBalancerClass(t *testing.T) {
 		app:             kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	want.Annotations = nil
 	want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
@@ -142,7 +143,7 @@ func TestLoadBalancerClass(t *testing.T) {
 			Message:            "no Tailscale hostname known yet, waiting for proxy pod to finish auth",
 		}},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, then verify reconcile again and verify
@@ -168,7 +169,7 @@ func TestLoadBalancerClass(t *testing.T) {
 			},
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Turn the service back into a ClusterIP service, which should make the
 	// operator clean up.
@@ -205,7 +206,7 @@ func TestLoadBalancerClass(t *testing.T) {
 			Type:      corev1.ServiceTypeClusterIP,
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestTailnetTargetFQDNAnnotation(t *testing.T) {
@@ -265,9 +266,9 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		app:               kubetypes.AppEgressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 	want := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
@@ -287,10 +288,10 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, want)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 
 	// Change the tailscale-target-fqdn annotation which should update the
 	// StatefulSet
@@ -377,9 +378,9 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		app:             kubetypes.AppEgressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 	want := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
@@ -399,10 +400,10 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, want)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 
 	// Change the tailscale-target-ip annotation which should update the
 	// StatefulSet
@@ -500,7 +501,7 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
 		},
 	}
 
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
@@ -571,7 +572,7 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
 		},
 	}
 
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestAnnotations(t *testing.T) {
@@ -628,9 +629,9 @@ func TestAnnotations(t *testing.T) {
 		app:             kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 	want := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
@@ -649,7 +650,7 @@ func TestAnnotations(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Turn the service back into a ClusterIP service, which should make the
 	// operator clean up.
@@ -677,7 +678,7 @@ func TestAnnotations(t *testing.T) {
 			Type:      corev1.ServiceTypeClusterIP,
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestAnnotationIntoLB(t *testing.T) {
@@ -734,9 +735,9 @@ func TestAnnotationIntoLB(t *testing.T) {
 		app:             kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, since it would have normally happened at
@@ -768,7 +769,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Remove Tailscale's annotation, and at the same time convert the service
 	// into a tailscale LoadBalancer.
@@ -779,8 +780,8 @@ func TestAnnotationIntoLB(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	// None of the proxy machinery should have changed...
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 	// ... but the service should have a LoadBalancer status.
 
 	want = &corev1.Service{
@@ -809,7 +810,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestLBIntoAnnotation(t *testing.T) {
@@ -864,9 +865,9 @@ func TestLBIntoAnnotation(t *testing.T) {
 		app:             kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, then verify reconcile again and verify
@@ -906,7 +907,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Turn the service back into a ClusterIP service, but also add the
 	// tailscale annotation.
@@ -925,8 +926,8 @@ func TestLBIntoAnnotation(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 
 	want = &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
@@ -946,7 +947,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestCustomHostname(t *testing.T) {
@@ -1004,9 +1005,9 @@ func TestCustomHostname(t *testing.T) {
 		app:             kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, o))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 	want := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
@@ -1026,7 +1027,7 @@ func TestCustomHostname(t *testing.T) {
 			Conditions: proxyCreatedCondition(clock),
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 
 	// Turn the service back into a ClusterIP service, which should make the
 	// operator clean up.
@@ -1057,7 +1058,7 @@ func TestCustomHostname(t *testing.T) {
 			Type:      corev1.ServiceTypeClusterIP,
 		},
 	}
-	expectEqual(t, fc, want, nil)
+	expectEqual(t, fc, want)
 }
 
 func TestCustomPriorityClassName(t *testing.T) {
@@ -1117,7 +1118,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 		app:               kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 }
 
 func TestProxyClassForService(t *testing.T) {
@@ -1129,7 +1130,7 @@ func TestProxyClassForService(t *testing.T) {
 				AcceptRoutes: true,
 			},
 			StatefulSet: &tsapi.StatefulSet{
-				Labels:      map[string]string{"foo": "bar"},
+				Labels:      tsapi.Labels{"foo": "bar"},
 				Annotations: map[string]string{"bar.io/foo": "some-val"},
 				Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
@@ -1185,9 +1186,9 @@ func TestProxyClassForService(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 		app:             kubetypes.AppIngressProxy,
 	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. The Service gets updated with tailscale.com/proxy-class label
 	// pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not
@@ -1196,8 +1197,8 @@ func TestProxyClassForService(t *testing.T) {
 		mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata")
 	})
 	expectReconciled(t, sr, "default", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
 
 	// 3. ProxyClass is set to Ready, the Service gets reconciled by the
 	// services-reconciler and the customization from the ProxyClass is
@@ -1212,7 +1213,7 @@ func TestProxyClassForService(t *testing.T) {
 	})
 	opts.proxyClass = pc.Name
 	expectReconciled(t, sr, "default", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 	expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
 
 	// 4. tailscale.com/proxy-class label is removed from the Service, the
@@ -1223,7 +1224,7 @@ func TestProxyClassForService(t *testing.T) {
 	})
 	opts.proxyClass = ""
 	expectReconciled(t, sr, "default", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 }
 
 func TestDefaultLoadBalancer(t *testing.T) {
@@ -1269,7 +1270,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 
 	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
 
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	o := configOpts{
 		stsName:         shortName,
 		secretName:      fullName,
@@ -1279,8 +1280,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 		app:             kubetypes.AppIngressProxy,
 	}
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
-
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 }
 
 func TestProxyFirewallMode(t *testing.T) {
@@ -1336,73 +1336,9 @@ func TestProxyFirewallMode(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 		app:             kubetypes.AppIngressProxy,
 	}
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 }
 
-func TestTailscaledConfigfileHash(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:                zl.Sugar(),
-		clock:                 clock,
-		isDefaultLoadBalancer: true,
-	}
-
-	// Create a service that we should manage, and check that the initial round
-	// of objects looks right.
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "10.20.30.40",
-			Type:      corev1.ServiceTypeLoadBalancer,
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
-	o := configOpts{
-		stsName:         shortName,
-		secretName:      fullName,
-		namespace:       "default",
-		parentType:      "svc",
-		hostname:        "default-test",
-		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "acf3467364b0a3ba9b8ee0dd772cb7c2f0bf585e288fa99b7fe4566009ed6041",
-		app:             kubetypes.AppIngressProxy,
-	}
-	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
-
-	// 2. Hostname gets changed, configfile is updated and a new hash value
-	// is produced.
-	mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
-		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
-	})
-	o.hostname = "another-test"
-	o.confFileHash = "d4cc13f09f55f4f6775689004f9a466723325b84d2b590692796bfe22aeaa389"
-	expectReconciled(t, sr, "default", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
-}
 func Test_isMagicDNSName(t *testing.T) {
 	tests := []struct {
 		in   string
@@ -1679,9 +1615,9 @@ func Test_authKeyRemoval(t *testing.T) {
 		app:             kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Apply update to the Secret that imitates the proxy setting device_id.
 	s := expectedSecret(t, fc, opts)
@@ -1693,7 +1629,7 @@ func Test_authKeyRemoval(t *testing.T) {
 	expectReconciled(t, sr, "default", "test")
 	opts.shouldRemoveAuthKey = true
 	opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
 }
 
 func Test_externalNameService(t *testing.T) {
@@ -1753,9 +1689,9 @@ func Test_externalNameService(t *testing.T) {
 		app:              kubetypes.AppIngressProxy,
 	}
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
 
 	// 2. Change the ExternalName and verify that changes get propagated.
 	mustUpdate(t, sr, "default", "test", func(s *corev1.Service) {
@@ -1763,7 +1699,107 @@ func Test_externalNameService(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	opts.clusterTargetDNS = "bar.com"
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
+}
+
+func Test_metricsResourceCreation(t *testing.T) {
+	pc := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
+		Spec:       tsapi.ProxyClassSpec{},
+		Status: tsapi.ProxyClassStatus{
+			Conditions: []metav1.Condition{{
+				Status:             metav1.ConditionTrue,
+				Type:               string(tsapi.ProxyClassReady),
+				ObservedGeneration: 1,
+			}}},
+	}
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Labels:    map[string]string{LabelProxyClass: "metrics"},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	}
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pc, svc).
+		WithStatusSubresource(pc).
+		Build()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			operatorNamespace: "operator-ns",
+		},
+		logger: zl.Sugar(),
+		clock:  clock,
+	}
+	expectReconciled(t, sr, "default", "test")
+	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
+	opts := configOpts{
+		stsName:            shortName,
+		secretName:         fullName,
+		namespace:          "default",
+		parentType:         "svc",
+		tailscaleNamespace: "operator-ns",
+		hostname:           "default-test",
+		namespaced:         true,
+		proxyType:          proxyTypeIngressService,
+		app:                kubetypes.AppIngressProxy,
+		resourceVersion:    "1",
+	}
+
+	// 1. Enable metrics- expect metrics Service to be created
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec = tsapi.ProxyClassSpec{Metrics: &tsapi.Metrics{Enable: true}}
+	})
+	expectReconciled(t, sr, "default", "test")
+	opts.enableMetrics = true
+	expectEqual(t, fc, expectedMetricsService(opts))
+
+	// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
+	})
+	expectReconciled(t, sr, "default", "test")
+
+	// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
+	mustCreate(t, fc, crd)
+	expectReconciled(t, sr, "default", "test")
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 4. A change to ServiceMonitor config gets reflected in the ServiceMonitor resource
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar"}
+	})
+	expectReconciled(t, sr, "default", "test")
+	opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
+	opts.resourceVersion = "2"
+	expectEqual(t, fc, expectedMetricsService(opts))
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 5. Disable metrics- expect metrics Service to be deleted
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics = nil
+	})
+	expectReconciled(t, sr, "default", "test")
+	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(opts.stsName))
+	// ServiceMonitor gets garbage collected when Service gets deleted (it has OwnerReference of the Service
+	// object). We cannot test this using the fake client.
 }
 
 func toFQDN(t *testing.T, s string) dnsname.FQDN {