更新 cmd/derper/cert.go

将与域名验证相关的内容删除或注释

.github/workflows/checklocks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+      uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+      uses: github/codeql-action/init@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+      uses: github/codeql-action/autobuild@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+      uses: github/codeql-action/analyze@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,17 +23,17 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: go.mod
           cache: false
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
+        uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0
         with:
-          version: v1.60
+          version: v1.64
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -30,7 +30,7 @@ jobs:
           token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
           payload: |
             {
-              "channel": "C05PXRM304B",
+              "channel": "C08FGKZCQTW",
               "blocks": [
                 {
                   "type": "section",

.github/workflows/installer.yml

@@ -36,7 +36,6 @@ jobs:
           - "ubuntu:24.04"
           - "elementary/docker:stable"
           - "elementary/docker:unstable"
-          - "parrotsec/core:lts-amd64"
           - "parrotsec/core:latest"
           - "kalilinux/kali-rolling"
           - "kalilinux/kali-dev"
@@ -92,10 +91,7 @@ jobs:
         || contains(matrix.image, 'parrotsec')
         || contains(matrix.image, 'kalilinux')
     - name: checkout
-      # We cannot use v4, as it requires a newer glibc version than some of the
-      # tested images provide. See
-      # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/natlab-integrationtest.yml

@@ -0,0 +1,27 @@
+# Run some natlab integration tests.
+# See https://github.com/tailscale/tailscale/issues/13038
+name: "natlab-integrationtest"
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - "tstest/integration/nat/nat_test.go"
+jobs:
+  natlab-integrationtest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Install qemu
+        run: |
+          sudo rm /var/lib/man-db/auto-update
+          sudo apt-get -y update
+          sudo apt-get -y remove man-db
+          sudo apt-get install -y qemu-system-x86 qemu-utils
+      - name: Run natlab integration tests
+        run: |
+          ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat  --run-vm-tests

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -64,7 +64,6 @@ jobs:
       matrix:
         include:
           - goarch: amd64
-            coverflags: "-coverprofile=/tmp/coverage.out"
           - goarch: amd64
             buildflags: "-race"
             shard: '1/3'
@@ -78,9 +77,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -119,15 +118,10 @@ jobs:
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: test all
-      run: NOBASHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ${{matrix.coverflags}} ./... ${{matrix.buildflags}}
+      run: NOBASHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}}
       env:
         GOARCH: ${{ matrix.goarch }}
         TS_TEST_SHARD: ${{ matrix.shard }}
-    - name: Publish to coveralls.io
-      if: matrix.coverflags != '' # only publish results if we've tracked coverage
-      uses: shogo82148/actions-goveralls@v1
-      with:
-        path-to-profile: /tmp/coverage.out
     - name: bench all
       run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done)
       env:
@@ -145,21 +139,25 @@ jobs:
           echo "Build/test created untracked files in the repo (file names above)."
           exit 1
         fi
-
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
   windows:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Install Go
-      uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+      uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -182,6 +180,11 @@ jobs:
       # Somewhere in the layers (powershell?)
       # the equals signs cause great confusion.
       run: go test ./... -bench . -benchtime 1x -run "^$"
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   privileged:
     runs-on: ubuntu-22.04
@@ -190,7 +193,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +205,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +217,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +261,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -289,13 +292,18 @@ jobs:
         GOOS: ${{ matrix.goos }}
         GOARCH: ${{ matrix.goarch }}
         CGO_ENABLED: "0"
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   ios: # similar to cross above, but iOS can't build most of the repo. So, just
        #make it build a few smoke packages.
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -323,9 +331,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -348,6 +356,11 @@ jobs:
         GOARCH: ${{ matrix.goarch }}
         GOARM: ${{ matrix.goarm }}
         CGO_ENABLED: "0"
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   android:
     # similar to cross above, but android fails to build a few pieces of the
@@ -356,7 +369,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -371,9 +384,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Restore Cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -400,12 +413,17 @@ jobs:
       run: |
         ./tool/go run ./cmd/tsconnect --fast-compression build
         ./tool/go run ./cmd/tsconnect --fast-compression build-pkg
+    - name: Tidy cache
+      shell: bash
+      run: |
+        find $(go env GOCACHE) -type f -mmin +90 -delete
+        find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete
 
   tailscale_go: # Subset of tests that depend on our custom Go toolchain.
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -467,7 +485,7 @@ jobs:
       run: |
         echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+      uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
@@ -477,17 +495,17 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
-        find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check
+        find . -name 'depaware.txt' | xargs -n1 dirname | xargs ./tool/go run github.com/tailscale/depaware --check --internal
 
   go_generate:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -500,7 +518,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -512,7 +530,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -528,7 +546,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck

.github/workflows/update-flake.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Run update-flakes
         run: ./update-flake.sh
@@ -36,7 +36,7 @@ jobs:
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6
+        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Run go get
         run: |
@@ -35,7 +35,7 @@ jobs:
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6
+        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

.golangci.yml

@@ -26,16 +26,11 @@ issues:
 
 # Per-linter settings are contained in this top-level key
 linters-settings:
-  # Enable all rules by default; we don't use invisible unicode runes.
-  bidichk:
-
   gofmt:
     rewrite-rules:
       - pattern: 'interface{}'
         replacement: 'any'
 
-  goimports:
-
   govet:
     # Matches what we use in corp as of 2023-12-07
     enable:
@@ -78,8 +73,6 @@ linters-settings:
           # analyzer doesn't support type declarations
           #- github.com/tailscale/tailscale/types/logger.Logf
 
-  misspell:
-
   revive:
     enable-all-rules: false
     ignore-generated-header: true

ALPINE.txt

@@ -1 +1 @@
-3.18
+3.19

Dockerfile

@@ -27,7 +27,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.23-alpine AS build-env
+FROM golang:1.24-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
@@ -62,8 +62,10 @@ RUN GOARCH=$TARGETARCH go install -ldflags="\
       -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \
       -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot
 
-FROM alpine:3.18
+FROM alpine:3.19
 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables
+RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables
+RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables
 
 COPY --from=build-env /go/bin/* /usr/local/bin/
 # For compat with the previous run.sh, although ideally you should be

Dockerfile.base

@@ -1,5 +1,12 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
-FROM alpine:3.18
-RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables iputils
+FROM alpine:3.19
+RUN apk add --no-cache ca-certificates iptables iptables-legacy iproute2 ip6tables iputils
+# Alpine 3.19 replaces legacy iptables with nftables based implementation.  We
+# can't be certain that all hosts that run Tailscale containers currently
+# suppport nftables, so link back to legacy for backwards compatibility reasons.
+# TODO(irbekrm): add some way how to determine if we still run on nodes that
+# don't support nftables, so that we can eventually remove these symlinks.
+RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables
+RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables

Makefile

@@ -17,7 +17,7 @@ lint: ## Run golangci-lint
 updatedeps: ## Update depaware deps
 	# depaware (via x/tools/go/packages) shells back to "go", so make sure the "go"
 	# it finds in its $$PATH is the right one.
-	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update \
+	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --internal \
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \
@@ -27,7 +27,7 @@ updatedeps: ## Update depaware deps
 depaware: ## Run depaware checks
 	# depaware (via x/tools/go/packages) shells back to "go", so make sure the "go"
 	# it finds in its $$PATH is the right one.
-	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check \
+	PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --internal \
 		tailscale.com/cmd/tailscaled \
 		tailscale.com/cmd/tailscale \
 		tailscale.com/cmd/derper \

VERSION.txt

@@ -1 +1 @@
-1.79.0
+1.83.0

appc/appconnector.go

@@ -289,9 +289,11 @@ func (e *AppConnector) updateDomains(domains []string) {
 				toRemove = append(toRemove, netip.PrefixFrom(a, a.BitLen()))
 			}
 		}
-		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err)
-		}
+		e.queue.Add(func() {
+			if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+				e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err)
+			}
+		})
 	}
 
 	e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards)
@@ -310,11 +312,6 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 		return
 	}
 
-	if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
-		e.logf("failed to advertise routes: %v: %v", routes, err)
-		return
-	}
-
 	var toRemove []netip.Prefix
 
 	// If we're storing routes and know e.controlRoutes is a good
@@ -338,9 +335,14 @@ nextRoute:
 		}
 	}
 
-	if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-		e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
-	}
+	e.queue.Add(func() {
+		if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
+			e.logf("failed to advertise routes: %v: %v", routes, err)
+		}
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
+		}
+	})
 
 	e.controlRoutes = routes
 	if err := e.storeRoutesLocked(); err != nil {
@@ -374,13 +376,13 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr {
 // response is being returned over the PeerAPI. The response is parsed and
 // matched against the configured domains, if matched the routeAdvertiser is
 // advised to advertise the discovered route.
-func (e *AppConnector) ObserveDNSResponse(res []byte) {
+func (e *AppConnector) ObserveDNSResponse(res []byte) error {
 	var p dnsmessage.Parser
 	if _, err := p.Start(res); err != nil {
-		return
+		return err
 	}
 	if err := p.SkipAllQuestions(); err != nil {
-		return
+		return err
 	}
 
 	// cnameChain tracks a chain of CNAMEs for a given query in order to reverse
@@ -399,12 +401,12 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			break
 		}
 		if err != nil {
-			return
+			return err
 		}
 
 		if h.Class != dnsmessage.ClassINET {
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 		}
@@ -413,7 +415,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA:
 		default:
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 
@@ -427,7 +429,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		if h.Type == dnsmessage.TypeCNAME {
 			res, err := p.CNAMEResource()
 			if err != nil {
-				return
+				return err
 			}
 			cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".")
 			if len(cname) == 0 {
@@ -441,20 +443,20 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		case dnsmessage.TypeA:
 			r, err := p.AResource()
 			if err != nil {
-				return
+				return err
 			}
 			addr := netip.AddrFrom4(r.A)
 			mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
 		case dnsmessage.TypeAAAA:
 			r, err := p.AAAAResource()
 			if err != nil {
-				return
+				return err
 			}
 			addr := netip.AddrFrom16(r.AAAA)
 			mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
 		default:
 			if err := p.SkipAnswer(); err != nil {
-				return
+				return err
 			}
 			continue
 		}
@@ -485,6 +487,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 			e.scheduleAdvertisement(domain, toAdvertise...)
 		}
 	}
+	return nil
 }
 
 // starting from the given domain that resolved to an address, find it, or any

appc/appconnector_test.go

@@ -8,6 +8,7 @@ import (
 	"net/netip"
 	"reflect"
 	"slices"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -69,7 +70,9 @@ func TestUpdateRoutes(t *testing.T) {
 		a.updateDomains([]string{"*.example.com"})
 
 		// This route should be collapsed into the range
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 
 		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
@@ -77,11 +80,14 @@ func TestUpdateRoutes(t *testing.T) {
 		}
 
 		// This route should not be collapsed or removed
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 
 		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
 		a.updateRoutes(routes)
+		a.Wait(ctx)
 
 		slices.SortFunc(rc.Routes(), prefixCompare)
 		rc.SetRoutes(slices.Compact(rc.Routes()))
@@ -101,6 +107,7 @@ func TestUpdateRoutes(t *testing.T) {
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
+	ctx := context.Background()
 	for _, shouldStore := range []bool{false, true} {
 		rc := &appctest.RouteCollector{}
 		var a *AppConnector
@@ -113,6 +120,7 @@ func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
 		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
 		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
 		a.updateRoutes(routes)
+		a.Wait(ctx)
 
 		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
 			t.Fatalf("got %v, want %v", rc.Routes(), routes)
@@ -130,7 +138,9 @@ func TestDomainRoutes(t *testing.T) {
 			a = NewAppConnector(t.Logf, rc, nil, nil)
 		}
 		a.updateDomains([]string{"example.com"})
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(context.Background())
 
 		want := map[string][]netip.Addr{
@@ -155,7 +165,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		}
 
 		// a has no domains configured, so it should not advertise any routes
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
@@ -163,7 +175,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
 		a.updateDomains([]string{"example.com"})
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
@@ -172,7 +186,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		// a CNAME record chain should result in a route being added if the chain
 		// matches a routed domain.
 		a.updateDomains([]string{"www.example.com", "example.com"})
-		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com.")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -181,7 +197,9 @@ func TestObserveDNSResponse(t *testing.T) {
 
 		// a CNAME record chain should result in a route being added if the chain
 		// even if only found in the middle of the chain
-		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org.")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -190,14 +208,18 @@ func TestObserveDNSResponse(t *testing.T) {
 
 		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
 
 		// don't re-advertise routes that have already been advertised
-		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if !slices.Equal(rc.Routes(), wantRoutes) {
 			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -207,7 +229,9 @@ func TestObserveDNSResponse(t *testing.T) {
 		pfx := netip.MustParsePrefix("192.0.2.0/24")
 		a.updateRoutes([]netip.Prefix{pfx})
 		wantRoutes = append(wantRoutes, pfx)
-		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if !slices.Equal(rc.Routes(), wantRoutes) {
 			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -230,7 +254,9 @@ func TestWildcardDomains(t *testing.T) {
 		}
 
 		a.updateDomains([]string{"*.example.com"})
-		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		if err := a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")); err != nil {
+			t.Errorf("ObserveDNSResponse: %v", err)
+		}
 		a.Wait(ctx)
 		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
 			t.Errorf("routes: got %v; want %v", got, want)
@@ -438,10 +464,16 @@ func TestUpdateDomainRouteRemoval(t *testing.T) {
 		// adding domains doesn't immediately cause any routes to be advertised
 		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
 
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
-		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
+		for _, res := range [][]byte{
+			dnsResponse("a.example.com.", "1.2.3.1"),
+			dnsResponse("a.example.com.", "1.2.3.2"),
+			dnsResponse("b.example.com.", "1.2.3.3"),
+			dnsResponse("b.example.com.", "1.2.3.4"),
+		} {
+			if err := a.ObserveDNSResponse(res); err != nil {
+				t.Errorf("ObserveDNSResponse: %v", err)
+			}
+		}
 		a.Wait(ctx)
 		// observing dns responses causes routes to be advertised
 		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
@@ -487,10 +519,16 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) {
 		// adding domains doesn't immediately cause any routes to be advertised
 		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
 
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
-		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
-		a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
-		a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
+		for _, res := range [][]byte{
+			dnsResponse("a.example.com.", "1.2.3.1"),
+			dnsResponse("a.example.com.", "1.2.3.2"),
+			dnsResponse("1.b.example.com.", "1.2.3.3"),
+			dnsResponse("2.b.example.com.", "1.2.3.4"),
+		} {
+			if err := a.ObserveDNSResponse(res); err != nil {
+				t.Errorf("ObserveDNSResponse: %v", err)
+			}
+		}
 		a.Wait(ctx)
 		// observing dns responses causes routes to be advertised
 		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
@@ -602,3 +640,57 @@ func TestMetricBucketsAreSorted(t *testing.T) {
 		t.Errorf("metricStoreRoutesNBuckets must be in order")
 	}
 }
+
+// TestUpdateRoutesDeadlock is a regression test for a deadlock in
+// LocalBackend<->AppConnector interaction. When using real LocalBackend as the
+// routeAdvertiser, calls to Advertise/UnadvertiseRoutes can end up calling
+// back into AppConnector via authReconfig. If everything is called
+// synchronously, this results in a deadlock on AppConnector.mu.
+func TestUpdateRoutesDeadlock(t *testing.T) {
+	ctx := context.Background()
+	rc := &appctest.RouteCollector{}
+	a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+
+	advertiseCalled := new(atomic.Bool)
+	unadvertiseCalled := new(atomic.Bool)
+	rc.AdvertiseCallback = func() {
+		// Call something that requires a.mu to be held.
+		a.DomainRoutes()
+		advertiseCalled.Store(true)
+	}
+	rc.UnadvertiseCallback = func() {
+		// Call something that requires a.mu to be held.
+		a.DomainRoutes()
+		unadvertiseCalled.Store(true)
+	}
+
+	a.updateDomains([]string{"example.com"})
+	a.Wait(ctx)
+
+	// Trigger rc.AdveriseRoute.
+	a.updateRoutes(
+		[]netip.Prefix{
+			netip.MustParsePrefix("127.0.0.1/32"),
+			netip.MustParsePrefix("127.0.0.2/32"),
+		},
+	)
+	a.Wait(ctx)
+	// Trigger rc.UnadveriseRoute.
+	a.updateRoutes(
+		[]netip.Prefix{
+			netip.MustParsePrefix("127.0.0.1/32"),
+		},
+	)
+	a.Wait(ctx)
+
+	if !advertiseCalled.Load() {
+		t.Error("AdvertiseRoute was not called")
+	}
+	if !unadvertiseCalled.Load() {
+		t.Error("UnadvertiseRoute was not called")
+	}
+
+	if want := []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}; !slices.Equal(slices.Compact(rc.Routes()), want) {
+		t.Fatalf("got %v, want %v", rc.Routes(), want)
+	}
+}

appc/appctest/appctest.go

@@ -11,12 +11,22 @@ import (
 
 // RouteCollector is a test helper that collects the list of routes advertised
 type RouteCollector struct {
+	// AdvertiseCallback (optional) is called synchronously from
+	// AdvertiseRoute.
+	AdvertiseCallback func()
+	// UnadvertiseCallback (optional) is called synchronously from
+	// UnadvertiseRoute.
+	UnadvertiseCallback func()
+
 	routes        []netip.Prefix
 	removedRoutes []netip.Prefix
 }
 
 func (rc *RouteCollector) AdvertiseRoute(pfx ...netip.Prefix) error {
 	rc.routes = append(rc.routes, pfx...)
+	if rc.AdvertiseCallback != nil {
+		rc.AdvertiseCallback()
+	}
 	return nil
 }
 
@@ -30,6 +40,9 @@ func (rc *RouteCollector) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 			rc.removedRoutes = append(rc.removedRoutes, r)
 		}
 	}
+	if rc.UnadvertiseCallback != nil {
+		rc.UnadvertiseCallback()
+	}
 	return nil
 }
 

build_dist.sh

@@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do
 	--extra-small)
 		shift
 		ldflags="$ldflags -w -s"
-		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion"
+		tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan,ts_omit_capture"
 		;;
 	--box)
 		shift

build_docker.sh

@@ -16,7 +16,7 @@ eval "$(./build_dist.sh shellvars)"
 
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
-DEFAULT_BASE="tailscale/alpine-base:3.18"
+DEFAULT_BASE="tailscale/alpine-base:3.19"
 # Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
 # Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
 # annotations defined here will be overriden by release scripts that call this script.

client/local/local.go

@@ -3,13 +3,15 @@
 
 //go:build go1.22
 
-package tailscale
+// Package local contains a Go client for the Tailscale LocalAPI.
+package local
 
 import (
 	"bytes"
 	"cmp"
 	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -43,11 +45,11 @@ import (
 	"tailscale.com/util/syspolicy/setting"
 )
 
-// defaultLocalClient is the default LocalClient when using the legacy
+// defaultClient is the default Client when using the legacy
 // package-level functions.
-var defaultLocalClient LocalClient
+var defaultClient Client
 
-// LocalClient is a client to Tailscale's "LocalAPI", communicating with the
+// Client is a client to Tailscale's "LocalAPI", communicating with the
 // Tailscale daemon on the local machine. Its API is not necessarily stable and
 // subject to changes between releases. Some API calls have stricter
 // compatibility guarantees, once they've been widely adopted. See method docs
@@ -57,7 +59,7 @@ var defaultLocalClient LocalClient
 //
 // Any exported fields should be set before using methods on the type
 // and not changed thereafter.
-type LocalClient struct {
+type Client struct {
 	// Dial optionally specifies an alternate func that connects to the local
 	// machine's tailscaled or equivalent. If nil, a default is used.
 	Dial func(ctx context.Context, network, addr string) (net.Conn, error)
@@ -91,21 +93,21 @@ type LocalClient struct {
 	tsClientOnce sync.Once
 }
 
-func (lc *LocalClient) socket() string {
+func (lc *Client) socket() string {
 	if lc.Socket != "" {
 		return lc.Socket
 	}
 	return paths.DefaultTailscaledSocket()
 }
 
-func (lc *LocalClient) dialer() func(ctx context.Context, network, addr string) (net.Conn, error) {
+func (lc *Client) dialer() func(ctx context.Context, network, addr string) (net.Conn, error) {
 	if lc.Dial != nil {
 		return lc.Dial
 	}
 	return lc.defaultDialer
 }
 
-func (lc *LocalClient) defaultDialer(ctx context.Context, network, addr string) (net.Conn, error) {
+func (lc *Client) defaultDialer(ctx context.Context, network, addr string) (net.Conn, error) {
 	if addr != "local-tailscaled.sock:80" {
 		return nil, fmt.Errorf("unexpected URL address %q", addr)
 	}
@@ -131,7 +133,7 @@ func (lc *LocalClient) defaultDialer(ctx context.Context, network, addr string)
 // authenticating to the local Tailscale daemon vary by platform.
 //
 // DoLocalRequest may mutate the request to add Authorization headers.
-func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error) {
+func (lc *Client) DoLocalRequest(req *http.Request) (*http.Response, error) {
 	req.Header.Set("Tailscale-Cap", strconv.Itoa(int(tailcfg.CurrentCapabilityVersion)))
 	lc.tsClientOnce.Do(func() {
 		lc.tsClient = &http.Client{
@@ -148,7 +150,7 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 	return lc.tsClient.Do(req)
 }
 
-func (lc *LocalClient) doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
+func (lc *Client) doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
 	res, err := lc.DoLocalRequest(req)
 	if err == nil {
 		if server := res.Header.Get("Tailscale-Version"); server != "" && server != envknob.IPCVersion() && onVersionMismatch != nil {
@@ -237,12 +239,17 @@ func SetVersionMismatchHandler(f func(clientVer, serverVer string)) {
 	onVersionMismatch = f
 }
 
-func (lc *LocalClient) send(ctx context.Context, method, path string, wantStatus int, body io.Reader) ([]byte, error) {
-	slurp, _, err := lc.sendWithHeaders(ctx, method, path, wantStatus, body, nil)
+func (lc *Client) send(ctx context.Context, method, path string, wantStatus int, body io.Reader) ([]byte, error) {
+	var headers http.Header
+	if reason := apitype.RequestReasonKey.Value(ctx); reason != "" {
+		reasonBase64 := base64.StdEncoding.EncodeToString([]byte(reason))
+		headers = http.Header{apitype.RequestReasonHeader: {reasonBase64}}
+	}
+	slurp, _, err := lc.sendWithHeaders(ctx, method, path, wantStatus, body, headers)
 	return slurp, err
 }
 
-func (lc *LocalClient) sendWithHeaders(
+func (lc *Client) sendWithHeaders(
 	ctx context.Context,
 	method,
 	path string,
@@ -281,15 +288,15 @@ type httpStatusError struct {
 	HTTPStatus int
 }
 
-func (lc *LocalClient) get200(ctx context.Context, path string) ([]byte, error) {
+func (lc *Client) get200(ctx context.Context, path string) ([]byte, error) {
 	return lc.send(ctx, "GET", path, 200, nil)
 }
 
 // WhoIs returns the owner of the remoteAddr, which must be an IP or IP:port.
 //
-// Deprecated: use LocalClient.WhoIs.
+// Deprecated: use Client.WhoIs.
 func WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
-	return defaultLocalClient.WhoIs(ctx, remoteAddr)
+	return defaultClient.WhoIs(ctx, remoteAddr)
 }
 
 func decodeJSON[T any](b []byte) (ret T, err error) {
@@ -307,7 +314,7 @@ func decodeJSON[T any](b []byte) (ret T, err error) {
 // For connections proxied by tailscaled, this looks up the owner of the given
 // address as TCP first, falling back to UDP; if you want to only check a
 // specific address family, use WhoIsProto.
-func (lc *LocalClient) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
+func (lc *Client) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr))
 	if err != nil {
 		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
@@ -324,7 +331,7 @@ var ErrPeerNotFound = errors.New("peer not found")
 // WhoIsNodeKey returns the owner of the given wireguard public key.
 //
 // If not found, the error is ErrPeerNotFound.
-func (lc *LocalClient) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) {
+func (lc *Client) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(key.String()))
 	if err != nil {
 		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
@@ -339,7 +346,7 @@ func (lc *LocalClient) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*a
 // IP:port, for the given protocol (tcp or udp).
 //
 // If not found, the error is ErrPeerNotFound.
-func (lc *LocalClient) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) {
+func (lc *Client) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?proto="+url.QueryEscape(proto)+"&addr="+url.QueryEscape(remoteAddr))
 	if err != nil {
 		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
@@ -351,19 +358,19 @@ func (lc *LocalClient) WhoIsProto(ctx context.Context, proto, remoteAddr string)
 }
 
 // Goroutines returns a dump of the Tailscale daemon's current goroutines.
-func (lc *LocalClient) Goroutines(ctx context.Context) ([]byte, error) {
+func (lc *Client) Goroutines(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/goroutines")
 }
 
 // DaemonMetrics returns the Tailscale daemon's metrics in
 // the Prometheus text exposition format.
-func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
+func (lc *Client) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
 // UserMetrics returns the user metrics in
 // the Prometheus text exposition format.
-func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
+func (lc *Client) UserMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/usermetrics")
 }
 
@@ -372,7 +379,7 @@ func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
 // metric is created and initialized to delta.
 //
 // IncrementCounter does not support gauge metrics or negative delta values.
-func (lc *LocalClient) IncrementCounter(ctx context.Context, name string, delta int) error {
+func (lc *Client) IncrementCounter(ctx context.Context, name string, delta int) error {
 	type metricUpdate struct {
 		Name  string `json:"name"`
 		Type  string `json:"type"`
@@ -391,7 +398,7 @@ func (lc *LocalClient) IncrementCounter(ctx context.Context, name string, delta
 
 // TailDaemonLogs returns a stream the Tailscale daemon's logs as they arrive.
 // Close the context to stop the stream.
-func (lc *LocalClient) TailDaemonLogs(ctx context.Context) (io.Reader, error) {
+func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/logtap", nil)
 	if err != nil {
 		return nil, err
@@ -407,7 +414,7 @@ func (lc *LocalClient) TailDaemonLogs(ctx context.Context) (io.Reader, error) {
 }
 
 // Pprof returns a pprof profile of the Tailscale daemon.
-func (lc *LocalClient) Pprof(ctx context.Context, pprofType string, sec int) ([]byte, error) {
+func (lc *Client) Pprof(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string
 	if sec < 0 || sec > 300 {
 		return nil, errors.New("duration out of range")
@@ -440,7 +447,7 @@ type BugReportOpts struct {
 //
 // The opts type specifies options to pass to the Tailscale daemon when
 // generating this bug report.
-func (lc *LocalClient) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) {
+func (lc *Client) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) {
 	qparams := make(url.Values)
 	if opts.Note != "" {
 		qparams.Set("note", opts.Note)
@@ -485,13 +492,13 @@ func (lc *LocalClient) BugReportWithOpts(ctx context.Context, opts BugReportOpts
 //
 // This is the same as calling BugReportWithOpts and only specifying the Note
 // field.
-func (lc *LocalClient) BugReport(ctx context.Context, note string) (string, error) {
+func (lc *Client) BugReport(ctx context.Context, note string) (string, error) {
 	return lc.BugReportWithOpts(ctx, BugReportOpts{Note: note})
 }
 
 // DebugAction invokes a debug action, such as "rebind" or "restun".
 // These are development tools and subject to change or removal over time.
-func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
+func (lc *Client) DebugAction(ctx context.Context, action string) error {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil)
 	if err != nil {
 		return fmt.Errorf("error %w: %s", err, body)
@@ -502,7 +509,7 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
 // DebugActionBody invokes a debug action with a body parameter, such as
 // "debug-force-prefer-derp".
 // These are development tools and subject to change or removal over time.
-func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
+func (lc *Client) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, rbody)
 	if err != nil {
 		return fmt.Errorf("error %w: %s", err, body)
@@ -512,7 +519,7 @@ func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody
 
 // DebugResultJSON invokes a debug action and returns its result as something JSON-able.
 // These are development tools and subject to change or removal over time.
-func (lc *LocalClient) DebugResultJSON(ctx context.Context, action string) (any, error) {
+func (lc *Client) DebugResultJSON(ctx context.Context, action string) (any, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error %w: %s", err, body)
@@ -555,7 +562,7 @@ type DebugPortmapOpts struct {
 // process.
 //
 // opts can be nil; if so, default values will be used.
-func (lc *LocalClient) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) {
+func (lc *Client) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) {
 	vals := make(url.Values)
 	if opts == nil {
 		opts = &DebugPortmapOpts{}
@@ -590,7 +597,7 @@ func (lc *LocalClient) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts)
 
 // SetDevStoreKeyValue set a statestore key/value. It's only meant for development.
 // The schema (including when keys are re-read) is not a stable interface.
-func (lc *LocalClient) SetDevStoreKeyValue(ctx context.Context, key, value string) error {
+func (lc *Client) SetDevStoreKeyValue(ctx context.Context, key, value string) error {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/dev-set-state-store?"+(url.Values{
 		"key":   {key},
 		"value": {value},
@@ -604,7 +611,7 @@ func (lc *LocalClient) SetDevStoreKeyValue(ctx context.Context, key, value strin
 // SetComponentDebugLogging sets component's debug logging enabled for
 // the provided duration. If the duration is in the past, the debug logging
 // is disabled.
-func (lc *LocalClient) SetComponentDebugLogging(ctx context.Context, component string, d time.Duration) error {
+func (lc *Client) SetComponentDebugLogging(ctx context.Context, component string, d time.Duration) error {
 	body, err := lc.send(ctx, "POST",
 		fmt.Sprintf("/localapi/v0/component-debug-logging?component=%s&secs=%d",
 			url.QueryEscape(component), int64(d.Seconds())), 200, nil)
@@ -625,25 +632,25 @@ func (lc *LocalClient) SetComponentDebugLogging(ctx context.Context, component s
 
 // Status returns the Tailscale daemon's status.
 func Status(ctx context.Context) (*ipnstate.Status, error) {
-	return defaultLocalClient.Status(ctx)
+	return defaultClient.Status(ctx)
 }
 
 // Status returns the Tailscale daemon's status.
-func (lc *LocalClient) Status(ctx context.Context) (*ipnstate.Status, error) {
+func (lc *Client) Status(ctx context.Context) (*ipnstate.Status, error) {
 	return lc.status(ctx, "")
 }
 
 // StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
 func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
-	return defaultLocalClient.StatusWithoutPeers(ctx)
+	return defaultClient.StatusWithoutPeers(ctx)
 }
 
 // StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
-func (lc *LocalClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
+func (lc *Client) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 	return lc.status(ctx, "?peers=false")
 }
 
-func (lc *LocalClient) status(ctx context.Context, queryString string) (*ipnstate.Status, error) {
+func (lc *Client) status(ctx context.Context, queryString string) (*ipnstate.Status, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/status"+queryString)
 	if err != nil {
 		return nil, err
@@ -654,7 +661,7 @@ func (lc *LocalClient) status(ctx context.Context, queryString string) (*ipnstat
 // IDToken is a request to get an OIDC ID token for an audience.
 // The token can be presented to any resource provider which offers OIDC
 // Federation.
-func (lc *LocalClient) IDToken(ctx context.Context, aud string) (*tailcfg.TokenResponse, error) {
+func (lc *Client) IDToken(ctx context.Context, aud string) (*tailcfg.TokenResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/id-token?aud="+url.QueryEscape(aud))
 	if err != nil {
 		return nil, err
@@ -666,14 +673,14 @@ func (lc *LocalClient) IDToken(ctx context.Context, aud string) (*tailcfg.TokenR
 // received by the Tailscale daemon in its staging/cache directory but not yet
 // transferred by the user's CLI or GUI client and written to a user's home
 // directory somewhere.
-func (lc *LocalClient) WaitingFiles(ctx context.Context) ([]apitype.WaitingFile, error) {
+func (lc *Client) WaitingFiles(ctx context.Context) ([]apitype.WaitingFile, error) {
 	return lc.AwaitWaitingFiles(ctx, 0)
 }
 
 // AwaitWaitingFiles is like WaitingFiles but takes a duration to await for an answer.
 // If the duration is 0, it will return immediately. The duration is respected at second
 // granularity only. If no files are available, it returns (nil, nil).
-func (lc *LocalClient) AwaitWaitingFiles(ctx context.Context, d time.Duration) ([]apitype.WaitingFile, error) {
+func (lc *Client) AwaitWaitingFiles(ctx context.Context, d time.Duration) ([]apitype.WaitingFile, error) {
 	path := "/localapi/v0/files/?waitsec=" + fmt.Sprint(int(d.Seconds()))
 	body, err := lc.get200(ctx, path)
 	if err != nil {
@@ -682,12 +689,12 @@ func (lc *LocalClient) AwaitWaitingFiles(ctx context.Context, d time.Duration) (
 	return decodeJSON[[]apitype.WaitingFile](body)
 }
 
-func (lc *LocalClient) DeleteWaitingFile(ctx context.Context, baseName string) error {
+func (lc *Client) DeleteWaitingFile(ctx context.Context, baseName string) error {
 	_, err := lc.send(ctx, "DELETE", "/localapi/v0/files/"+url.PathEscape(baseName), http.StatusNoContent, nil)
 	return err
 }
 
-func (lc *LocalClient) GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, size int64, err error) {
+func (lc *Client) GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, size int64, err error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/files/"+url.PathEscape(baseName), nil)
 	if err != nil {
 		return nil, 0, err
@@ -708,7 +715,7 @@ func (lc *LocalClient) GetWaitingFile(ctx context.Context, baseName string) (rc
 	return res.Body, res.ContentLength, nil
 }
 
-func (lc *LocalClient) FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
+func (lc *Client) FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/file-targets")
 	if err != nil {
 		return nil, err
@@ -720,7 +727,7 @@ func (lc *LocalClient) FileTargets(ctx context.Context) ([]apitype.FileTarget, e
 //
 // A size of -1 means unknown.
 // The name parameter is the original filename, not escaped.
-func (lc *LocalClient) PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
+func (lc *Client) PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
 	req, err := http.NewRequestWithContext(ctx, "PUT", "http://"+apitype.LocalAPIHost+"/localapi/v0/file-put/"+string(target)+"/"+url.PathEscape(name), r)
 	if err != nil {
 		return err
@@ -743,7 +750,7 @@ func (lc *LocalClient) PushFile(ctx context.Context, target tailcfg.StableNodeID
 // CheckIPForwarding asks the local Tailscale daemon whether it looks like the
 // machine is properly configured to forward IP packets as a subnet router
 // or exit node.
-func (lc *LocalClient) CheckIPForwarding(ctx context.Context) error {
+func (lc *Client) CheckIPForwarding(ctx context.Context) error {
 	body, err := lc.get200(ctx, "/localapi/v0/check-ip-forwarding")
 	if err != nil {
 		return err
@@ -763,7 +770,7 @@ func (lc *LocalClient) CheckIPForwarding(ctx context.Context) error {
 // CheckUDPGROForwarding asks the local Tailscale daemon whether it looks like
 // the machine is optimally configured to forward UDP packets as a subnet router
 // or exit node.
-func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
+func (lc *Client) CheckUDPGROForwarding(ctx context.Context) error {
 	body, err := lc.get200(ctx, "/localapi/v0/check-udp-gro-forwarding")
 	if err != nil {
 		return err
@@ -784,7 +791,7 @@ func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
 // node. This can be done to improve performance of tailnet nodes acting as exit
 // nodes or subnet routers.
 // See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
-func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
+func (lc *Client) SetUDPGROForwarding(ctx context.Context) error {
 	body, err := lc.get200(ctx, "/localapi/v0/set-udp-gro-forwarding")
 	if err != nil {
 		return err
@@ -807,12 +814,12 @@ func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
 // work. Currently (2022-04-18) this only checks for SSH server compatibility.
 // Note that EditPrefs does the same validation as this, so call CheckPrefs before
 // EditPrefs is not necessary.
-func (lc *LocalClient) CheckPrefs(ctx context.Context, p *ipn.Prefs) error {
+func (lc *Client) CheckPrefs(ctx context.Context, p *ipn.Prefs) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/check-prefs", http.StatusOK, jsonBody(p))
 	return err
 }
 
-func (lc *LocalClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
+func (lc *Client) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/prefs")
 	if err != nil {
 		return nil, err
@@ -824,7 +831,12 @@ func (lc *LocalClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
 	return &p, nil
 }
 
-func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
+// EditPrefs updates the [ipn.Prefs] of the current Tailscale profile, applying the changes in mp.
+// It returns an error if the changes cannot be applied, such as due to the caller's access rights
+// or a policy restriction. An optional reason or justification for the request can be
+// provided as a context value using [apitype.RequestReasonKey]. If permitted by policy,
+// access may be granted, and the reason will be logged for auditing purposes.
+func (lc *Client) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
 	body, err := lc.send(ctx, "PATCH", "/localapi/v0/prefs", http.StatusOK, jsonBody(mp))
 	if err != nil {
 		return nil, err
@@ -833,7 +845,7 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 }
 
 // GetEffectivePolicy returns the effective policy for the specified scope.
-func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+func (lc *Client) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
 	scopeID, err := scope.MarshalText()
 	if err != nil {
 		return nil, err
@@ -847,7 +859,7 @@ func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.Pol
 
 // ReloadEffectivePolicy reloads the effective policy for the specified scope
 // by reading and merging policy settings from all applicable policy sources.
-func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+func (lc *Client) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
 	scopeID, err := scope.MarshalText()
 	if err != nil {
 		return nil, err
@@ -861,7 +873,7 @@ func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.
 
 // GetDNSOSConfig returns the system DNS configuration for the current device.
 // That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
-func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
+func (lc *Client) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
 	if err != nil {
 		return nil, err
@@ -876,7 +888,7 @@ func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig
 // QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
 // It returns the raw DNS response bytes and the resolvers that were used to answer the query
 // (often just one, but can be more if we raced multiple resolvers).
-func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
+func (lc *Client) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
 	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
 	if err != nil {
 		return nil, nil, err
@@ -889,20 +901,20 @@ func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType stri
 }
 
 // StartLoginInteractive starts an interactive login.
-func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
+func (lc *Client) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)
 	return err
 }
 
 // Start applies the configuration specified in opts, and starts the
 // state machine.
-func (lc *LocalClient) Start(ctx context.Context, opts ipn.Options) error {
+func (lc *Client) Start(ctx context.Context, opts ipn.Options) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/start", http.StatusNoContent, jsonBody(opts))
 	return err
 }
 
 // Logout logs out the current node.
-func (lc *LocalClient) Logout(ctx context.Context) error {
+func (lc *Client) Logout(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/logout", http.StatusNoContent, nil)
 	return err
 }
@@ -921,7 +933,7 @@ func (lc *LocalClient) Logout(ctx context.Context) error {
 // This is a low-level interface; it's expected that most Tailscale
 // users use a higher level interface to getting/using TLS
 // certificates.
-func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
+func (lc *Client) SetDNS(ctx context.Context, name, value string) error {
 	v := url.Values{}
 	v.Set("name", name)
 	v.Set("value", value)
@@ -935,7 +947,7 @@ func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
 // tailscaled), a FQDN, or an IP address.
 //
 // The ctx is only used for the duration of the call, not the lifetime of the net.Conn.
-func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
+func (lc *Client) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
 	return lc.UserDial(ctx, "tcp", host, port)
 }
 
@@ -946,7 +958,7 @@ func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (n
 //
 // The ctx is only used for the duration of the call, not the lifetime of the
 // net.Conn.
-func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
+func (lc *Client) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
 	connCh := make(chan net.Conn, 1)
 	trace := httptrace.ClientTrace{
 		GotConn: func(info httptrace.GotConnInfo) {
@@ -997,7 +1009,7 @@ func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port
 
 // CurrentDERPMap returns the current DERPMap that is being used by the local tailscaled.
 // It is intended to be used with netcheck to see availability of DERPs.
-func (lc *LocalClient) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
+func (lc *Client) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
 	var derpMap tailcfg.DERPMap
 	res, err := lc.send(ctx, "GET", "/localapi/v0/derpmap", 200, nil)
 	if err != nil {
@@ -1013,9 +1025,9 @@ func (lc *LocalClient) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, er
 //
 // It returns a cached certificate from disk if it's still valid.
 //
-// Deprecated: use LocalClient.CertPair.
+// Deprecated: use Client.CertPair.
 func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
-	return defaultLocalClient.CertPair(ctx, domain)
+	return defaultClient.CertPair(ctx, domain)
 }
 
 // CertPair returns a cert and private key for the provided DNS domain.
@@ -1023,7 +1035,7 @@ func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err e
 // It returns a cached certificate from disk if it's still valid.
 //
 // API maturity: this is considered a stable API.
-func (lc *LocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+func (lc *Client) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
 	return lc.CertPairWithValidity(ctx, domain, 0)
 }
 
@@ -1036,7 +1048,7 @@ func (lc *LocalClient) CertPair(ctx context.Context, domain string) (certPEM, ke
 // valid, but for less than minValidity, it will be synchronously renewed.
 //
 // API maturity: this is considered a stable API.
-func (lc *LocalClient) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) {
+func (lc *Client) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) {
 	res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil)
 	if err != nil {
 		return nil, nil, err
@@ -1062,9 +1074,9 @@ func (lc *LocalClient) CertPairWithValidity(ctx context.Context, domain string,
 // It's the right signature to use as the value of
 // tls.Config.GetCertificate.
 //
-// Deprecated: use LocalClient.GetCertificate.
+// Deprecated: use Client.GetCertificate.
 func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	return defaultLocalClient.GetCertificate(hi)
+	return defaultClient.GetCertificate(hi)
 }
 
 // GetCertificate fetches a TLS certificate for the TLS ClientHello in hi.
@@ -1075,7 +1087,7 @@ func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
 // tls.Config.GetCertificate.
 //
 // API maturity: this is considered a stable API.
-func (lc *LocalClient) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+func (lc *Client) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
 	if hi == nil || hi.ServerName == "" {
 		return nil, errors.New("no SNI ServerName")
 	}
@@ -1101,13 +1113,13 @@ func (lc *LocalClient) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate
 
 // ExpandSNIName expands bare label name into the most likely actual TLS cert name.
 //
-// Deprecated: use LocalClient.ExpandSNIName.
+// Deprecated: use Client.ExpandSNIName.
 func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
-	return defaultLocalClient.ExpandSNIName(ctx, name)
+	return defaultClient.ExpandSNIName(ctx, name)
 }
 
 // ExpandSNIName expands bare label name into the most likely actual TLS cert name.
-func (lc *LocalClient) ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
+func (lc *Client) ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
 	st, err := lc.StatusWithoutPeers(ctx)
 	if err != nil {
 		return "", false
@@ -1135,7 +1147,7 @@ type PingOpts struct {
 
 // Ping sends a ping of the provided type to the provided IP and waits
 // for its response. The opts type specifies additional options.
-func (lc *LocalClient) PingWithOpts(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType, opts PingOpts) (*ipnstate.PingResult, error) {
+func (lc *Client) PingWithOpts(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType, opts PingOpts) (*ipnstate.PingResult, error) {
 	v := url.Values{}
 	v.Set("ip", ip.String())
 	v.Set("size", strconv.Itoa(opts.Size))
@@ -1149,12 +1161,12 @@ func (lc *LocalClient) PingWithOpts(ctx context.Context, ip netip.Addr, pingtype
 
 // Ping sends a ping of the provided type to the provided IP and waits
 // for its response.
-func (lc *LocalClient) Ping(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType) (*ipnstate.PingResult, error) {
+func (lc *Client) Ping(ctx context.Context, ip netip.Addr, pingtype tailcfg.PingType) (*ipnstate.PingResult, error) {
 	return lc.PingWithOpts(ctx, ip, pingtype, PingOpts{})
 }
 
 // NetworkLockStatus fetches information about the tailnet key authority, if one is configured.
-func (lc *LocalClient) NetworkLockStatus(ctx context.Context) (*ipnstate.NetworkLockStatus, error) {
+func (lc *Client) NetworkLockStatus(ctx context.Context) (*ipnstate.NetworkLockStatus, error) {
 	body, err := lc.send(ctx, "GET", "/localapi/v0/tka/status", 200, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error: %w", err)
@@ -1165,7 +1177,7 @@ func (lc *LocalClient) NetworkLockStatus(ctx context.Context) (*ipnstate.Network
 // NetworkLockInit initializes the tailnet key authority.
 //
 // TODO(tom): Plumb through disablement secrets.
-func (lc *LocalClient) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.NetworkLockStatus, error) {
+func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.NetworkLockStatus, error) {
 	var b bytes.Buffer
 	type initRequest struct {
 		Keys               []tka.Key
@@ -1186,7 +1198,7 @@ func (lc *LocalClient) NetworkLockInit(ctx context.Context, keys []tka.Key, disa
 
 // NetworkLockWrapPreauthKey wraps a pre-auth key with information to
 // enable unattended bringup in the locked tailnet.
-func (lc *LocalClient) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) {
+func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) {
 	encodedPrivate, err := tkaKey.MarshalText()
 	if err != nil {
 		return "", err
@@ -1209,7 +1221,7 @@ func (lc *LocalClient) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey
 }
 
 // NetworkLockModify adds and/or removes key(s) to the tailnet key authority.
-func (lc *LocalClient) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error {
+func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error {
 	var b bytes.Buffer
 	type modifyRequest struct {
 		AddKeys    []tka.Key
@@ -1228,7 +1240,7 @@ func (lc *LocalClient) NetworkLockModify(ctx context.Context, addKeys, removeKey
 
 // NetworkLockSign signs the specified node-key and transmits that signature to the control plane.
 // rotationPublic, if specified, must be an ed25519 public key.
-func (lc *LocalClient) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error {
+func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error {
 	var b bytes.Buffer
 	type signRequest struct {
 		NodeKey        key.NodePublic
@@ -1246,7 +1258,7 @@ func (lc *LocalClient) NetworkLockSign(ctx context.Context, nodeKey key.NodePubl
 }
 
 // NetworkLockAffectedSigs returns all signatures signed by the specified keyID.
-func (lc *LocalClient) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) {
+func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/tka/affected-sigs", 200, bytes.NewReader(keyID))
 	if err != nil {
 		return nil, fmt.Errorf("error: %w", err)
@@ -1255,7 +1267,7 @@ func (lc *LocalClient) NetworkLockAffectedSigs(ctx context.Context, keyID tkatyp
 }
 
 // NetworkLockLog returns up to maxEntries number of changes to network-lock state.
-func (lc *LocalClient) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.NetworkLockUpdate, error) {
+func (lc *Client) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.NetworkLockUpdate, error) {
 	v := url.Values{}
 	v.Set("limit", fmt.Sprint(maxEntries))
 	body, err := lc.send(ctx, "GET", "/localapi/v0/tka/log?"+v.Encode(), 200, nil)
@@ -1266,7 +1278,7 @@ func (lc *LocalClient) NetworkLockLog(ctx context.Context, maxEntries int) ([]ip
 }
 
 // NetworkLockForceLocalDisable forcibly shuts down network lock on this node.
-func (lc *LocalClient) NetworkLockForceLocalDisable(ctx context.Context) error {
+func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error {
 	// This endpoint expects an empty JSON stanza as the payload.
 	var b bytes.Buffer
 	if err := json.NewEncoder(&b).Encode(struct{}{}); err != nil {
@@ -1281,7 +1293,7 @@ func (lc *LocalClient) NetworkLockForceLocalDisable(ctx context.Context) error {
 
 // NetworkLockVerifySigningDeeplink verifies the network lock deeplink contained
 // in url and returns information extracted from it.
-func (lc *LocalClient) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) {
+func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) {
 	vr := struct {
 		URL string
 	}{url}
@@ -1295,7 +1307,7 @@ func (lc *LocalClient) NetworkLockVerifySigningDeeplink(ctx context.Context, url
 }
 
 // NetworkLockGenRecoveryAUM generates an AUM for recovering from a tailnet-lock key compromise.
-func (lc *LocalClient) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) {
+func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) {
 	vr := struct {
 		Keys     []tkatype.KeyID
 		ForkFrom string
@@ -1310,7 +1322,7 @@ func (lc *LocalClient) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys
 }
 
 // NetworkLockCosignRecoveryAUM co-signs a recovery AUM using the node's tailnet lock key.
-func (lc *LocalClient) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) {
+func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) {
 	r := bytes.NewReader(aum.Serialize())
 	body, err := lc.send(ctx, "POST", "/localapi/v0/tka/cosign-recovery-aum", 200, r)
 	if err != nil {
@@ -1321,7 +1333,7 @@ func (lc *LocalClient) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka
 }
 
 // NetworkLockSubmitRecoveryAUM submits a recovery AUM to the control plane.
-func (lc *LocalClient) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error {
+func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error {
 	r := bytes.NewReader(aum.Serialize())
 	_, err := lc.send(ctx, "POST", "/localapi/v0/tka/submit-recovery-aum", 200, r)
 	if err != nil {
@@ -1332,7 +1344,7 @@ func (lc *LocalClient) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka
 
 // SetServeConfig sets or replaces the serving settings.
 // If config is nil, settings are cleared and serving is disabled.
-func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConfig) error {
+func (lc *Client) SetServeConfig(ctx context.Context, config *ipn.ServeConfig) error {
 	h := make(http.Header)
 	if config != nil {
 		h.Set("If-Match", config.ETag)
@@ -1347,7 +1359,7 @@ func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConf
 // DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be
 // run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over
 // to another replica whilst there is still some grace period for the existing connections to terminate.
-func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
+func (lc *Client) DisconnectControl(ctx context.Context) error {
 	_, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil)
 	if err != nil {
 		return fmt.Errorf("error disconnecting control: %w", err)
@@ -1356,7 +1368,7 @@ func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
 }
 
 // NetworkLockDisable shuts down network-lock across the tailnet.
-func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) error {
+func (lc *Client) NetworkLockDisable(ctx context.Context, secret []byte) error {
 	if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil {
 		return fmt.Errorf("error: %w", err)
 	}
@@ -1366,7 +1378,7 @@ func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) er
 // GetServeConfig return the current serve config.
 //
 // If the serve config is empty, it returns (nil, nil).
-func (lc *LocalClient) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) {
+func (lc *Client) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) {
 	body, h, err := lc.sendWithHeaders(ctx, "GET", "/localapi/v0/serve-config", 200, nil, nil)
 	if err != nil {
 		return nil, fmt.Errorf("getting serve config: %w", err)
@@ -1441,7 +1453,7 @@ func (r jsonReader) Read(p []byte) (n int, err error) {
 }
 
 // ProfileStatus returns the current profile and the list of all profiles.
-func (lc *LocalClient) ProfileStatus(ctx context.Context) (current ipn.LoginProfile, all []ipn.LoginProfile, err error) {
+func (lc *Client) ProfileStatus(ctx context.Context) (current ipn.LoginProfile, all []ipn.LoginProfile, err error) {
 	body, err := lc.send(ctx, "GET", "/localapi/v0/profiles/current", 200, nil)
 	if err != nil {
 		return
@@ -1459,7 +1471,7 @@ func (lc *LocalClient) ProfileStatus(ctx context.Context) (current ipn.LoginProf
 }
 
 // ReloadConfig reloads the config file, if possible.
-func (lc *LocalClient) ReloadConfig(ctx context.Context) (ok bool, err error) {
+func (lc *Client) ReloadConfig(ctx context.Context) (ok bool, err error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/reload-config", 200, nil)
 	if err != nil {
 		return
@@ -1477,13 +1489,13 @@ func (lc *LocalClient) ReloadConfig(ctx context.Context) (ok bool, err error) {
 // SwitchToEmptyProfile creates and switches to a new unnamed profile. The new
 // profile is not assigned an ID until it is persisted after a successful login.
 // In order to login to the new profile, the user must call LoginInteractive.
-func (lc *LocalClient) SwitchToEmptyProfile(ctx context.Context) error {
+func (lc *Client) SwitchToEmptyProfile(ctx context.Context) error {
 	_, err := lc.send(ctx, "PUT", "/localapi/v0/profiles/", http.StatusCreated, nil)
 	return err
 }
 
 // SwitchProfile switches to the given profile.
-func (lc *LocalClient) SwitchProfile(ctx context.Context, profile ipn.ProfileID) error {
+func (lc *Client) SwitchProfile(ctx context.Context, profile ipn.ProfileID) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/profiles/"+url.PathEscape(string(profile)), 204, nil)
 	return err
 }
@@ -1491,7 +1503,7 @@ func (lc *LocalClient) SwitchProfile(ctx context.Context, profile ipn.ProfileID)
 // DeleteProfile removes the profile with the given ID.
 // If the profile is the current profile, an empty profile
 // will be selected as if SwitchToEmptyProfile was called.
-func (lc *LocalClient) DeleteProfile(ctx context.Context, profile ipn.ProfileID) error {
+func (lc *Client) DeleteProfile(ctx context.Context, profile ipn.ProfileID) error {
 	_, err := lc.send(ctx, "DELETE", "/localapi/v0/profiles"+url.PathEscape(string(profile)), http.StatusNoContent, nil)
 	return err
 }
@@ -1508,7 +1520,7 @@ func (lc *LocalClient) DeleteProfile(ctx context.Context, profile ipn.ProfileID)
 // to block until the feature has been enabled.
 //
 // 2023-08-09: Valid feature values are "serve" and "funnel".
-func (lc *LocalClient) QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) {
+func (lc *Client) QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) {
 	v := url.Values{"feature": {feature}}
 	body, err := lc.send(ctx, "POST", "/localapi/v0/query-feature?"+v.Encode(), 200, nil)
 	if err != nil {
@@ -1517,7 +1529,7 @@ func (lc *LocalClient) QueryFeature(ctx context.Context, feature string) (*tailc
 	return decodeJSON[*tailcfg.QueryFeatureResponse](body)
 }
 
-func (lc *LocalClient) DebugDERPRegion(ctx context.Context, regionIDOrCode string) (*ipnstate.DebugDERPRegionReport, error) {
+func (lc *Client) DebugDERPRegion(ctx context.Context, regionIDOrCode string) (*ipnstate.DebugDERPRegionReport, error) {
 	v := url.Values{"region": {regionIDOrCode}}
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug-derp-region?"+v.Encode(), 200, nil)
 	if err != nil {
@@ -1527,7 +1539,7 @@ func (lc *LocalClient) DebugDERPRegion(ctx context.Context, regionIDOrCode strin
 }
 
 // DebugPacketFilterRules returns the packet filter rules for the current device.
-func (lc *LocalClient) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterRule, error) {
+func (lc *Client) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterRule, error) {
 	body, err := lc.send(ctx, "POST", "/localapi/v0/debug-packet-filter-rules", 200, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error %w: %s", err, body)
@@ -1538,7 +1550,7 @@ func (lc *LocalClient) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.Fi
 // DebugSetExpireIn marks the current node key to expire in d.
 //
 // This is meant primarily for debug and testing.
-func (lc *LocalClient) DebugSetExpireIn(ctx context.Context, d time.Duration) error {
+func (lc *Client) DebugSetExpireIn(ctx context.Context, d time.Duration) error {
 	v := url.Values{"expiry": {fmt.Sprint(time.Now().Add(d).Unix())}}
 	_, err := lc.send(ctx, "POST", "/localapi/v0/set-expiry-sooner?"+v.Encode(), 200, nil)
 	return err
@@ -1548,7 +1560,7 @@ func (lc *LocalClient) DebugSetExpireIn(ctx context.Context, d time.Duration) er
 //
 // The provided context does not determine the lifetime of the
 // returned io.ReadCloser.
-func (lc *LocalClient) StreamDebugCapture(ctx context.Context) (io.ReadCloser, error) {
+func (lc *Client) StreamDebugCapture(ctx context.Context) (io.ReadCloser, error) {
 	req, err := http.NewRequestWithContext(ctx, "POST", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-capture", nil)
 	if err != nil {
 		return nil, err
@@ -1574,7 +1586,7 @@ func (lc *LocalClient) StreamDebugCapture(ctx context.Context) (io.ReadCloser, e
 // resources.
 //
 // A default set of ipn.Notify messages are returned but the set can be modified by mask.
-func (lc *LocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*IPNBusWatcher, error) {
+func (lc *Client) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*IPNBusWatcher, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET",
 		"http://"+apitype.LocalAPIHost+"/localapi/v0/watch-ipn-bus?mask="+fmt.Sprint(mask),
 		nil)
@@ -1600,7 +1612,7 @@ func (lc *LocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt)
 // CheckUpdate returns a tailcfg.ClientVersion indicating whether or not an update is available
 // to be installed via the LocalAPI. In case the LocalAPI can't install updates, it returns a
 // ClientVersion that says that we are up to date.
-func (lc *LocalClient) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion, error) {
+func (lc *Client) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/update/check")
 	if err != nil {
 		return nil, err
@@ -1616,7 +1628,7 @@ func (lc *LocalClient) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion,
 // To turn it on, there must have been a previously used exit node.
 // The most previously used one is reused.
 // This is a convenience method for GUIs. To select an actual one, update the prefs.
-func (lc *LocalClient) SetUseExitNode(ctx context.Context, on bool) error {
+func (lc *Client) SetUseExitNode(ctx context.Context, on bool) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/set-use-exit-node-enabled?enabled="+strconv.FormatBool(on), http.StatusOK, nil)
 	return err
 }
@@ -1624,7 +1636,7 @@ func (lc *LocalClient) SetUseExitNode(ctx context.Context, on bool) error {
 // DriveSetServerAddr instructs Taildrive to use the server at addr to access
 // the filesystem. This is used on platforms like Windows and MacOS to let
 // Taildrive know to use the file server running in the GUI app.
-func (lc *LocalClient) DriveSetServerAddr(ctx context.Context, addr string) error {
+func (lc *Client) DriveSetServerAddr(ctx context.Context, addr string) error {
 	_, err := lc.send(ctx, "PUT", "/localapi/v0/drive/fileserver-address", http.StatusCreated, strings.NewReader(addr))
 	return err
 }
@@ -1632,14 +1644,14 @@ func (lc *LocalClient) DriveSetServerAddr(ctx context.Context, addr string) erro
 // DriveShareSet adds or updates the given share in the list of shares that
 // Taildrive will serve to remote nodes. If a share with the same name already
 // exists, the existing share is replaced/updated.
-func (lc *LocalClient) DriveShareSet(ctx context.Context, share *drive.Share) error {
+func (lc *Client) DriveShareSet(ctx context.Context, share *drive.Share) error {
 	_, err := lc.send(ctx, "PUT", "/localapi/v0/drive/shares", http.StatusCreated, jsonBody(share))
 	return err
 }
 
 // DriveShareRemove removes the share with the given name from the list of
 // shares that Taildrive will serve to remote nodes.
-func (lc *LocalClient) DriveShareRemove(ctx context.Context, name string) error {
+func (lc *Client) DriveShareRemove(ctx context.Context, name string) error {
 	_, err := lc.send(
 		ctx,
 		"DELETE",
@@ -1650,7 +1662,7 @@ func (lc *LocalClient) DriveShareRemove(ctx context.Context, name string) error
 }
 
 // DriveShareRename renames the share from old to new name.
-func (lc *LocalClient) DriveShareRename(ctx context.Context, oldName, newName string) error {
+func (lc *Client) DriveShareRename(ctx context.Context, oldName, newName string) error {
 	_, err := lc.send(
 		ctx,
 		"POST",
@@ -1662,7 +1674,7 @@ func (lc *LocalClient) DriveShareRename(ctx context.Context, oldName, newName st
 
 // DriveShareList returns the list of shares that drive is currently serving
 // to remote nodes.
-func (lc *LocalClient) DriveShareList(ctx context.Context) ([]*drive.Share, error) {
+func (lc *Client) DriveShareList(ctx context.Context) ([]*drive.Share, error) {
 	result, err := lc.get200(ctx, "/localapi/v0/drive/shares")
 	if err != nil {
 		return nil, err
@@ -1673,7 +1685,7 @@ func (lc *LocalClient) DriveShareList(ctx context.Context) ([]*drive.Share, erro
 }
 
 // IPNBusWatcher is an active subscription (watch) of the local tailscaled IPN bus.
-// It's returned by LocalClient.WatchIPNBus.
+// It's returned by Client.WatchIPNBus.
 //
 // It must be closed when done.
 type IPNBusWatcher struct {
@@ -1697,7 +1709,7 @@ func (w *IPNBusWatcher) Close() error {
 }
 
 // Next returns the next ipn.Notify from the stream.
-// If the context from LocalClient.WatchIPNBus is done, that error is returned.
+// If the context from Client.WatchIPNBus is done, that error is returned.
 func (w *IPNBusWatcher) Next() (ipn.Notify, error) {
 	var n ipn.Notify
 	if err := w.dec.Decode(&n); err != nil {
@@ -1710,7 +1722,7 @@ func (w *IPNBusWatcher) Next() (ipn.Notify, error) {
 }
 
 // SuggestExitNode requests an exit node suggestion and returns the exit node's details.
-func (lc *LocalClient) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) {
+func (lc *Client) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/suggest-exit-node")
 	if err != nil {
 		return apitype.ExitNodeSuggestionResponse{}, err

client/local/local_test.go

@@ -3,7 +3,7 @@
 
 //go:build go1.19
 
-package tailscale
+package local
 
 import (
 	"context"
@@ -41,7 +41,7 @@ func TestWhoIsPeerNotFound(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	lc := &LocalClient{
+	lc := &Client{
 		Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
 			var std net.Dialer
 			return std.DialContext(ctx, network, ts.Listener.Addr().(*net.TCPAddr).String())

client/systray/systray.go

@@ -26,7 +26,7 @@ import (
 	"github.com/atotto/clipboard"
 	dbus "github.com/godbus/dbus/v5"
 	"github.com/toqueteos/webbrowser"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -67,11 +67,16 @@ func (menu *Menu) Run() {
 type Menu struct {
 	mu sync.Mutex // protects the entire Menu
 
-	lc          tailscale.LocalClient
+	lc          local.Client
 	status      *ipnstate.Status
 	curProfile  ipn.LoginProfile
 	allProfiles []ipn.LoginProfile
 
+	// readonly is whether the systray app is running in read-only mode.
+	// This is set if LocalAPI returns a permission error,
+	// typically because the user needs to run `tailscale set --operator=$USER`.
+	readonly bool
+
 	bgCtx    context.Context // ctx for background tasks not involving menu item clicks
 	bgCancel context.CancelFunc
 
@@ -153,6 +158,8 @@ func (menu *Menu) updateState() {
 	defer menu.mu.Unlock()
 	menu.init()
 
+	menu.readonly = false
+
 	var err error
 	menu.status, err = menu.lc.Status(menu.bgCtx)
 	if err != nil {
@@ -160,6 +167,9 @@ func (menu *Menu) updateState() {
 	}
 	menu.curProfile, menu.allProfiles, err = menu.lc.ProfileStatus(menu.bgCtx)
 	if err != nil {
+		if local.IsAccessDeniedError(err) {
+			menu.readonly = true
+		}
 		log.Print(err)
 	}
 }
@@ -182,6 +192,15 @@ func (menu *Menu) rebuild() {
 
 	systray.ResetMenu()
 
+	if menu.readonly {
+		const readonlyMsg = "No permission to manage Tailscale.\nSee tailscale.com/s/cli-operator"
+		m := systray.AddMenuItem(readonlyMsg, "")
+		onClick(ctx, m, func(_ context.Context) {
+			webbrowser.Open("https://tailscale.com/s/cli-operator")
+		})
+		systray.AddSeparator()
+	}
+
 	menu.connect = systray.AddMenuItem("Connect", "")
 	menu.disconnect = systray.AddMenuItem("Disconnect", "")
 	menu.disconnect.Hide()
@@ -222,28 +241,35 @@ func (menu *Menu) rebuild() {
 		setAppIcon(disconnected)
 	}
 
+	if menu.readonly {
+		menu.connect.Disable()
+		menu.disconnect.Disable()
+	}
+
 	account := "Account"
 	if pt := profileTitle(menu.curProfile); pt != "" {
 		account = pt
 	}
-	accounts := systray.AddMenuItem(account, "")
-	setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL)
-	time.Sleep(newMenuDelay)
-	for _, profile := range menu.allProfiles {
-		title := profileTitle(profile)
-		var item *systray.MenuItem
-		if profile.ID == menu.curProfile.ID {
-			item = accounts.AddSubMenuItemCheckbox(title, "", true)
-		} else {
-			item = accounts.AddSubMenuItem(title, "")
-		}
-		setRemoteIcon(item, profile.UserProfile.ProfilePicURL)
-		onClick(ctx, item, func(ctx context.Context) {
-			select {
-			case <-ctx.Done():
-			case menu.accountsCh <- profile.ID:
+	if !menu.readonly {
+		accounts := systray.AddMenuItem(account, "")
+		setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL)
+		time.Sleep(newMenuDelay)
+		for _, profile := range menu.allProfiles {
+			title := profileTitle(profile)
+			var item *systray.MenuItem
+			if profile.ID == menu.curProfile.ID {
+				item = accounts.AddSubMenuItemCheckbox(title, "", true)
+			} else {
+				item = accounts.AddSubMenuItem(title, "")
 			}
-		})
+			setRemoteIcon(item, profile.UserProfile.ProfilePicURL)
+			onClick(ctx, item, func(ctx context.Context) {
+				select {
+				case <-ctx.Done():
+				case menu.accountsCh <- profile.ID:
+				}
+			})
+		}
 	}
 
 	if menu.status != nil && menu.status.Self != nil && len(menu.status.Self.TailscaleIPs) > 0 {
@@ -255,7 +281,9 @@ func (menu *Menu) rebuild() {
 	}
 	systray.AddSeparator()
 
-	menu.rebuildExitNodeMenu(ctx)
+	if !menu.readonly {
+		menu.rebuildExitNodeMenu(ctx)
+	}
 
 	if menu.status != nil {
 		menu.more = systray.AddMenuItem("More settings", "")

client/tailscale/acl.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/netip"
+	"net/url"
 )
 
 // ACLRow defines a rule that grants access by a set of users or groups to a set
@@ -83,7 +84,7 @@ func (c *Client) ACL(ctx context.Context) (acl *ACL, err error) {
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl")
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -97,7 +98,7 @@ func (c *Client) ACL(ctx context.Context) (acl *ACL, err error) {
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	// Otherwise, try to decode the response.
@@ -126,7 +127,7 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl?details=1", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl", url.Values{"details": {"1"}})
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -138,7 +139,7 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	data := struct {
@@ -146,7 +147,7 @@ func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 		Warnings []string `json:"warnings"`
 	}{}
 	if err := json.Unmarshal(b, &data); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("json.Unmarshal %q: %w", b, err)
 	}
 
 	acl = &ACLHuJSON{
@@ -184,7 +185,7 @@ func (e ACLTestError) Error() string {
 }
 
 func (c *Client) aclPOSTRequest(ctx context.Context, body []byte, avoidCollisions bool, etag, acceptHeader string) ([]byte, string, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewBuffer(body))
 	if err != nil {
 		return nil, "", err
@@ -328,7 +329,7 @@ type ACLPreview struct {
 }
 
 func (c *Client) previewACLPostRequest(ctx context.Context, body []byte, previewType string, previewFor string) (res *ACLPreviewResponse, err error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl/preview", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl", "preview")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewBuffer(body))
 	if err != nil {
 		return nil, err
@@ -350,7 +351,7 @@ func (c *Client) previewACLPostRequest(ctx context.Context, body []byte, preview
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 	if err = json.Unmarshal(b, &res); err != nil {
 		return nil, err
@@ -488,7 +489,7 @@ func (c *Client) ValidateACLJSON(ctx context.Context, source, dest string) (test
 		return nil, err
 	}
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/acl/validate", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("acl", "validate")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewBuffer(postData))
 	if err != nil {
 		return nil, err

client/tailscale/apitype/apitype.go

@@ -7,11 +7,29 @@ package apitype
 import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/dnstype"
+	"tailscale.com/util/ctxkey"
 )
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
 
+// RequestReasonHeader is the header used to pass justification for a LocalAPI request,
+// such as when a user wants to perform an action they don't have permission for,
+// and a policy allows it with justification. As of 2025-01-29, it is only used to
+// allow a user to disconnect Tailscale when the "always-on" mode is enabled.
+//
+// The header value is base64-encoded using the standard encoding defined in RFC 4648.
+//
+// See tailscale/corp#26146.
+const RequestReasonHeader = "X-Tailscale-Reason"
+
+// RequestReasonKey is the context key used to pass the request reason
+// when making a LocalAPI request via [local.Client].
+// It's value is a raw string. An empty string means no reason was provided.
+//
+// See tailscale/corp#26146.
+var RequestReasonKey = ctxkey.New(RequestReasonHeader, "")
+
 // WhoIsResponse is the JSON type returned by tailscaled debug server's /whois?ip=$IP handler.
 // In successful whois responses, Node and UserProfile are never nil.
 type WhoIsResponse struct {

client/tailscale/devices.go

@@ -79,6 +79,13 @@ type Device struct {
 	// Tailscale have attempted to collect this from the device but it has not
 	// opted in, PostureIdentity will have Disabled=true.
 	PostureIdentity *DevicePostureIdentity `json:"postureIdentity"`
+
+	// TailnetLockKey is the tailnet lock public key of the node as a hex string.
+	TailnetLockKey string `json:"tailnetLockKey,omitempty"`
+
+	// TailnetLockErr indicates an issue with the tailnet lock node-key signature
+	// on this device. This field is only populated when tailnet lock is enabled.
+	TailnetLockErr string `json:"tailnetLockError,omitempty"`
 }
 
 type DevicePostureIdentity struct {
@@ -131,7 +138,7 @@ func (c *Client) Devices(ctx context.Context, fields *DeviceFieldsOpts) (deviceL
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/devices", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("devices")
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -149,7 +156,7 @@ func (c *Client) Devices(ctx context.Context, fields *DeviceFieldsOpts) (deviceL
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var devices GetDevicesResponse
@@ -188,7 +195,7 @@ func (c *Client) Device(ctx context.Context, deviceID string, fields *DeviceFiel
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	err = json.Unmarshal(b, &device)
@@ -221,7 +228,7 @@ func (c *Client) DeleteDevice(ctx context.Context, deviceID string) (err error)
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 	return nil
 }
@@ -253,7 +260,7 @@ func (c *Client) SetAuthorized(ctx context.Context, deviceID string, authorized
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 
 	return nil
@@ -281,7 +288,7 @@ func (c *Client) SetTags(ctx context.Context, deviceID string, tags []string) er
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 
 	return nil

client/tailscale/dns.go

@@ -44,7 +44,7 @@ type DNSPreferences struct {
 }
 
 func (c *Client) dnsGETRequest(ctx context.Context, endpoint string) ([]byte, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/dns/%s", c.baseURL(), c.tailnet, endpoint)
+	path := c.BuildTailnetURL("dns", endpoint)
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -57,14 +57,14 @@ func (c *Client) dnsGETRequest(ctx context.Context, endpoint string) ([]byte, er
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	return b, nil
 }
 
 func (c *Client) dnsPOSTRequest(ctx context.Context, endpoint string, postData any) ([]byte, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/dns/%s", c.baseURL(), c.tailnet, endpoint)
+	path := c.BuildTailnetURL("dns", endpoint)
 	data, err := json.Marshal(&postData)
 	if err != nil {
 		return nil, err
@@ -84,7 +84,7 @@ func (c *Client) dnsPOSTRequest(ctx context.Context, endpoint string, postData a
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	return b, nil

client/tailscale/example/servetls/servetls.go

@@ -11,13 +11,14 @@ import (
 	"log"
 	"net/http"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 )
 
 func main() {
+	var lc local.Client
 	s := &http.Server{
 		TLSConfig: &tls.Config{
-			GetCertificate: tailscale.GetCertificate,
+			GetCertificate: lc.GetCertificate,
 		},
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			io.WriteString(w, "<h1>Hello from Tailscale!</h1> It works.")

client/tailscale/keys.go

@@ -40,7 +40,7 @@ type KeyDeviceCreateCapabilities struct {
 
 // Keys returns the list of keys for the current user.
 func (c *Client) Keys(ctx context.Context) ([]string, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("keys")
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -51,7 +51,7 @@ func (c *Client) Keys(ctx context.Context) ([]string, error) {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var keys struct {
@@ -99,7 +99,7 @@ func (c *Client) CreateKeyWithExpiry(ctx context.Context, caps KeyCapabilities,
 		return "", nil, err
 	}
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys", c.baseURL(), c.tailnet)
+	path := c.BuildTailnetURL("keys")
 	req, err := http.NewRequestWithContext(ctx, "POST", path, bytes.NewReader(bs))
 	if err != nil {
 		return "", nil, err
@@ -110,7 +110,7 @@ func (c *Client) CreateKeyWithExpiry(ctx context.Context, caps KeyCapabilities,
 		return "", nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return "", nil, handleErrorResponse(b, resp)
+		return "", nil, HandleErrorResponse(b, resp)
 	}
 
 	var key struct {
@@ -126,7 +126,7 @@ func (c *Client) CreateKeyWithExpiry(ctx context.Context, caps KeyCapabilities,
 // Key returns the metadata for the given key ID. Currently, capabilities are
 // only returned for auth keys, API keys only return general metadata.
 func (c *Client) Key(ctx context.Context, id string) (*Key, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys/%s", c.baseURL(), c.tailnet, id)
+	path := c.BuildTailnetURL("keys", id)
 	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -137,7 +137,7 @@ func (c *Client) Key(ctx context.Context, id string) (*Key, error) {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var key Key
@@ -149,7 +149,7 @@ func (c *Client) Key(ctx context.Context, id string) (*Key, error) {
 
 // DeleteKey deletes the key with the given ID.
 func (c *Client) DeleteKey(ctx context.Context, id string) error {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/keys/%s", c.baseURL(), c.tailnet, id)
+	path := c.BuildTailnetURL("keys", id)
 	req, err := http.NewRequestWithContext(ctx, "DELETE", path, nil)
 	if err != nil {
 		return err
@@ -160,7 +160,7 @@ func (c *Client) DeleteKey(ctx context.Context, id string) error {
 		return err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 	return nil
 }

client/tailscale/localclient_aliases.go

@@ -0,0 +1,106 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package tailscale
+
+import (
+	"context"
+	"crypto/tls"
+
+	"tailscale.com/client/local"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/ipn/ipnstate"
+)
+
+// ErrPeerNotFound is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+var ErrPeerNotFound = local.ErrPeerNotFound
+
+// LocalClient is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type LocalClient = local.Client
+
+// IPNBusWatcher is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type IPNBusWatcher = local.IPNBusWatcher
+
+// BugReportOpts is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type BugReportOpts = local.BugReportOpts
+
+// DebugPortMapOpts is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type DebugPortmapOpts = local.DebugPortmapOpts
+
+// PingOpts is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+type PingOpts = local.PingOpts
+
+// GetCertificate is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return local.GetCertificate(hi)
+}
+
+// SetVersionMismatchHandler is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func SetVersionMismatchHandler(f func(clientVer, serverVer string)) {
+	local.SetVersionMismatchHandler(f)
+}
+
+// IsAccessDeniedError is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func IsAccessDeniedError(err error) bool {
+	return local.IsAccessDeniedError(err)
+}
+
+// IsPreconditionsFailedError is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func IsPreconditionsFailedError(err error) bool {
+	return local.IsPreconditionsFailedError(err)
+}
+
+// WhoIs is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
+	return local.WhoIs(ctx, remoteAddr)
+}
+
+// Status is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func Status(ctx context.Context) (*ipnstate.Status, error) {
+	return local.Status(ctx)
+}
+
+// StatusWithoutPeers is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
+	return local.StatusWithoutPeers(ctx)
+}
+
+// CertPair is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+	return local.CertPair(ctx, domain)
+}
+
+// ExpandSNIName is an alias for tailscale.com/client/local.
+//
+// Deprecated: import tailscale.com/client/local instead.
+func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) {
+	return local.ExpandSNIName(ctx, name)
+}

client/tailscale/routes.go

@@ -44,7 +44,7 @@ func (c *Client) Routes(ctx context.Context, deviceID string) (routes *Routes, e
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var sr Routes
@@ -84,7 +84,7 @@ func (c *Client) SetRoutes(ctx context.Context, deviceID string, subnets []netip
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
+		return nil, HandleErrorResponse(b, resp)
 	}
 
 	var srr *Routes

client/tailscale/tailnet.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"net/url"
 
 	"tailscale.com/util/httpm"
 )
@@ -22,7 +21,7 @@ func (c *Client) TailnetDeleteRequest(ctx context.Context, tailnetID string) (er
 		}
 	}()
 
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s", c.baseURL(), url.PathEscape(string(tailnetID)))
+	path := c.BuildTailnetURL("tailnet")
 	req, err := http.NewRequestWithContext(ctx, httpm.DELETE, path, nil)
 	if err != nil {
 		return err
@@ -35,7 +34,7 @@ func (c *Client) TailnetDeleteRequest(ctx context.Context, tailnetID string) (er
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
+		return HandleErrorResponse(b, resp)
 	}
 
 	return nil

client/tailscale/tailscale.go

@@ -3,11 +3,12 @@
 
 //go:build go1.19
 
-// Package tailscale contains Go clients for the Tailscale LocalAPI and
-// Tailscale control plane API.
+// Package tailscale contains a Go client for the Tailscale control plane API.
 //
-// Warning: this package is in development and makes no API compatibility
-// promises as of 2022-04-29. It is subject to change at any time.
+// This package is only intended for internal and transitional use.
+//
+// Deprecated: the official control plane client is available at
+// tailscale.com/client/tailscale/v2.
 package tailscale
 
 import (
@@ -16,13 +17,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
+	"path"
 )
 
 // I_Acknowledge_This_API_Is_Unstable must be set true to use this package
-// for now. It was added 2022-04-29 when it was moved to this git repo
-// and will be removed when the public API has settled.
-//
-// TODO(bradfitz): remove this after the we're happy with the public API.
+// for now. This package is being replaced by tailscale.com/client/tailscale/v2.
 var I_Acknowledge_This_API_Is_Unstable = false
 
 // TODO: use url.PathEscape() for deviceID and tailnets when constructing requests.
@@ -36,6 +36,8 @@ const maxReadSize = 10 << 20
 //
 // Use NewClient to instantiate one. Exported fields should be set before
 // the client is used and not changed thereafter.
+//
+// Deprecated: use tailscale.com/client/tailscale/v2 instead.
 type Client struct {
 	// tailnet is the globally unique identifier for a Tailscale network, such
 	// as "example.com" or "user@gmail.com".
@@ -63,6 +65,46 @@ func (c *Client) httpClient() *http.Client {
 	return http.DefaultClient
 }
 
+// BuildURL builds a url to http(s)://<apiserver>/api/v2/<slash-separated-pathElements>
+// using the given pathElements. It url escapes each path element, so the
+// caller doesn't need to worry about that. The last item of pathElements can
+// be of type url.Values to add a query string to the URL.
+//
+// For example, BuildURL(devices, 5) with the default server URL would result in
+// https://api.tailscale.com/api/v2/devices/5.
+func (c *Client) BuildURL(pathElements ...any) string {
+	elem := make([]string, 1, len(pathElements)+1)
+	elem[0] = "/api/v2"
+	var query string
+	for i, pathElement := range pathElements {
+		if uv, ok := pathElement.(url.Values); ok && i == len(pathElements)-1 {
+			query = uv.Encode()
+		} else {
+			elem = append(elem, url.PathEscape(fmt.Sprint(pathElement)))
+		}
+	}
+	url := c.baseURL() + path.Join(elem...)
+	if query != "" {
+		url += "?" + query
+	}
+	return url
+}
+
+// BuildTailnetURL builds a url to http(s)://<apiserver>/api/v2/tailnet/<tailnet>/<slash-separated-pathElements>
+// using the given pathElements. It url escapes each path element, so the
+// caller doesn't need to worry about that. The last item of pathElements can
+// be of type url.Values to add a query string to the URL.
+//
+// For example, BuildTailnetURL(policy, validate) with the default server URL and a tailnet of "example.com"
+// would result in https://api.tailscale.com/api/v2/tailnet/example.com/policy/validate.
+func (c *Client) BuildTailnetURL(pathElements ...any) string {
+	allElements := make([]any, 2, len(pathElements)+2)
+	allElements[0] = "tailnet"
+	allElements[1] = c.tailnet
+	allElements = append(allElements, pathElements...)
+	return c.BuildURL(allElements...)
+}
+
 func (c *Client) baseURL() string {
 	if c.BaseURL != "" {
 		return c.BaseURL
@@ -98,6 +140,8 @@ func (c *Client) setAuth(r *http.Request) {
 // If httpClient is nil, then http.DefaultClient is used.
 // "api.tailscale.com" is set as the BaseURL for the returned client
 // and can be changed manually by the user.
+//
+// Deprecated: use tailscale.com/client/tailscale/v2 instead.
 func NewClient(tailnet string, auth AuthMethod) *Client {
 	return &Client{
 		tailnet:   tailnet,
@@ -148,12 +192,14 @@ func (e ErrResponse) Error() string {
 	return fmt.Sprintf("Status: %d, Message: %q", e.Status, e.Message)
 }
 
-// handleErrorResponse decodes the error message from the server and returns
+// HandleErrorResponse decodes the error message from the server and returns
 // an ErrResponse from it.
-func handleErrorResponse(b []byte, resp *http.Response) error {
+//
+// Deprecated: use tailscale.com/client/tailscale/v2 instead.
+func HandleErrorResponse(b []byte, resp *http.Response) error {
 	var errResp ErrResponse
 	if err := json.Unmarshal(b, &errResp); err != nil {
-		return err
+		return fmt.Errorf("json.Unmarshal %q: %w", b, err)
 	}
 	errResp.Status = resp.StatusCode
 	return errResp

client/tailscale/tailscale_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package tailscale
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestClientBuildURL(t *testing.T) {
+	c := Client{BaseURL: "http://127.0.0.1:1234"}
+	for _, tt := range []struct {
+		desc     string
+		elements []any
+		want     string
+	}{
+		{
+			desc:     "single-element",
+			elements: []any{"devices"},
+			want:     "http://127.0.0.1:1234/api/v2/devices",
+		},
+		{
+			desc:     "multiple-elements",
+			elements: []any{"tailnet", "example.com"},
+			want:     "http://127.0.0.1:1234/api/v2/tailnet/example.com",
+		},
+		{
+			desc:     "escape-element",
+			elements: []any{"tailnet", "example dot com?foo=bar"},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example%20dot%20com%3Ffoo=bar`,
+		},
+		{
+			desc:     "url.Values",
+			elements: []any{"tailnet", "example.com", "acl", url.Values{"details": {"1"}}},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example.com/acl?details=1`,
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			got := c.BuildURL(tt.elements...)
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestClientBuildTailnetURL(t *testing.T) {
+	c := Client{
+		BaseURL: "http://127.0.0.1:1234",
+		tailnet: "example.com",
+	}
+	for _, tt := range []struct {
+		desc     string
+		elements []any
+		want     string
+	}{
+		{
+			desc:     "single-element",
+			elements: []any{"devices"},
+			want:     "http://127.0.0.1:1234/api/v2/tailnet/example.com/devices",
+		},
+		{
+			desc:     "multiple-elements",
+			elements: []any{"devices", 123},
+			want:     "http://127.0.0.1:1234/api/v2/tailnet/example.com/devices/123",
+		},
+		{
+			desc:     "escape-element",
+			elements: []any{"foo bar?baz=qux"},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example.com/foo%20bar%3Fbaz=qux`,
+		},
+		{
+			desc:     "url.Values",
+			elements: []any{"acl", url.Values{"details": {"1"}}},
+			want:     `http://127.0.0.1:1234/api/v2/tailnet/example.com/acl?details=1`,
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			got := c.BuildTailnetURL(tt.elements...)
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

client/web/src/components/views/login-view.tsx

@@ -1,13 +1,11 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-import React, { useState } from "react"
+import React from "react"
 import { useAPI } from "src/api"
 import TailscaleIcon from "src/assets/icons/tailscale-icon.svg?react"
 import { NodeData } from "src/types"
 import Button from "src/ui/button"
-import Collapsible from "src/ui/collapsible"
-import Input from "src/ui/input"
 
 /**
  * LoginView is rendered when the client is not authenticated
@@ -15,8 +13,6 @@ import Input from "src/ui/input"
  */
 export default function LoginView({ data }: { data: NodeData }) {
   const api = useAPI()
-  const [controlURL, setControlURL] = useState<string>("")
-  const [authKey, setAuthKey] = useState<string>("")
 
   return (
     <div className="mb-8 py-6 px-8 bg-white rounded-md shadow-2xl">
@@ -88,8 +84,6 @@ export default function LoginView({ data }: { data: NodeData }) {
                 action: "up",
                 data: {
                   Reauthenticate: true,
-                  ControlURL: controlURL,
-                  AuthKey: authKey,
                 },
               })
             }
@@ -98,34 +92,6 @@ export default function LoginView({ data }: { data: NodeData }) {
           >
             Log In
           </Button>
-          <Collapsible trigger="Advanced options">
-            <h4 className="font-medium mb-1 mt-2">Auth Key</h4>
-            <p className="text-sm text-gray-500">
-              Connect with a pre-authenticated key.{" "}
-              <a
-                href="https://tailscale.com/kb/1085/auth-keys/"
-                className="link"
-                target="_blank"
-                rel="noreferrer"
-              >
-                Learn more &rarr;
-              </a>
-            </p>
-            <Input
-              className="mt-2"
-              value={authKey}
-              onChange={(e) => setAuthKey(e.target.value)}
-              placeholder="tskey-auth-XXX"
-            />
-            <h4 className="font-medium mt-3 mb-1">Server URL</h4>
-            <p className="text-sm text-gray-500">Base URL of control server.</p>
-            <Input
-              className="mt-2"
-              value={controlURL}
-              onChange={(e) => setControlURL(e.target.value)}
-              placeholder="https://login.tailscale.com/"
-            />
-          </Collapsible>
         </>
       )}
     </div>

client/web/web.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/gorilla/csrf"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
@@ -50,7 +50,7 @@ type Server struct {
 	mode ServerMode
 
 	logf    logger.Logf
-	lc      *tailscale.LocalClient
+	lc      *local.Client
 	timeNow func() time.Time
 
 	// devMode indicates that the server run with frontend assets
@@ -125,9 +125,9 @@ type ServerOpts struct {
 	// PathPrefix is the URL prefix added to requests by CGI or reverse proxy.
 	PathPrefix string
 
-	// LocalClient is the tailscale.LocalClient to use for this web server.
+	// LocalClient is the local.Client to use for this web server.
 	// If nil, a new one will be created.
-	LocalClient *tailscale.LocalClient
+	LocalClient *local.Client
 
 	// TimeNow optionally provides a time function.
 	// time.Now is used as default.
@@ -166,7 +166,7 @@ func NewServer(opts ServerOpts) (s *Server, err error) {
 		return nil, fmt.Errorf("invalid Mode provided")
 	}
 	if opts.LocalClient == nil {
-		opts.LocalClient = &tailscale.LocalClient{}
+		opts.LocalClient = &local.Client{}
 	}
 	s = &Server{
 		mode:        opts.Mode,
@@ -203,25 +203,9 @@ func NewServer(opts ServerOpts) (s *Server, err error) {
 	}
 	s.assetsHandler, s.assetsCleanup = assetsHandler(s.devMode)
 
-	var metric string // clientmetric to report on startup
-
-	// Create handler for "/api" requests with CSRF protection.
-	// We don't require secure cookies, since the web client is regularly used
-	// on network appliances that are served on local non-https URLs.
-	// The client is secured by limiting the interface it listens on,
-	// or by authenticating requests before they reach the web client.
-	csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false))
-	switch s.mode {
-	case LoginServerMode:
-		s.apiHandler = csrfProtect(http.HandlerFunc(s.serveLoginAPI))
-		metric = "web_login_client_initialization"
-	case ReadOnlyServerMode:
-		s.apiHandler = csrfProtect(http.HandlerFunc(s.serveLoginAPI))
-		metric = "web_readonly_client_initialization"
-	case ManageServerMode:
-		s.apiHandler = csrfProtect(http.HandlerFunc(s.serveAPI))
-		metric = "web_client_initialization"
-	}
+	var metric string
+	s.apiHandler, metric = s.modeAPIHandler(s.mode)
+	s.apiHandler = s.withCSRF(s.apiHandler)
 
 	// Don't block startup on reporting metric.
 	// Report in separate go routine with 5 second timeout.
@@ -234,6 +218,39 @@ func NewServer(opts ServerOpts) (s *Server, err error) {
 	return s, nil
 }
 
+func (s *Server) withCSRF(h http.Handler) http.Handler {
+	csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false))
+
+	// ref https://github.com/tailscale/tailscale/pull/14822
+	// signal to the CSRF middleware that the request is being served over
+	// plaintext HTTP to skip TLS-only header checks.
+	withSetPlaintext := func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			r = csrf.PlaintextHTTPRequest(r)
+			h.ServeHTTP(w, r)
+		})
+	}
+
+	// NB: the order of the withSetPlaintext and csrfProtect calls is important
+	// to ensure that we signal to the CSRF middleware that the request is being
+	// served over plaintext HTTP and not over TLS as it presumes by default.
+	return withSetPlaintext(csrfProtect(h))
+}
+
+func (s *Server) modeAPIHandler(mode ServerMode) (http.Handler, string) {
+	switch mode {
+	case LoginServerMode:
+		return http.HandlerFunc(s.serveLoginAPI), "web_login_client_initialization"
+	case ReadOnlyServerMode:
+		return http.HandlerFunc(s.serveLoginAPI), "web_readonly_client_initialization"
+	case ManageServerMode:
+		return http.HandlerFunc(s.serveAPI), "web_client_initialization"
+	default: // invalid mode
+		log.Fatalf("invalid mode: %v", mode)
+	}
+	return nil, ""
+}
+
 func (s *Server) Shutdown() {
 	s.logf("web.Server: shutting down")
 	if s.assetsCleanup != nil {
@@ -318,7 +335,8 @@ func (s *Server) requireTailscaleIP(w http.ResponseWriter, r *http.Request) (han
 		ipv6ServiceHost = "[" + tsaddr.TailscaleServiceIPv6String + "]"
 	)
 	// allow requests on quad-100 (or ipv6 equivalent)
-	if r.Host == ipv4ServiceHost || r.Host == ipv6ServiceHost {
+	host := strings.TrimSuffix(r.Host, ":80")
+	if host == ipv4ServiceHost || host == ipv6ServiceHost {
 		return false
 	}
 

client/web/web_test.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/http/cookiejar"
 	"net/http/httptest"
 	"net/netip"
 	"net/url"
@@ -20,7 +21,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/client/tailscale"
+	"github.com/gorilla/csrf"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -120,7 +122,7 @@ func TestServeAPI(t *testing.T) {
 
 	s := &Server{
 		mode:    ManageServerMode,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 		timeNow: time.Now,
 	}
 
@@ -288,7 +290,7 @@ func TestGetTailscaleBrowserSession(t *testing.T) {
 
 	s := &Server{
 		timeNow: time.Now,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 	}
 
 	// Add some browser sessions to cache state.
@@ -457,7 +459,7 @@ func TestAuthorizeRequest(t *testing.T) {
 
 	s := &Server{
 		mode:    ManageServerMode,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 		timeNow: time.Now,
 	}
 	validCookie := "ts-cookie"
@@ -572,7 +574,7 @@ func TestServeAuth(t *testing.T) {
 
 	s := &Server{
 		mode:        ManageServerMode,
-		lc:          &tailscale.LocalClient{Dial: lal.Dial},
+		lc:          &local.Client{Dial: lal.Dial},
 		timeNow:     func() time.Time { return timeNow },
 		newAuthURL:  mockNewAuthURL,
 		waitAuthURL: mockWaitAuthURL,
@@ -914,7 +916,7 @@ func TestServeAPIAuthMetricLogging(t *testing.T) {
 
 	s := &Server{
 		mode:        ManageServerMode,
-		lc:          &tailscale.LocalClient{Dial: lal.Dial},
+		lc:          &local.Client{Dial: lal.Dial},
 		timeNow:     func() time.Time { return timeNow },
 		newAuthURL:  mockNewAuthURL,
 		waitAuthURL: mockWaitAuthURL,
@@ -1126,7 +1128,7 @@ func TestRequireTailscaleIP(t *testing.T) {
 
 	s := &Server{
 		mode:    ManageServerMode,
-		lc:      &tailscale.LocalClient{Dial: lal.Dial},
+		lc:      &local.Client{Dial: lal.Dial},
 		timeNow: time.Now,
 		logf:    t.Logf,
 	}
@@ -1175,6 +1177,16 @@ func TestRequireTailscaleIP(t *testing.T) {
 			target:      "http://[fd7a:115c:a1e0::53]/",
 			wantHandled: false,
 		},
+		{
+			name:        "quad-100:80",
+			target:      "http://100.100.100.100:80/",
+			wantHandled: false,
+		},
+		{
+			name:        "ipv6-service-addr:80",
+			target:      "http://[fd7a:115c:a1e0::53]:80/",
+			wantHandled: false,
+		},
 	}
 
 	for _, tt := range tests {
@@ -1477,3 +1489,83 @@ func mockWaitAuthURL(_ context.Context, id string, src tailcfg.NodeID) (*tailcfg
 		return nil, errors.New("unknown id")
 	}
 }
+
+func TestCSRFProtect(t *testing.T) {
+	s := &Server{}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /test/csrf-token", func(w http.ResponseWriter, r *http.Request) {
+		token := csrf.Token(r)
+		_, err := io.WriteString(w, token)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	mux.HandleFunc("POST /test/csrf-protected", func(w http.ResponseWriter, r *http.Request) {
+		_, err := io.WriteString(w, "ok")
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	h := s.withCSRF(mux)
+	ser := httptest.NewServer(h)
+	defer ser.Close()
+
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatalf("unable to construct cookie jar: %v", err)
+	}
+
+	client := ser.Client()
+	client.Jar = jar
+
+	// make GET request to populate cookie jar
+	resp, err := client.Get(ser.URL + "/test/csrf-token")
+	if err != nil {
+		t.Fatalf("unable to make request: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v", resp.Status)
+	}
+	tokenBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unable to read body: %v", err)
+	}
+
+	csrfToken := strings.TrimSpace(string(tokenBytes))
+	if csrfToken == "" {
+		t.Fatal("empty csrf token")
+	}
+
+	// make a POST request without the CSRF header; ensure it fails
+	resp, err = client.Post(ser.URL+"/test/csrf-protected", "text/plain", nil)
+	if err != nil {
+		t.Fatalf("unable to make request: %v", err)
+	}
+	if resp.StatusCode != http.StatusForbidden {
+		t.Fatalf("unexpected status: %v", resp.Status)
+	}
+
+	// make a POST request with the CSRF header; ensure it succeeds
+	req, err := http.NewRequest("POST", ser.URL+"/test/csrf-protected", nil)
+	if err != nil {
+		t.Fatalf("error building request: %v", err)
+	}
+	req.Header.Set("X-CSRF-Token", csrfToken)
+	resp, err = client.Do(req)
+	if err != nil {
+		t.Fatalf("unable to make request: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v", resp.Status)
+	}
+	defer resp.Body.Close()
+	out, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unable to read body: %v", err)
+	}
+	if string(out) != "ok" {
+		t.Fatalf("unexpected body: %q", out)
+	}
+}

clientupdate/clientupdate.go

@@ -27,6 +27,8 @@ import (
 	"strconv"
 	"strings"
 
+	"tailscale.com/hostinfo"
+	"tailscale.com/types/lazy"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 	"tailscale.com/version"
@@ -169,6 +171,12 @@ func NewUpdater(args Arguments) (*Updater, error) {
 type updateFunction func() error
 
 func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
+	hi := hostinfo.New()
+	// We don't know how to update custom tsnet binaries, it's up to the user.
+	if hi.Package == "tsnet" {
+		return nil, false
+	}
+
 	switch runtime.GOOS {
 	case "windows":
 		return up.updateWindows, true
@@ -242,9 +250,13 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 	return nil, false
 }
 
+var canAutoUpdateCache lazy.SyncValue[bool]
+
 // CanAutoUpdate reports whether auto-updating via the clientupdate package
 // is supported for the current os/distro.
-func CanAutoUpdate() bool {
+func CanAutoUpdate() bool { return canAutoUpdateCache.Get(canAutoUpdateUncached) }
+
+func canAutoUpdateUncached() bool {
 	if version.IsMacSysExt() {
 		// Macsys uses Sparkle for auto-updates, which doesn't have an update
 		// function in this package.

cmd/containerboot/certs.go

@@ -0,0 +1,156 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/util/goroutines"
+	"tailscale.com/util/mak"
+)
+
+// certManager is responsible for issuing certificates for known domains and for
+// maintaining a loop that re-attempts issuance daily.
+// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for
+// HA Ingress HTTPS endpoints ('write' replicas).
+type certManager struct {
+	lc      localClient
+	tracker goroutines.Tracker // tracks running goroutines
+	mu      sync.Mutex         // guards the following
+	// certLoops contains a map of DNS names, for which we currently need to
+	// manage certs to cancel functions that allow stopping a goroutine when
+	// we no longer need to manage certs for the DNS name.
+	certLoops map[string]context.CancelFunc
+}
+
+// ensureCertLoops ensures that, for all currently managed Service HTTPS
+// endpoints, there is a cert loop responsible for issuing and ensuring the
+// renewal of the TLS certs.
+// ServeConfig must not be nil.
+func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error {
+	if sc == nil {
+		return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
+	}
+	currentDomains := make(map[string]bool)
+	const httpsPort = "443"
+	for _, service := range sc.Services {
+		for hostPort := range service.Web {
+			domain, port, err := net.SplitHostPort(string(hostPort))
+			if err != nil {
+				return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
+			}
+			if port != httpsPort { // HA Ingress' HTTP endpoint
+				continue
+			}
+			currentDomains[domain] = true
+		}
+	}
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	for domain := range currentDomains {
+		if _, exists := cm.certLoops[domain]; !exists {
+			cancelCtx, cancel := context.WithCancel(ctx)
+			mak.Set(&cm.certLoops, domain, cancel)
+			// Note that most of the issuance anyway happens
+			// serially because the cert client has a shared lock
+			// that's held during any issuance.
+			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
+		}
+	}
+
+	// Stop goroutines for domain names that are no longer in the config.
+	for domain, cancel := range cm.certLoops {
+		if !currentDomains[domain] {
+			cancel()
+			delete(cm.certLoops, domain)
+		}
+	}
+	return nil
+}
+
+// runCertLoop:
+// - calls localAPI certificate endpoint to ensure that certs are issued for the
+// given domain name
+// - calls localAPI certificate endpoint daily to ensure that certs are renewed
+// - if certificate issuance failed retries after an exponential backoff period
+// starting at 1 minute and capped at 24 hours. Reset the backoff once issuance succeeds.
+// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get
+// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write'
+// replica does not get any HTTPS requests.
+// https://letsencrypt.org/docs/integration-guide/#retrying-failures
+func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
+	const (
+		normalInterval   = 24 * time.Hour  // regular renewal check
+		initialRetry     = 1 * time.Minute // initial backoff after a failure
+		maxRetryInterval = 24 * time.Hour  // max backoff period
+	)
+	timer := time.NewTimer(0) // fire off timer immediately
+	defer timer.Stop()
+	retryCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			// We call the certificate endpoint, but don't do anything
+			// with the returned certs here.
+			// The call to the certificate endpoint will ensure that
+			// certs are issued/renewed as needed and stored in the
+			// relevant state store. For example, for HA Ingress
+			// 'write' replica, the cert and key will be stored in a
+			// Kubernetes Secret named after the domain for which we
+			// are issuing.
+			// Note that renewals triggered by the call to the
+			// certificates endpoint here and by renewal check
+			// triggered during a call to node's HTTPS endpoint
+			// share the same state/renewal lock mechanism, so we
+			// should not run into redundant issuances during
+			// concurrent renewal checks.
+			// TODO(irbekrm): maybe it is worth adding a new
+			// issuance endpoint that explicitly only triggers
+			// issuance and stores certs in the relevant store, but
+			// does not return certs to the caller?
+
+			// An issuance holds a shared lock, so we need to avoid
+			// a situation where other services cannot issue certs
+			// because a single one is holding the lock.
+			ctxT, cancel := context.WithTimeout(ctx, time.Second*300)
+			defer cancel()
+			_, _, err := cm.lc.CertPair(ctxT, domain)
+			if err != nil {
+				log.Printf("error refreshing certificate for %s: %v", domain, err)
+			}
+			var nextInterval time.Duration
+			// TODO(irbekrm): distinguish between LE rate limit
+			// errors and other error types like transient network
+			// errors.
+			if err == nil {
+				retryCount = 0
+				nextInterval = normalInterval
+			} else {
+				retryCount++
+				// Calculate backoff: initialRetry * 2^(retryCount-1)
+				// For retryCount=1: 1min * 2^0 = 1min
+				// For retryCount=2: 1min * 2^1 = 2min
+				// For retryCount=3: 1min * 2^2 = 4min
+				backoff := initialRetry * time.Duration(1<<(retryCount-1))
+				if backoff > maxRetryInterval {
+					backoff = maxRetryInterval
+				}
+				nextInterval = backoff
+				log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n",
+					domain, retryCount, err, nextInterval)
+			}
+			timer.Reset(nextInterval)
+		}
+	}
+}

cmd/containerboot/certs_test.go

@@ -0,0 +1,229 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+)
+
+// TestEnsureCertLoops tests that the certManager correctly starts and stops
+// update loops for certs when the serve config changes. It tracks goroutine
+// count and uses that as a validator that the expected number of cert loops are
+// running.
+func TestEnsureCertLoops(t *testing.T) {
+	tests := []struct {
+		name              string
+		initialConfig     *ipn.ServeConfig
+		updatedConfig     *ipn.ServeConfig
+		initialGoroutines int64 // after initial serve config is applied
+		updatedGoroutines int64 // after updated serve config is applied
+		wantErr           bool
+	}{
+		{
+			name:              "empty_serve_config",
+			initialConfig:     &ipn.ServeConfig{},
+			initialGoroutines: 0,
+		},
+		{
+			name:              "nil_serve_config",
+			initialConfig:     nil,
+			initialGoroutines: 0,
+			wantErr:           true,
+		},
+		{
+			name:          "empty_to_one_service",
+			initialConfig: &ipn.ServeConfig{},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 0,
+			updatedGoroutines: 1,
+		},
+		{
+			name: "single_service",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+		},
+		{
+			name: "multiple_services",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // one loop per domain across all services
+		},
+		{
+			name: "ignore_non_https_ports",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+							"my-app.tailnetxyz.ts.net:80":  {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1, // only one loop for the 443 endpoint
+		},
+		{
+			name: "remove_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // initially two loops (one per service)
+			updatedGoroutines: 1, // one loop after removing service2
+		},
+		{
+			name: "add_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+			updatedGoroutines: 2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			cm := &certManager{
+				lc:        &fakeLocalClient{},
+				certLoops: make(map[string]context.CancelFunc),
+			}
+
+			allDone := make(chan bool, 1)
+			defer cm.tracker.AddDoneCallback(func() {
+				cm.mu.Lock()
+				defer cm.mu.Unlock()
+				if cm.tracker.RunningGoroutines() > 0 {
+					return
+				}
+				select {
+				case allDone <- true:
+				default:
+				}
+			})()
+
+			err := cm.ensureCertLoops(ctx, tt.initialConfig)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ensureCertLoops() error = %v", err)
+			}
+
+			if got := cm.tracker.RunningGoroutines(); got != tt.initialGoroutines {
+				t.Errorf("after initial config: got %d running goroutines, want %d", got, tt.initialGoroutines)
+			}
+
+			if tt.updatedConfig != nil {
+				if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil {
+					t.Fatalf("ensureCertLoops() error on update = %v", err)
+				}
+
+				// Although starting goroutines and cancelling
+				// the context happens in the main goroutine, it
+				// the actual goroutine exit when a context is
+				// cancelled does not- so wait for a bit for the
+				// running goroutine count to reach the expected
+				// number.
+				deadline := time.After(5 * time.Second)
+				for {
+					if got := cm.tracker.RunningGoroutines(); got == tt.updatedGoroutines {
+						break
+					}
+					select {
+					case <-deadline:
+						t.Fatalf("timed out waiting for goroutine count to reach %d, currently at %d",
+							tt.updatedGoroutines, cm.tracker.RunningGoroutines())
+					case <-time.After(10 * time.Millisecond):
+						continue
+					}
+				}
+			}
+
+			if tt.updatedGoroutines == 0 {
+				return // no goroutines to wait for
+			}
+			// cancel context to make goroutines exit
+			cancel()
+			select {
+			case <-time.After(5 * time.Second):
+				t.Fatal("timed out waiting for goroutine to finish")
+			case <-allDone:
+			}
+		})
+	}
+}

cmd/containerboot/healthz.go

@@ -6,9 +6,12 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"net/http"
 	"sync"
+
+	"tailscale.com/kube/kubetypes"
 )
 
 // healthz is a simple health check server, if enabled it returns 200 OK if
@@ -17,6 +20,7 @@ import (
 type healthz struct {
 	sync.Mutex
 	hasAddrs bool
+	podIPv4  string
 }
 
 func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -24,7 +28,10 @@ func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer h.Unlock()
 
 	if h.hasAddrs {
-		w.Write([]byte("ok"))
+		w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4)
+		if _, err := w.Write([]byte("ok")); err != nil {
+			http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError)
+		}
 	} else {
 		http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable)
 	}
@@ -43,8 +50,8 @@ func (h *healthz) update(healthy bool) {
 // healthHandlers registers a simple health handler at /healthz.
 // A containerized tailscale instance is considered healthy if
 // it has at least one tailnet IP address.
-func healthHandlers(mux *http.ServeMux) *healthz {
-	h := &healthz{}
+func healthHandlers(mux *http.ServeMux, podIPv4 string) *healthz {
+	h := &healthz{podIPv4: podIPv4}
 	mux.Handle("GET /healthz", h)
 	return h
 }

cmd/containerboot/kube.go

@@ -8,15 +8,22 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"log"
 	"net/http"
 	"net/netip"
 	"os"
+	"strings"
+	"time"
 
+	"tailscale.com/ipn"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
 	"tailscale.com/kube/kubetypes"
+	"tailscale.com/logtail/backoff"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 )
 
 // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
@@ -126,3 +133,62 @@ func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error {
 	}
 	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }
+
+// waitForConsistentState waits for tailscaled to finish writing state if it
+// looks like it's started. It is designed to reduce the likelihood that
+// tailscaled gets shut down in the window between authenticating to control
+// and finishing writing state. However, it's not bullet proof because we can't
+// atomically authenticate and write state.
+func (kc *kubeClient) waitForConsistentState(ctx context.Context) error {
+	var logged bool
+
+	bo := backoff.NewBackoff("", logger.Discard, 2*time.Second)
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		secret, err := kc.GetSecret(ctx, kc.stateSecret)
+		if ctx.Err() != nil || kubeclient.IsNotFoundErr(err) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("getting Secret %q: %v", kc.stateSecret, err)
+		}
+
+		if hasConsistentState(secret.Data) {
+			return nil
+		}
+
+		if !logged {
+			log.Printf("Waiting for tailscaled to finish writing state to Secret %q", kc.stateSecret)
+			logged = true
+		}
+		bo.BackOff(ctx, errors.New("")) // Fake error to trigger actual sleep.
+	}
+}
+
+// hasConsistentState returns true is there is either no state or the full set
+// of expected keys are present.
+func hasConsistentState(d map[string][]byte) bool {
+	var (
+		_, hasCurrent = d[string(ipn.CurrentProfileStateKey)]
+		_, hasKnown   = d[string(ipn.KnownProfilesStateKey)]
+		_, hasMachine = d[string(ipn.MachineKeyStateKey)]
+		hasProfile    bool
+	)
+
+	for k := range d {
+		if strings.HasPrefix(k, "profile-") {
+			if hasProfile {
+				return false // We only expect one profile.
+			}
+			hasProfile = true
+		}
+	}
+
+	// Approximate check, we don't want to reimplement all of profileManager.
+	return (hasCurrent && hasKnown && hasMachine && hasProfile) ||
+		(!hasCurrent && !hasKnown && !hasMachine && !hasProfile)
+}

cmd/containerboot/kube_test.go

@@ -9,8 +9,10 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"tailscale.com/ipn"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
 )
@@ -205,3 +207,34 @@ func TestSetupKube(t *testing.T) {
 		})
 	}
 }
+
+func TestWaitForConsistentState(t *testing.T) {
+	data := map[string][]byte{
+		// Missing _current-profile.
+		string(ipn.KnownProfilesStateKey): []byte(""),
+		string(ipn.MachineKeyStateKey):    []byte(""),
+		"profile-foo":                     []byte(""),
+	}
+	kc := &kubeClient{
+		Client: &kubeclient.FakeClient{
+			GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+				return &kubeapi.Secret{
+					Data: data,
+				}, nil
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if err := kc.waitForConsistentState(ctx); err != context.DeadlineExceeded {
+		t.Fatalf("expected DeadlineExceeded, got %v", err)
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	data[string(ipn.CurrentProfileStateKey)] = []byte("")
+	if err := kc.waitForConsistentState(ctx); err != nil {
+		t.Fatalf("expected nil, got %v", err)
+	}
+}

cmd/containerboot/main.go

@@ -137,53 +137,83 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) {
 }
 
 func main() {
+	if err := run(); err != nil && !errors.Is(err, context.Canceled) {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
 	log.SetPrefix("boot: ")
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	cfg, err := configFromEnv()
 	if err != nil {
-		log.Fatalf("invalid configuration: %v", err)
+		return fmt.Errorf("invalid configuration: %w", err)
 	}
 
 	if !cfg.UserspaceMode {
 		if err := ensureTunFile(cfg.Root); err != nil {
-			log.Fatalf("Unable to create tuntap device file: %v", err)
+			return fmt.Errorf("unable to create tuntap device file: %w", err)
 		}
 		if cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.Routes != nil || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" {
 			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTargetIP, cfg.TailnetTargetIP, cfg.TailnetTargetFQDN, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
-					log.Fatalf("You can either set the sysctls as a privileged initContainer, or run the tailscale container with privileged=true.")
+					return fmt.Errorf("you can either set the sysctls as a privileged initContainer, or run the tailscale container with privileged=true.")
 				} else {
-					log.Fatalf("You can fix this by running the container with privileged=true, or the equivalent in your container runtime that permits access to sysctls.")
+					return fmt.Errorf("you can fix this by running the container with privileged=true, or the equivalent in your container runtime that permits access to sysctls.")
 				}
 			}
 		}
 	}
 
-	// Context is used for all setup stuff until we're in steady
+	// Root context for the whole containerboot process, used to make sure
+	// shutdown signals are promptly and cleanly handled.
+	ctx, cancel := contextWithExitSignalWatch()
+	defer cancel()
+
+	// bootCtx is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
-	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	bootCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 
 	var kc *kubeClient
 	if cfg.InKubernetes {
 		kc, err = newKubeClient(cfg.Root, cfg.KubeSecret)
 		if err != nil {
-			log.Fatalf("error initializing kube client: %v", err)
+			return fmt.Errorf("error initializing kube client: %w", err)
 		}
 		if err := cfg.setupKube(bootCtx, kc); err != nil {
-			log.Fatalf("error setting up for running on Kubernetes: %v", err)
+			return fmt.Errorf("error setting up for running on Kubernetes: %w", err)
 		}
 	}
 
 	client, daemonProcess, err := startTailscaled(bootCtx, cfg)
 	if err != nil {
-		log.Fatalf("failed to bring up tailscale: %v", err)
+		return fmt.Errorf("failed to bring up tailscale: %w", err)
 	}
 	killTailscaled := func() {
+		if hasKubeStateStore(cfg) {
+			// Check we're not shutting tailscaled down while it's still writing
+			// state. If we authenticate and fail to write all the state, we'll
+			// never recover automatically.
+			//
+			// The default termination grace period for a Pod is 30s. We wait 25s at
+			// most so that we still reserve some of that budget for tailscaled
+			// to receive and react to a SIGTERM before the SIGKILL that k8s
+			// will send at the end of the grace period.
+			ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+			defer cancel()
+
+			log.Printf("Checking for consistent state")
+			err := kc.waitForConsistentState(ctx)
+			if err != nil {
+				log.Printf("Error waiting for consistent state on shutdown: %v", err)
+			}
+		}
+		log.Printf("Sending SIGTERM to tailscaled")
 		if err := daemonProcess.Signal(unix.SIGTERM); err != nil {
 			log.Fatalf("error shutting tailscaled down: %v", err)
 		}
@@ -191,17 +221,18 @@ func main() {
 	defer killTailscaled()
 
 	var healthCheck *healthz
+	ep := &egressProxy{}
 	if cfg.HealthCheckAddrPort != "" {
 		mux := http.NewServeMux()
 
 		log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
-		healthCheck = healthHandlers(mux)
+		healthCheck = healthHandlers(mux, cfg.PodIPv4)
 
 		close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
 		defer close()
 	}
 
-	if cfg.localMetricsEnabled() || cfg.localHealthEnabled() {
+	if cfg.localMetricsEnabled() || cfg.localHealthEnabled() || cfg.egressSvcsTerminateEPEnabled() {
 		mux := http.NewServeMux()
 
 		if cfg.localMetricsEnabled() {
@@ -211,7 +242,11 @@ func main() {
 
 		if cfg.localHealthEnabled() {
 			log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
-			healthCheck = healthHandlers(mux)
+			healthCheck = healthHandlers(mux, cfg.PodIPv4)
+		}
+		if cfg.EgressProxiesCfgPath != "" {
+			log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP)
+			ep.registerHandlers(mux)
 		}
 
 		close := runHTTPServer(mux, cfg.LocalAddrPort)
@@ -226,7 +261,7 @@ func main() {
 
 	w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
 	if err != nil {
-		log.Fatalf("failed to watch tailscaled for updates: %v", err)
+		return fmt.Errorf("failed to watch tailscaled for updates: %w", err)
 	}
 
 	// Now that we've started tailscaled, we can symlink the socket to the
@@ -262,18 +297,18 @@ func main() {
 		didLogin = true
 		w.Close()
 		if err := tailscaleUp(bootCtx, cfg); err != nil {
-			return fmt.Errorf("failed to auth tailscale: %v", err)
+			return fmt.Errorf("failed to auth tailscale: %w", err)
 		}
 		w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 		if err != nil {
-			return fmt.Errorf("rewatching tailscaled for updates after auth: %v", err)
+			return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err)
 		}
 		return nil
 	}
 
 	if isTwoStepConfigAlwaysAuth(cfg) {
 		if err := authTailscale(); err != nil {
-			log.Fatalf("failed to auth tailscale: %v", err)
+			return fmt.Errorf("failed to auth tailscale: %w", err)
 		}
 	}
 
@@ -281,7 +316,7 @@ authLoop:
 	for {
 		n, err := w.Next()
 		if err != nil {
-			log.Fatalf("failed to read from tailscaled: %v", err)
+			return fmt.Errorf("failed to read from tailscaled: %w", err)
 		}
 
 		if n.State != nil {
@@ -290,10 +325,10 @@ authLoop:
 				if isOneStepConfig(cfg) {
 					// This could happen if this is the first time tailscaled was run for this
 					// device and the auth key was not passed via the configfile.
-					log.Fatalf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
+					return fmt.Errorf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
 				}
 				if err := authTailscale(); err != nil {
-					log.Fatalf("failed to auth tailscale: %v", err)
+					return fmt.Errorf("failed to auth tailscale: %w", err)
 				}
 			case ipn.NeedsMachineAuth:
 				log.Printf("machine authorization required, please visit the admin panel")
@@ -313,14 +348,11 @@ authLoop:
 
 	w.Close()
 
-	ctx, cancel := contextWithExitSignalWatch()
-	defer cancel()
-
 	if isTwoStepConfigAuthOnce(cfg) {
 		// Now that we are authenticated, we can set/reset any of the
 		// settings that we need to.
 		if err := tailscaleSet(ctx, cfg); err != nil {
-			log.Fatalf("failed to auth tailscale: %v", err)
+			return fmt.Errorf("failed to auth tailscale: %w", err)
 		}
 	}
 
@@ -329,11 +361,11 @@ authLoop:
 	if cfg.ServeConfigPath != "" {
 		log.Printf("serve proxy: unsetting previous config")
 		if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil {
-			log.Fatalf("failed to unset serve config: %v", err)
+			return fmt.Errorf("failed to unset serve config: %w", err)
 		}
 		if hasKubeStateStore(cfg) {
 			if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil {
-				log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err)
+				return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err)
 			}
 		}
 	}
@@ -344,19 +376,19 @@ authLoop:
 		// wipe it, but it's good hygiene.
 		log.Printf("Deleting authkey from kube secret")
 		if err := kc.deleteAuthKey(ctx); err != nil {
-			log.Fatalf("deleting authkey from kube secret: %v", err)
+			return fmt.Errorf("deleting authkey from kube secret: %w", err)
 		}
 	}
 
 	if hasKubeStateStore(cfg) {
 		if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil {
-			log.Fatalf("storing capability version and UID: %v", err)
+			return fmt.Errorf("storing capability version and UID: %w", err)
 		}
 	}
 
 	w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 	if err != nil {
-		log.Fatalf("rewatching tailscaled for updates after auth: %v", err)
+		return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err)
 	}
 
 	// If tailscaled config was read from a mounted file, watch the file for updates and reload.
@@ -386,7 +418,7 @@ authLoop:
 	if isL3Proxy(cfg) {
 		nfr, err = newNetfilterRunner(log.Printf)
 		if err != nil {
-			log.Fatalf("error creating new netfilter runner: %v", err)
+			return fmt.Errorf("error creating new netfilter runner: %w", err)
 		}
 	}
 
@@ -457,9 +489,9 @@ runLoop:
 			killTailscaled()
 			break runLoop
 		case err := <-errChan:
-			log.Fatalf("failed to read from tailscaled: %v", err)
+			return fmt.Errorf("failed to read from tailscaled: %w", err)
 		case err := <-cfgWatchErrChan:
-			log.Fatalf("failed to watch tailscaled config: %v", err)
+			return fmt.Errorf("failed to watch tailscaled config: %w", err)
 		case n := <-notifyChan:
 			if n.State != nil && *n.State != ipn.Running {
 				// Something's gone wrong and we've left the authenticated state.
@@ -467,7 +499,7 @@ runLoop:
 				// control flow required to make it work now is hard. So, just crash
 				// the container and rely on the container runtime to restart us,
 				// whereupon we'll go through initial auth again.
-				log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
+				return fmt.Errorf("tailscaled left running state (now in state %q), exiting", *n.State)
 			}
 			if n.NetMap != nil {
 				addrs = n.NetMap.SelfNode.Addresses().AsSlice()
@@ -485,7 +517,7 @@ runLoop:
 				deviceID := n.NetMap.SelfNode.StableID()
 				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceID, &deviceID) {
 					if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil {
-						log.Fatalf("storing device ID in Kubernetes Secret: %v", err)
+						return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err)
 					}
 				}
 				if cfg.TailnetTargetFQDN != "" {
@@ -522,12 +554,12 @@ runLoop:
 								rulesInstalled = true
 								log.Printf("Installing forwarding rules for destination %v", ea.String())
 								if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil {
-									log.Fatalf("installing egress proxy rules for destination %s: %v", ea.String(), err)
+									return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err)
 								}
 							}
 						}
 						if !rulesInstalled {
-							log.Fatalf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT())
+							return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT())
 						}
 					}
 					currentEgressIPs = newCurentEgressIPs
@@ -535,7 +567,7 @@ runLoop:
 				if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged {
 					log.Printf("Installing proxy rules")
 					if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil {
-						log.Fatalf("installing ingress proxy rules: %v", err)
+						return fmt.Errorf("installing ingress proxy rules: %w", err)
 					}
 				}
 				if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged {
@@ -551,7 +583,7 @@ runLoop:
 					if backendsHaveChanged {
 						log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs)
 						if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil {
-							log.Fatalf("error installing ingress proxy rules: %v", err)
+							return fmt.Errorf("error installing ingress proxy rules: %w", err)
 						}
 					}
 					resetTimer(false)
@@ -573,7 +605,7 @@ runLoop:
 				if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 {
 					log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP)
 					if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil {
-						log.Fatalf("installing egress proxy rules: %v", err)
+						return fmt.Errorf("installing egress proxy rules: %w", err)
 					}
 				}
 				// If this is a L7 cluster ingress proxy (set up
@@ -585,7 +617,7 @@ runLoop:
 				if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 {
 					log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP)
 					if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil {
-						log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err)
+						return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err)
 					}
 				}
 				currentIPs = newCurrentIPs
@@ -604,7 +636,7 @@ runLoop:
 				deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()}
 				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceEndpoints, &deviceEndpoints) {
 					if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
-						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
+						return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err)
 					}
 				}
 
@@ -614,7 +646,7 @@ runLoop:
 
 				if cfg.ServeConfigPath != "" {
 					triggerWatchServeConfigChanges.Do(func() {
-						go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc)
+						go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg)
 					})
 				}
 
@@ -639,20 +671,21 @@ runLoop:
 					// will then continuously monitor the config file and netmap updates and
 					// reconfigure the firewall rules as needed. If any of its operations fail, it
 					// will crash this node.
-					if cfg.EgressSvcsCfgPath != "" {
-						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressSvcsCfgPath)
+					if cfg.EgressProxiesCfgPath != "" {
+						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath)
 						egressSvcsNotify = make(chan ipn.Notify)
-						ep := egressProxy{
-							cfgPath:      cfg.EgressSvcsCfgPath,
+						opts := egressProxyRunOpts{
+							cfgPath:      cfg.EgressProxiesCfgPath,
 							nfr:          nfr,
 							kc:           kc,
+							tsClient:     client,
 							stateSecret:  cfg.KubeSecret,
 							netmapChan:   egressSvcsNotify,
 							podIPv4:      cfg.PodIPv4,
 							tailnetAddrs: addrs,
 						}
 						go func() {
-							if err := ep.run(ctx, n); err != nil {
+							if err := ep.run(ctx, n, opts); err != nil {
 								egressSvcsErrorChan <- err
 							}
 						}()
@@ -694,16 +727,18 @@ runLoop:
 			if backendsHaveChanged && len(addrs) != 0 {
 				log.Printf("Backend address change detected, installing proxy rules for backends %v", newBackendAddrs)
 				if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil {
-					log.Fatalf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err)
+					return fmt.Errorf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err)
 				}
 			}
 			backendAddrs = newBackendAddrs
 			resetTimer(false)
 		case e := <-egressSvcsErrorChan:
-			log.Fatalf("egress proxy failed: %v", e)
+			return fmt.Errorf("egress proxy failed: %v", e)
 		}
 	}
 	wg.Wait()
+
+	return nil
 }
 
 // ensureTunFile checks that /dev/net/tun exists, creating it if
@@ -732,13 +767,13 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	ip4s, err := net.DefaultResolver.LookupIP(ctx, "ip4", name)
 	if err != nil {
 		if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) {
-			return nil, fmt.Errorf("error looking up IPv4 addresses: %v", err)
+			return nil, fmt.Errorf("error looking up IPv4 addresses: %w", err)
 		}
 	}
 	ip6s, err := net.DefaultResolver.LookupIP(ctx, "ip6", name)
 	if err != nil {
 		if e, ok := err.(*net.DNSError); !(ok && e.IsNotFound) {
-			return nil, fmt.Errorf("error looking up IPv6 addresses: %v", err)
+			return nil, fmt.Errorf("error looking up IPv6 addresses: %w", err)
 		}
 	}
 	if len(ip4s) == 0 && len(ip6s) == 0 {
@@ -751,7 +786,7 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
 func contextWithExitSignalWatch() (context.Context, func()) {
-	closeChan := make(chan string)
+	closeChan := make(chan struct{})
 	ctx, cancel := context.WithCancel(context.Background())
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
@@ -763,8 +798,11 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 			return
 		}
 	}()
+	closeOnce := sync.Once{}
 	f := func() {
-		closeChan <- "goodbye"
+		closeOnce.Do(func() {
+			close(closeChan)
+		})
 	}
 	return ctx, f
 }
@@ -817,7 +855,11 @@ func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) {
 
 	go func() {
 		if err := srv.Serve(ln); err != nil {
-			log.Fatalf("failed running server: %v", err)
+			if err != http.ErrServerClosed {
+				log.Fatalf("failed running server: %v", err)
+			} else {
+				log.Printf("HTTP server at %s closed", addr)
+			}
 		}
 	}()
 

cmd/containerboot/main_test.go

@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -32,6 +33,8 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/netmap"
@@ -48,26 +51,13 @@ func TestContainerBoot(t *testing.T) {
 	defer lapi.Close()
 
 	kube := kubeServer{FSRoot: d}
-	if err := kube.Start(); err != nil {
-		t.Fatal(err)
-	}
+	kube.Start(t)
 	defer kube.Close()
 
 	tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"}
-	tailscaledConfBytes, err := json.Marshal(tailscaledConf)
-	if err != nil {
-		t.Fatalf("error unmarshaling tailscaled config: %v", err)
-	}
 	serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}}
-	serveConfBytes, err := json.Marshal(serveConf)
-	if err != nil {
-		t.Fatalf("error unmarshaling serve config: %v", err)
-	}
-	egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}}
-	egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg)
-	if err != nil {
-		t.Fatalf("error unmarshaling egress services config: %v", err)
-	}
+	egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net")
+	egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net")
 
 	dirs := []string{
 		"var/lib",
@@ -84,16 +74,17 @@ func TestContainerBoot(t *testing.T) {
 		}
 	}
 	files := map[string][]byte{
-		"usr/bin/tailscaled":                         fakeTailscaled,
-		"usr/bin/tailscale":                          fakeTailscale,
-		"usr/bin/iptables":                           fakeTailscale,
-		"usr/bin/ip6tables":                          fakeTailscale,
-		"dev/net/tun":                                []byte(""),
-		"proc/sys/net/ipv4/ip_forward":               []byte("0"),
-		"proc/sys/net/ipv6/conf/all/forwarding":      []byte("0"),
-		"etc/tailscaled/cap-95.hujson":               tailscaledConfBytes,
-		"etc/tailscaled/serve-config.json":           serveConfBytes,
-		"etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes,
+		"usr/bin/tailscaled":                    fakeTailscaled,
+		"usr/bin/tailscale":                     fakeTailscale,
+		"usr/bin/iptables":                      fakeTailscale,
+		"usr/bin/ip6tables":                     fakeTailscale,
+		"dev/net/tun":                           []byte(""),
+		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
+		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
+		"etc/tailscaled/cap-95.hujson":          mustJSON(t, tailscaledConf),
+		"etc/tailscaled/serve-config.json":      mustJSON(t, serveConf),
+		filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg),
+		filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings):       []byte("4"),
 	}
 	resetFiles := func() {
 		for path, content := range files {
@@ -132,6 +123,9 @@ func TestContainerBoot(t *testing.T) {
 	healthURL := func(port int) string {
 		return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
 	}
+	egressSvcTerminateURL := func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d%s", port, kubetypes.EgessServicesPreshutdownEP)
+	}
 
 	capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
 
@@ -143,15 +137,29 @@ func TestContainerBoot(t *testing.T) {
 
 		// WantCmds is the commands that containerboot should run in this phase.
 		WantCmds []string
+
 		// WantKubeSecret is the secret keys/values that should exist in the
 		// kube secret.
 		WantKubeSecret map[string]string
+
+		// Update the kube secret with these keys/values at the beginning of the
+		// phase (simulates our fake tailscaled doing it).
+		UpdateKubeSecret map[string]string
+
 		// WantFiles files that should exist in the container and their
 		// contents.
 		WantFiles map[string]string
-		// WantFatalLog is the fatal log message we expect from containerboot.
-		// If set for a phase, the test will finish on that phase.
-		WantFatalLog string
+
+		// WantLog is a log message we expect from containerboot.
+		WantLog string
+
+		// If set for a phase, the test will expect containerboot to exit with
+		// this error code, and the test will finish on that phase without
+		// waiting for the successful startup log message.
+		WantExitCode *int
+
+		// The signal to send to containerboot at the start of the phase.
+		Signal *syscall.Signal
 
 		EndpointStatuses map[string]int
 	}
@@ -439,7 +447,8 @@ func TestContainerBoot(t *testing.T) {
 							},
 						},
 					},
-					WantFatalLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false",
+					WantLog:      "no forwarding rules for egress addresses [::1/128], host supports IPv6: false",
+					WantExitCode: ptr.To(1),
 				},
 			},
 		},
@@ -896,9 +905,10 @@ func TestContainerBoot(t *testing.T) {
 		{
 			Name: "egress_svcs_config_kube",
 			Env: map[string]string{
-				"KUBERNETES_SERVICE_HOST":        kube.Host,
-				"KUBERNETES_SERVICE_PORT_HTTPS":  kube.Port,
-				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"),
+				"TS_LOCAL_ADDR_PORT":            fmt.Sprintf("[::]:%d", localAddrPort),
 			},
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
@@ -912,28 +922,92 @@ func TestContainerBoot(t *testing.T) {
 					WantKubeSecret: map[string]string{
 						"authkey": "tskey-key",
 					},
+					EndpointStatuses: map[string]int{
+						egressSvcTerminateURL(localAddrPort): 200,
+					},
 				},
 				{
 					Notify: runningNotify,
 					WantKubeSecret: map[string]string{
+						"egress-services":  mustBase64(t, egressStatus),
 						"authkey":          "tskey-key",
 						"device_fqdn":      "test-node.test.ts.net",
 						"device_id":        "myID",
 						"device_ips":       `["100.64.0.1"]`,
 						"tailscale_capver": capver,
 					},
+					EndpointStatuses: map[string]int{
+						egressSvcTerminateURL(localAddrPort): 200,
+					},
 				},
 			},
 		},
 		{
 			Name: "egress_svcs_config_no_kube",
 			Env: map[string]string{
-				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
-				"TS_AUTHKEY":                     "tskey-key",
+				"TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"),
+				"TS_AUTHKEY":                    "tskey-key",
 			},
 			Phases: []phase{
 				{
-					WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
+					WantLog:      "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
+					WantExitCode: ptr.To(1),
+				},
+			},
+		},
+		{
+			Name: "kube_shutdown_during_state_write",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_ENABLE_HEALTH_CHECK":        "true",
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			Phases: []phase{
+				{
+					// Normal startup.
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					// SIGTERM before state is finished writing, should wait for
+					// consistent state before propagating SIGTERM to tailscaled.
+					Signal: ptr.To(unix.SIGTERM),
+					UpdateKubeSecret: map[string]string{
+						"_machinekey":  "foo",
+						"_profiles":    "foo",
+						"profile-baff": "foo",
+						// Missing "_current-profile" key.
+					},
+					WantKubeSecret: map[string]string{
+						"authkey":      "tskey-key",
+						"_machinekey":  "foo",
+						"_profiles":    "foo",
+						"profile-baff": "foo",
+					},
+					WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"",
+				},
+				{
+					// tailscaled has finished writing state, should propagate SIGTERM.
+					UpdateKubeSecret: map[string]string{
+						"_current-profile": "foo",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey":          "tskey-key",
+						"_machinekey":      "foo",
+						"_profiles":        "foo",
+						"profile-baff":     "foo",
+						"_current-profile": "foo",
+					},
+					WantLog:      "HTTP server at [::]:9002 closed",
+					WantExitCode: ptr.To(0),
 				},
 			},
 		},
@@ -981,26 +1055,36 @@ func TestContainerBoot(t *testing.T) {
 
 			var wantCmds []string
 			for i, p := range test.Phases {
+				for k, v := range p.UpdateKubeSecret {
+					kube.SetSecret(k, v)
+				}
 				lapi.Notify(p.Notify)
-				if p.WantFatalLog != "" {
+				if p.Signal != nil {
+					cmd.Process.Signal(*p.Signal)
+				}
+				if p.WantLog != "" {
 					err := tstest.WaitFor(2*time.Second, func() error {
-						state, err := cmd.Process.Wait()
-						if err != nil {
-							return err
-						}
-						if state.ExitCode() != 1 {
-							return fmt.Errorf("process exited with code %d but wanted %d", state.ExitCode(), 1)
-						}
-						waitLogLine(t, time.Second, cbOut, p.WantFatalLog)
+						waitLogLine(t, time.Second, cbOut, p.WantLog)
 						return nil
 					})
 					if err != nil {
 						t.Fatal(err)
 					}
+				}
+
+				if p.WantExitCode != nil {
+					state, err := cmd.Process.Wait()
+					if err != nil {
+						t.Fatal(err)
+					}
+					if state.ExitCode() != *p.WantExitCode {
+						t.Fatalf("phase %d: want exit code %d, got %d", i, *p.WantExitCode, state.ExitCode())
+					}
 
 					// Early test return, we don't expect the successful startup log message.
 					return
 				}
+
 				wantCmds = append(wantCmds, p.WantCmds...)
 				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
 				err := tstest.WaitFor(2*time.Second, func() error {
@@ -1056,6 +1140,9 @@ func TestContainerBoot(t *testing.T) {
 				}
 			}
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
+			if cmd.ProcessState != nil {
+				t.Fatalf("containerboot should be running but exited with exit code %d", cmd.ProcessState.ExitCode())
+			}
 		})
 	}
 }
@@ -1287,18 +1374,18 @@ func (k *kubeServer) Reset() {
 	k.secret = map[string]string{}
 }
 
-func (k *kubeServer) Start() error {
+func (k *kubeServer) Start(t *testing.T) {
 	root := filepath.Join(k.FSRoot, "var/run/secrets/kubernetes.io/serviceaccount")
 
 	if err := os.MkdirAll(root, 0700); err != nil {
-		return err
+		t.Fatal(err)
 	}
 
 	if err := os.WriteFile(filepath.Join(root, "namespace"), []byte("default"), 0600); err != nil {
-		return err
+		t.Fatal(err)
 	}
 	if err := os.WriteFile(filepath.Join(root, "token"), []byte("bearer_token"), 0600); err != nil {
-		return err
+		t.Fatal(err)
 	}
 
 	k.srv = httptest.NewTLSServer(k)
@@ -1307,13 +1394,11 @@ func (k *kubeServer) Start() error {
 
 	var cert bytes.Buffer
 	if err := pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: k.srv.Certificate().Raw}); err != nil {
-		return err
+		t.Fatal(err)
 	}
 	if err := os.WriteFile(filepath.Join(root, "ca.crt"), cert.Bytes(), 0600); err != nil {
-		return err
+		t.Fatal(err)
 	}
-
-	return nil
 }
 
 func (k *kubeServer) Close() {
@@ -1362,6 +1447,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, fmt.Sprintf("reading request body: %v", err), http.StatusInternalServerError)
 		return
 	}
+	defer r.Body.Close()
 
 	switch r.Method {
 	case "GET":
@@ -1394,13 +1480,32 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 				panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
 			}
 			for _, op := range req {
-				if op.Op != "remove" {
+				switch op.Op {
+				case "remove":
+					if !strings.HasPrefix(op.Path, "/data/") {
+						panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
+					}
+					delete(k.secret, strings.TrimPrefix(op.Path, "/data/"))
+				case "replace":
+					path, ok := strings.CutPrefix(op.Path, "/data/")
+					if !ok {
+						panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
+					}
+					req := make([]kubeclient.JSONPatch, 0)
+					if err := json.Unmarshal(bs, &req); err != nil {
+						panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
+					}
+
+					for _, patch := range req {
+						val, ok := patch.Value.(string)
+						if !ok {
+							panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value))
+						}
+						k.secret[path] = val
+					}
+				default:
 					panic(fmt.Sprintf("unsupported json-patch op %q", op.Op))
 				}
-				if !strings.HasPrefix(op.Path, "/data/") {
-					panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
-				}
-				delete(k.secret, strings.TrimPrefix(op.Path, "/data/"))
 			}
 		case "application/strategic-merge-patch+json":
 			req := struct {
@@ -1416,6 +1521,44 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 			panic(fmt.Sprintf("unknown content type %q", r.Header.Get("Content-Type")))
 		}
 	default:
-		panic(fmt.Sprintf("unhandled HTTP method %q", r.Method))
+		panic(fmt.Sprintf("unhandled HTTP request %s %s", r.Method, r.URL))
+	}
+}
+
+func mustBase64(t *testing.T, v any) string {
+	b := mustJSON(t, v)
+	s := base64.StdEncoding.WithPadding('=').EncodeToString(b)
+	return s
+}
+
+func mustJSON(t *testing.T, v any) []byte {
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("error converting %v to json: %v", v, err)
+	}
+	return b
+}
+
+// egress services status given one named tailnet target specified by FQDN. As written by the proxy to its state Secret.
+func egressSvcStatus(name, fqdn string) egressservices.Status {
+	return egressservices.Status{
+		Services: map[string]*egressservices.ServiceStatus{
+			name: {
+				TailnetTarget: egressservices.TailnetTarget{
+					FQDN: fqdn,
+				},
+			},
+		},
+	}
+}
+
+// egress config given one named tailnet target specified by FQDN.
+func egressSvcConfig(name, fqdn string) egressservices.Configs {
+	return egressservices.Configs{
+		name: egressservices.Config{
+			TailnetTarget: egressservices.TailnetTarget{
+				FQDN: fqdn,
+			},
+		},
 	}
 }

cmd/containerboot/metrics.go

@@ -10,7 +10,7 @@ import (
 	"io"
 	"net/http"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 )
 
@@ -18,7 +18,7 @@ import (
 // the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
 type metrics struct {
 	debugEndpoint string
-	lc            *tailscale.LocalClient
+	lc            *local.Client
 }
 
 func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
@@ -68,7 +68,7 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
 // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
 // endpoint if configured to ease migration for a breaking change serving user
 // metrics instead of debug metrics on the "metrics" port.
-func metricsHandlers(mux *http.ServeMux, lc *tailscale.LocalClient, debugAddrPort string) {
+func metricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) {
 	m := &metrics{
 		lc:            lc,
 		debugEndpoint: debugAddrPort,

cmd/containerboot/serve.go

@@ -17,7 +17,7 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/netmap"
@@ -28,20 +28,23 @@ import (
 // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
 // is written to when the certDomain changes, causing the serve config to be
 // re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient, kc *kubeClient) {
+func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings) {
 	if certDomainAtomic == nil {
 		panic("certDomainAtomic must not be nil")
 	}
+
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	if w, err := fsnotify.NewWatcher(); err != nil {
+		// Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor.
+		// See https://github.com/tailscale/tailscale/issues/15081
 		log.Printf("serve proxy: failed to create fsnotify watcher, timer-only mode: %v", err)
 		ticker := time.NewTicker(5 * time.Second)
 		defer ticker.Stop()
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
+		if err := w.Add(filepath.Dir(cfg.ServeConfigPath)); err != nil {
 			log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
 		}
 		eventChan = w.Events
@@ -49,6 +52,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 
 	var certDomain string
 	var prevServeConfig *ipn.ServeConfig
+	var cm certManager
+	if cfg.CertShareMode == "rw" {
+		cm = certManager{
+			lc: lc,
+		}
+	}
 	for {
 		select {
 		case <-ctx.Done():
@@ -61,12 +70,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			// k8s handles these mounts. So just re-read the file and apply it
 			// if it's changed.
 		}
-		sc, err := readServeConfig(path, certDomain)
+		sc, err := readServeConfig(cfg.ServeConfigPath, certDomain)
 		if err != nil {
 			log.Fatalf("serve proxy: failed to read serve config: %v", err)
 		}
 		if sc == nil {
-			log.Printf("serve proxy: no serve config at %q, skipping", path)
+			log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath)
 			continue
 		}
 		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
@@ -81,6 +90,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			}
 		}
 		prevServeConfig = sc
+		if cfg.CertShareMode != "rw" {
+			continue
+		}
+		if err := cm.ensureCertLoops(ctx, sc); err != nil {
+			log.Fatalf("serve proxy: error ensuring cert loops: %v", err)
+		}
 	}
 }
 
@@ -91,9 +106,10 @@ func certDomainFromNetmap(nm *netmap.NetworkMap) string {
 	return nm.DNS.CertDomains[0]
 }
 
-// localClient is a subset of tailscale.LocalClient that can be mocked for testing.
+// localClient is a subset of [local.Client] that can be mocked for testing.
 type localClient interface {
 	SetServeConfig(context.Context, *ipn.ServeConfig) error
+	CertPair(context.Context, string) ([]byte, []byte, error)
 }
 
 func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error {

cmd/containerboot/serve_test.go

@@ -12,7 +12,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/kubetypes"
 )
@@ -197,7 +197,7 @@ func TestReadServeConfig(t *testing.T) {
 }
 
 type fakeLocalClient struct {
-	*tailscale.LocalClient
+	*local.Client
 	setServeCalled bool
 }
 
@@ -206,6 +206,10 @@ func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConf
 	return nil
 }
 
+func (m *fakeLocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+	return nil, nil, nil
+}
+
 func TestHasHTTPSEndpoint(t *testing.T) {
 	tests := []struct {
 		name string

cmd/containerboot/services.go

@@ -11,18 +11,24 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net/http"
 	"net/netip"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/local"
 	"tailscale.com/ipn"
 	"tailscale.com/kube/egressservices"
 	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/httpm"
 	"tailscale.com/util/linuxfw"
 	"tailscale.com/util/mak"
 )
@@ -37,13 +43,15 @@ const tailscaleTunInterface = "tailscale0"
 // egressProxy knows how to configure firewall rules to route cluster traffic to
 // one or more tailnet services.
 type egressProxy struct {
-	cfgPath string // path to egress service config file
+	cfgPath string // path to a directory with egress services config files
 
 	nfr linuxfw.NetfilterRunner // never nil
 
 	kc          kubeclient.Client // never nil
 	stateSecret string            // name of the kube state Secret
 
+	tsClient *local.Client // never nil
+
 	netmapChan chan ipn.Notify // chan to receive netmap updates on
 
 	podIPv4 string // never empty string, currently only IPv4 is supported
@@ -55,15 +63,29 @@ type egressProxy struct {
 	// memory at all.
 	targetFQDNs map[string][]netip.Prefix
 
-	// used to configure firewall rules.
-	tailnetAddrs []netip.Prefix
+	tailnetAddrs []netip.Prefix // tailnet IPs of this tailnet device
+
+	// shortSleep is the backoff sleep between healthcheck endpoint calls - can be overridden in tests.
+	shortSleep time.Duration
+	// longSleep is the time to sleep after the routing rules are updated to increase the chance that kube
+	// proxies on all nodes have updated their routing configuration. It can be configured to 0 in
+	// tests.
+	longSleep time.Duration
+	// client is a client that can send HTTP requests.
+	client httpClient
+}
+
+// httpClient is a client that can send HTTP requests and can be mocked in tests.
+type httpClient interface {
+	Do(*http.Request) (*http.Response, error)
 }
 
 // run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when:
 // - the mounted egress config has changed
 // - the proxy's tailnet IP addresses have changed
 // - tailnet IPs have changed for any backend targets specified by tailnet FQDN
-func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
+func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRunOpts) error {
+	ep.configure(opts)
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	// TODO (irbekrm): take a look if this can be pulled into a single func
@@ -75,7 +97,7 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
-		if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil {
+		if err := w.Add(ep.cfgPath); err != nil {
 			return fmt.Errorf("failed to add fsnotify watch: %w", err)
 		}
 		eventChan = w.Events
@@ -85,28 +107,52 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
 		return err
 	}
 	for {
-		var err error
 		select {
 		case <-ctx.Done():
 			return nil
 		case <-tickChan:
-			err = ep.sync(ctx, n)
+			log.Printf("periodic sync, ensuring firewall config is up to date...")
 		case <-eventChan:
 			log.Printf("config file change detected, ensuring firewall config is up to date...")
-			err = ep.sync(ctx, n)
 		case n = <-ep.netmapChan:
 			shouldResync := ep.shouldResync(n)
-			if shouldResync {
-				log.Printf("netmap change detected, ensuring firewall config is up to date...")
-				err = ep.sync(ctx, n)
+			if !shouldResync {
+				continue
 			}
+			log.Printf("netmap change detected, ensuring firewall config is up to date...")
 		}
-		if err != nil {
+		if err := ep.sync(ctx, n); err != nil {
 			return fmt.Errorf("error syncing egress service config: %w", err)
 		}
 	}
 }
 
+type egressProxyRunOpts struct {
+	cfgPath      string
+	nfr          linuxfw.NetfilterRunner
+	kc           kubeclient.Client
+	tsClient     *local.Client
+	stateSecret  string
+	netmapChan   chan ipn.Notify
+	podIPv4      string
+	tailnetAddrs []netip.Prefix
+}
+
+// applyOpts configures egress proxy using the provided options.
+func (ep *egressProxy) configure(opts egressProxyRunOpts) {
+	ep.cfgPath = opts.cfgPath
+	ep.nfr = opts.nfr
+	ep.kc = opts.kc
+	ep.tsClient = opts.tsClient
+	ep.stateSecret = opts.stateSecret
+	ep.netmapChan = opts.netmapChan
+	ep.podIPv4 = opts.podIPv4
+	ep.tailnetAddrs = opts.tailnetAddrs
+	ep.client = &http.Client{} // default HTTP client
+	ep.shortSleep = time.Second
+	ep.longSleep = time.Second * 10
+}
+
 // sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if
 // any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current
 // firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such
@@ -327,7 +373,8 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s
 
 // getConfigs gets the mounted egress service configuration.
 func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) {
-	j, err := os.ReadFile(ep.cfgPath)
+	svcsCfg := filepath.Join(ep.cfgPath, egressservices.KeyEgressServices)
+	j, err := os.ReadFile(svcsCfg)
 	if os.IsNotExist(err) {
 		return nil, nil
 	}
@@ -569,3 +616,142 @@ func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
 	st1.PodIPv4 = ""
 	return reflect.DeepEqual(*st, *st1)
 }
+
+// registerHandlers adds a new handler to the provided ServeMux that can be called as a Kubernetes prestop hook to
+// delay shutdown till it's safe to do so.
+func (ep *egressProxy) registerHandlers(mux *http.ServeMux) {
+	mux.Handle(fmt.Sprintf("GET %s", kubetypes.EgessServicesPreshutdownEP), ep)
+}
+
+// ServeHTTP serves /internal-egress-services-preshutdown endpoint, when it receives a request, it periodically polls
+// the configured health check endpoint for each egress service till it the health check endpoint no longer hits this
+// proxy Pod. It uses the Pod-IPv4 header to verify if health check response is received from this Pod.
+func (ep *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	cfgs, err := ep.getConfigs()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("error retrieving egress services configs: %v", err), http.StatusInternalServerError)
+		return
+	}
+	if cfgs == nil {
+		if _, err := w.Write([]byte("safe to terminate")); err != nil {
+			http.Error(w, fmt.Sprintf("error writing termination status: %v", err), http.StatusInternalServerError)
+			return
+		}
+	}
+	hp, err := ep.getHEPPings()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("error determining the number of times health check endpoint should be pinged: %v", err), http.StatusInternalServerError)
+		return
+	}
+	ep.waitTillSafeToShutdown(r.Context(), cfgs, hp)
+}
+
+// waitTillSafeToShutdown looks up all egress targets configured to be proxied via this instance and, for each target
+// whose configuration includes a healthcheck endpoint, pings the endpoint till none of the responses
+// are returned by this instance or till the HTTP request times out. In practice, the endpoint will be a Kubernetes Service for whom one of the backends
+// would normally be this Pod. When this Pod is being deleted, the operator should have removed it from the Service
+// backends and eventually kube proxy routing rules should be updated to no longer route traffic for the Service to this
+// Pod.
+func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs *egressservices.Configs, hp int) {
+	if cfgs == nil || len(*cfgs) == 0 { // avoid sleeping if no services are configured
+		return
+	}
+	log.Printf("Ensuring that cluster traffic for egress targets is no longer routed via this Pod...")
+	wg := syncs.WaitGroup{}
+
+	for s, cfg := range *cfgs {
+		hep := cfg.HealthCheckEndpoint
+		if hep == "" {
+			log.Printf("Tailnet target %q does not have a cluster healthcheck specified, unable to verify if cluster traffic for the target is still routed via this Pod", s)
+			continue
+		}
+		svc := s
+		wg.Go(func() {
+			log.Printf("Ensuring that cluster traffic is no longer routed to %q via this Pod...", svc)
+			for {
+				if ctx.Err() != nil { // kubelet's HTTP request timeout
+					log.Printf("Cluster traffic for %s did not stop being routed to this Pod.", svc)
+					return
+				}
+				found, err := lookupPodRoute(ctx, hep, ep.podIPv4, hp, ep.client)
+				if err != nil {
+					log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err)
+					break
+				}
+				if !found {
+					log.Printf("service %q is no longer routed through this Pod", svc)
+					break
+				}
+				log.Printf("service %q is still routed through this Pod, waiting...", svc)
+				time.Sleep(ep.shortSleep)
+			}
+		})
+	}
+	wg.Wait()
+	// The check above really only checked that the routing rules are updated on this node. Sleep for a bit to
+	// ensure that the routing rules are updated on other nodes. TODO(irbekrm): this may or may not be good enough.
+	// If it's not good enough, we'd probably want to do something more complex, where the proxies check each other.
+	log.Printf("Sleeping for %s before shutdown to ensure that kube proxies on all nodes have updated routing configuration", ep.longSleep)
+	time.Sleep(ep.longSleep)
+}
+
+// lookupPodRoute calls the healthcheck endpoint repeat times and returns true if the endpoint returns with the podIP
+// header at least once.
+func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client httpClient) (bool, error) {
+	for range repeat {
+		f, err := lookup(ctx, hep, podIP, client)
+		if err != nil {
+			return false, err
+		}
+		if f {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// lookup calls the healthcheck endpoint and returns true if the response contains the podIP header.
+func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, hep, nil)
+	if err != nil {
+		return false, fmt.Errorf("error creating new HTTP request: %v", err)
+	}
+
+	// Close the TCP connection to ensure that the next request is routed to a different backend.
+	req.Close = true
+
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Printf("Endpoint %q can not be reached: %v, likely because there are no (more) healthy backends", hep, err)
+		return true, nil
+	}
+	defer resp.Body.Close()
+	gotIP := resp.Header.Get(kubetypes.PodIPv4Header)
+	return strings.EqualFold(podIP, gotIP), nil
+}
+
+// getHEPPings gets the number of pings that should be sent to a health check endpoint to ensure that each configured
+// backend is hit. This assumes that a health check endpoint is a Kubernetes Service and traffic to backend Pods is
+// round robin load balanced.
+func (ep *egressProxy) getHEPPings() (int, error) {
+	hepPingsPath := filepath.Join(ep.cfgPath, egressservices.KeyHEPPings)
+	j, err := os.ReadFile(hepPingsPath)
+	if os.IsNotExist(err) {
+		return 0, nil
+	}
+	if err != nil {
+		return -1, err
+	}
+	if len(j) == 0 || string(j) == "" {
+		return 0, nil
+	}
+	hp, err := strconv.Atoi(string(j))
+	if err != nil {
+		return -1, fmt.Errorf("error parsing hep pings as int: %v", err)
+	}
+	if hp < 0 {
+		log.Printf("[unexpected] hep pings is negative: %d", hp)
+		return 0, nil
+	}
+	return hp, nil
+}

cmd/containerboot/services_test.go

@@ -6,11 +6,18 @@
 package main
 
 import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
 	"net/netip"
 	"reflect"
+	"strings"
+	"sync"
 	"testing"
 
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
 )
 
 func Test_updatesForSvc(t *testing.T) {
@@ -173,3 +180,145 @@ func Test_updatesForSvc(t *testing.T) {
 		})
 	}
 }
+
+// A failure of this test will most likely look like a timeout.
+func TestWaitTillSafeToShutdown(t *testing.T) {
+	podIP := "10.0.0.1"
+	anotherIP := "10.0.0.2"
+
+	tests := []struct {
+		name string
+		// services is a map of service name to the number of calls to make to the healthcheck endpoint before
+		// returning a response that does NOT contain this Pod's IP in headers.
+		services       map[string]int
+		replicas       int
+		healthCheckSet bool
+	}{
+		{
+			name: "no_configs",
+		},
+		{
+			name: "one_service_immediately_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 0,
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_immediately_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 0,
+				"svc2": 0,
+				"svc3": 0,
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_no_healthcheck_endpoints",
+			services: map[string]int{
+				"svc1": 0,
+				"svc2": 0,
+				"svc3": 0,
+			},
+			replicas: 2,
+		},
+		{
+			name: "one_service_eventually_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_eventually_safe_to_shutdown",
+			services: map[string]int{
+				"svc1": 1, // After 1 call to health check endpoint, no longer returns this Pod's IP
+				"svc2": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP
+				"svc3": 5, // After 5 calls to the health check endpoint, no longer returns this Pod's IP
+			},
+			replicas:       2,
+			healthCheckSet: true,
+		},
+		{
+			name: "multiple_services_eventually_safe_to_shutdown_with_higher_replica_count",
+			services: map[string]int{
+				"svc1": 7,
+				"svc2": 10,
+			},
+			replicas:       5,
+			healthCheckSet: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfgs := &egressservices.Configs{}
+			switches := make(map[string]int)
+
+			for svc, callsToSwitch := range tt.services {
+				endpoint := fmt.Sprintf("http://%s.local", svc)
+				if tt.healthCheckSet {
+					(*cfgs)[svc] = egressservices.Config{
+						HealthCheckEndpoint: endpoint,
+					}
+				}
+				switches[endpoint] = callsToSwitch
+			}
+
+			ep := &egressProxy{
+				podIPv4: podIP,
+				client: &mockHTTPClient{
+					podIP:     podIP,
+					anotherIP: anotherIP,
+					switches:  switches,
+				},
+			}
+
+			ep.waitTillSafeToShutdown(context.Background(), cfgs, tt.replicas)
+		})
+	}
+}
+
+// mockHTTPClient is a client that receives an HTTP call for an egress service endpoint and returns a response with an
+// IP address in a 'Pod-IPv4' header. It can be configured to return one IP address for N calls, then switch to another
+// IP address to simulate a scenario where an IP is eventually no longer a backend for an endpoint.
+// TODO(irbekrm): to test this more thoroughly, we should have the client take into account the number of replicas and
+// return as if traffic was round robin load balanced across different Pods.
+type mockHTTPClient struct {
+	// podIP - initial IP address to return, that matches the current proxy's IP address.
+	podIP     string
+	anotherIP string
+	// after how many calls to an endpoint, the client should start returning 'anotherIP' instead of 'podIP.
+	switches map[string]int
+	mu       sync.Mutex // protects the following
+	// calls tracks the number of calls received.
+	calls map[string]int
+}
+
+func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	m.mu.Lock()
+	if m.calls == nil {
+		m.calls = make(map[string]int)
+	}
+
+	endpoint := req.URL.String()
+	m.calls[endpoint]++
+	calls := m.calls[endpoint]
+	m.mu.Unlock()
+
+	resp := &http.Response{
+		StatusCode: http.StatusOK,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader("")),
+	}
+
+	if calls <= m.switches[endpoint] {
+		resp.Header.Set(kubetypes.PodIPv4Header, m.podIP) // Pod is still routable
+	} else {
+		resp.Header.Set(kubetypes.PodIPv4Header, m.anotherIP) // Pod is no longer routable
+	}
+	return resp, nil
+}

cmd/containerboot/settings.go

@@ -64,16 +64,22 @@ type settings struct {
 	// when setting up rules to proxy cluster traffic to cluster ingress
 	// target.
 	// Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters
-	PodIP               string
-	PodIPv4             string
-	PodIPv6             string
-	PodUID              string
-	HealthCheckAddrPort string
-	LocalAddrPort       string
-	MetricsEnabled      bool
-	HealthCheckEnabled  bool
-	DebugAddrPort       string
-	EgressSvcsCfgPath   string
+	PodIP                string
+	PodIPv4              string
+	PodIPv6              string
+	PodUID               string
+	HealthCheckAddrPort  string
+	LocalAddrPort        string
+	MetricsEnabled       bool
+	HealthCheckEnabled   bool
+	DebugAddrPort        string
+	EgressProxiesCfgPath string
+	// CertShareMode is set for Kubernetes Pods running cert share mode.
+	// Possible values are empty (containerboot doesn't run any certs
+	// logic),  'ro' (for Pods that shold never attempt to issue/renew
+	// certs) and 'rw' for Pods that should manage the TLS certs shared
+	// amongst the replicas.
+	CertShareMode string
 }
 
 func configFromEnv() (*settings, error) {
@@ -107,7 +113,7 @@ func configFromEnv() (*settings, error) {
 		MetricsEnabled:                        defaultBool("TS_ENABLE_METRICS", false),
 		HealthCheckEnabled:                    defaultBool("TS_ENABLE_HEALTH_CHECK", false),
 		DebugAddrPort:                         defaultEnv("TS_DEBUG_ADDR_PORT", ""),
-		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
+		EgressProxiesCfgPath:                  defaultEnv("TS_EGRESS_PROXIES_CONFIG_PATH", ""),
 		PodUID:                                defaultEnv("POD_UID", ""),
 	}
 	podIPs, ok := os.LookupEnv("POD_IPS")
@@ -128,6 +134,17 @@ func configFromEnv() (*settings, error) {
 			cfg.PodIPv6 = parsed.String()
 		}
 	}
+	// If cert share is enabled, set the replica as read or write. Only 0th
+	// replica should be able to write.
+	isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false)
+	if isInCertShareMode {
+		cfg.CertShareMode = "ro"
+		podName := os.Getenv("POD_NAME")
+		if strings.HasSuffix(podName, "-0") {
+			cfg.CertShareMode = "rw"
+		}
+	}
+
 	if err := cfg.validate(); err != nil {
 		return nil, fmt.Errorf("invalid configuration: %v", err)
 	}
@@ -186,7 +203,7 @@ func (s *settings) validate() error {
 			return fmt.Errorf("error parsing TS_HEALTHCHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
 		}
 	}
-	if s.localMetricsEnabled() || s.localHealthEnabled() {
+	if s.localMetricsEnabled() || s.localHealthEnabled() || s.EgressProxiesCfgPath != "" {
 		if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
 			return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
 		}
@@ -199,8 +216,8 @@ func (s *settings) validate() error {
 	if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" {
 		return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT")
 	}
-	if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
-		return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
+	if s.EgressProxiesCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
+		return errors.New("TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
 	}
 	return nil
 }
@@ -291,7 +308,7 @@ func isOneStepConfig(cfg *settings) bool {
 // as an L3 proxy, proxying to an endpoint provided via one of the config env
 // vars.
 func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != ""
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressProxiesCfgPath != ""
 }
 
 // hasKubeStateStore returns true if the state must be stored in a Kubernetes
@@ -308,6 +325,10 @@ func (cfg *settings) localHealthEnabled() bool {
 	return cfg.LocalAddrPort != "" && cfg.HealthCheckEnabled
 }
 
+func (cfg *settings) egressSvcsTerminateEPEnabled() bool {
+	return cfg.LocalAddrPort != "" && cfg.EgressProxiesCfgPath != ""
+}
+
 // defaultEnv returns the value of the given envvar name, or defVal if
 // unset.
 func defaultEnv(name, defVal string) string {

cmd/containerboot/tailscaled.go

@@ -20,10 +20,10 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 )
 
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Process, error) {
 	args := tailscaledArgs(cfg)
 	// tailscaled runs without context, since it needs to persist
 	// beyond the startup timeout in ctx.
@@ -33,6 +33,9 @@ func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
+	if cfg.CertShareMode != "" {
+		cmd.Env = append(os.Environ(), "TS_CERT_SHARE_MODE="+cfg.CertShareMode)
+	}
 	log.Printf("Starting tailscaled")
 	if err := cmd.Start(); err != nil {
 		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
@@ -42,19 +45,19 @@ func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient
 	log.Printf("Waiting for tailscaled socket")
 	for {
 		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
+			return nil, nil, errors.New("timed out waiting for tailscaled socket")
 		}
 		_, err := os.Stat(cfg.Socket)
 		if errors.Is(err, fs.ErrNotExist) {
 			time.Sleep(100 * time.Millisecond)
 			continue
 		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
+			return nil, nil, fmt.Errorf("error waiting for tailscaled socket: %w", err)
 		}
 		break
 	}
 
-	tsClient := &tailscale.LocalClient{
+	tsClient := &local.Client{
 		Socket:        cfg.Socket,
 		UseSocketOnly: true,
 	}
@@ -170,14 +173,17 @@ func tailscaleSet(ctx context.Context, cfg *settings) error {
 	return nil
 }
 
-func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscale.LocalClient, errCh chan<- error) {
+func watchTailscaledConfigChanges(ctx context.Context, path string, lc *local.Client, errCh chan<- error) {
 	var (
 		tickChan          <-chan time.Time
+		eventChan         <-chan fsnotify.Event
+		errChan           <-chan error
 		tailscaledCfgDir  = filepath.Dir(path)
 		prevTailscaledCfg []byte
 	)
-	w, err := fsnotify.NewWatcher()
-	if err != nil {
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		// Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor.
+		// See https://github.com/tailscale/tailscale/issues/15081
 		log.Printf("tailscaled config watch: failed to create fsnotify watcher, timer-only mode: %v", err)
 		ticker := time.NewTicker(5 * time.Second)
 		defer ticker.Stop()
@@ -188,6 +194,8 @@ func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscal
 			errCh <- fmt.Errorf("failed to add fsnotify watch: %w", err)
 			return
 		}
+		eventChan = w.Events
+		errChan = w.Errors
 	}
 	b, err := os.ReadFile(path)
 	if err != nil {
@@ -205,11 +213,11 @@ func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscal
 		select {
 		case <-ctx.Done():
 			return
-		case err := <-w.Errors:
+		case err := <-errChan:
 			errCh <- fmt.Errorf("watcher error: %w", err)
 			return
 		case <-tickChan:
-		case event := <-w.Events:
+		case event := <-eventChan:
 			if event.Name != toWatch {
 				continue
 			}

cmd/derper/cert.go

@@ -4,16 +4,28 @@
 package main
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
+	"log"
+	"math/big"
 	"net"
 	"net/http"
+	"os"
 	"path/filepath"
 	"regexp"
+	"time"
 
 	"golang.org/x/crypto/acme/autocert"
+	"tailscale.com/tailcfg"
 )
 
 var unsafeHostnameCharacters = regexp.MustCompile(`[^a-zA-Z0-9-\.]`)
@@ -65,8 +77,18 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	crtPath := filepath.Join(certdir, keyname+".crt")
 	keyPath := filepath.Join(certdir, keyname+".key")
 	cert, err := tls.LoadX509KeyPair(crtPath, keyPath)
+	hostnameIP := net.ParseIP(hostname) // or nil if hostname isn't an IP address
 	if err != nil {
-		return nil, fmt.Errorf("can not load x509 key pair for hostname %q: %w", keyname, err)
+		// If the hostname is an IP address, automatically create a
+		// self-signed certificate for it.
+		var certp *tls.Certificate
+		if os.IsNotExist(err) && hostnameIP != nil {
+			certp, err = createSelfSignedIPCert(crtPath, keyPath, hostname)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("can not load x509 key pair for hostname %q: %w", keyname, err)
+		}
+		cert = *certp
 	}
 	// ensure hostname matches with the certificate
 	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
@@ -76,6 +98,18 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	if err := x509Cert.VerifyHostname(hostname); err != nil {
 		return nil, fmt.Errorf("cert invalid for hostname %q: %w", hostname, err)
 	}
+	if hostnameIP != nil {
+		// If the hostname is an IP address, print out information on how to
+		// confgure this in the derpmap.
+		dn := &tailcfg.DERPNode{
+			Name:     "custom",
+			RegionID: 900,
+			HostName: hostname,
+			CertName: fmt.Sprintf("sha256-raw:%-02x", sha256.Sum256(x509Cert.Raw)),
+		}
+		dnJSON, _ := json.Marshal(dn)
+		log.Printf("Using self-signed certificate for IP address %q. Configure it in DERPMap using: (https://tailscale.com/s/custom-derp)\n  %s", hostname, dnJSON)
+	}
 	return &manualCertManager{
 		cert:       &cert,
 		hostname:   hostname,
@@ -94,18 +128,85 @@ func (m *manualCertManager) TLSConfig() *tls.Config {
 }
 
 func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	if hi.ServerName != m.hostname && !m.noHostname {
-		return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
-	}
+	// if hi.ServerName != m.hostname && !m.noHostname {
+	// 	return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
+	// }
 
 	// Return a shallow copy of the cert so the caller can append to its
 	// Certificate field.
-	certCopy := new(tls.Certificate)
-	*certCopy = *m.cert
-	certCopy.Certificate = certCopy.Certificate[:len(certCopy.Certificate):len(certCopy.Certificate)]
-	return certCopy, nil
+	// certCopy := new(tls.Certificate)
+	// *certCopy = *m.cert
+	// certCopy.Certificate = certCopy.Certificate[:len(certCopy.Certificate):len(certCopy.Certificate)]
+	// return certCopy, nil
+	return m.cert, nil
 }
 
 func (m *manualCertManager) HTTPHandler(fallback http.Handler) http.Handler {
 	return fallback
 }
+
+func createSelfSignedIPCert(crtPath, keyPath, ipStr string) (*tls.Certificate, error) {
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return nil, fmt.Errorf("invalid IP address: %s", ipStr)
+	}
+
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate EC private key: %v", err)
+	}
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate serial number: %v", err)
+	}
+
+	now := time.Now()
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName: ipStr,
+		},
+		NotBefore: now,
+		NotAfter:  now.AddDate(1, 0, 0), // expires in 1 year; a bit over that is rejected by macOS etc
+
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	// Set the IP as a SAN.
+	template.IPAddresses = []net.IP{ip}
+
+	// Create the self-signed certificate.
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create certificate: %v", err)
+	}
+
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+
+	keyBytes, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal EC private key: %v", err)
+	}
+
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes})
+
+	if err := os.MkdirAll(filepath.Dir(crtPath), 0700); err != nil {
+		return nil, fmt.Errorf("failed to create directory for certificate: %v", err)
+	}
+	if err := os.WriteFile(crtPath, certPEM, 0644); err != nil {
+		return nil, fmt.Errorf("failed to write certificate to %s: %v", crtPath, err)
+	}
+	if err := os.WriteFile(keyPath, keyPEM, 0600); err != nil {
+		return nil, fmt.Errorf("failed to write key to %s: %v", keyPath, err)
+	}
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tls.Certificate: %v", err)
+	}
+	return &tlsCert, nil
+}

cmd/derper/cert_test.go

@@ -4,19 +4,29 @@
 package main
 
 import (
+	"context"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"fmt"
 	"math/big"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/net/netmon"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 // Verify that in --certmode=manual mode, we can use a bare IP address
@@ -95,3 +105,66 @@ func TestCertIP(t *testing.T) {
 		t.Fatalf("GetCertificate returned nil")
 	}
 }
+
+// Test that we can dial a raw IP without using a hostname and without a WebPKI
+// cert, validating the cert against the signature of the cert in the DERP map's
+// DERPNode.
+//
+// See https://github.com/tailscale/tailscale/issues/11776.
+func TestPinnedCertRawIP(t *testing.T) {
+	td := t.TempDir()
+	cp, err := NewManualCertManager(td, "127.0.0.1")
+	if err != nil {
+		t.Fatalf("NewManualCertManager: %v", err)
+	}
+
+	cert, err := cp.TLSConfig().GetCertificate(&tls.ClientHelloInfo{
+		ServerName: "127.0.0.1",
+	})
+	if err != nil {
+		t.Fatalf("GetCertificate: %v", err)
+	}
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("Listen: %v", err)
+	}
+	defer ln.Close()
+
+	ds := derp.NewServer(key.NewNode(), t.Logf)
+
+	derpHandler := derphttp.Handler(ds)
+	mux := http.NewServeMux()
+	mux.Handle("/derp", derpHandler)
+
+	var hs http.Server
+	hs.Handler = mux
+	hs.TLSConfig = cp.TLSConfig()
+	go hs.ServeTLS(ln, "", "")
+
+	lnPort := ln.Addr().(*net.TCPAddr).Port
+
+	reg := &tailcfg.DERPRegion{
+		RegionID: 900,
+		Nodes: []*tailcfg.DERPNode{
+			{
+				RegionID: 900,
+				HostName: "127.0.0.1",
+				CertName: fmt.Sprintf("sha256-raw:%-02x", sha256.Sum256(cert.Leaf.Raw)),
+				DERPPort: lnPort,
+			},
+		},
+	}
+
+	netMon := netmon.NewStatic()
+	dc := derphttp.NewRegionClient(key.NewNode(), t.Logf, netMon, func() *tailcfg.DERPRegion {
+		return reg
+	})
+	defer dc.Close()
+
+	_, connClose, _, err := dc.DialRegionTLS(context.Background(), reg)
+	if err != nil {
+		t.Fatalf("DialRegionTLS: %v", err)
+	}
+	defer connClose.Close()
+}

cmd/derper/depaware.txt

@@ -51,9 +51,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
    L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
+        github.com/tailscale/setec/client/setec                      from tailscale.com/cmd/derper
+        github.com/tailscale/setec/types/api                         from github.com/tailscale/setec/client/setec
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
-     💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
+     💣 go4.org/mem                                                  from tailscale.com/client/local+
         go4.org/netipx                                               from tailscale.com/net/tsaddr
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         google.golang.org/protobuf/encoding/protodelim               from github.com/prometheus/common/expfmt
@@ -86,17 +88,19 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
      💣 tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
+        tailscale.com/client/local                                   from tailscale.com/client/tailscale+
         tailscale.com/client/tailscale                               from tailscale.com/derp
-        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale
+        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/derp                                           from tailscale.com/cmd/derper+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/derper
         tailscale.com/disco                                          from tailscale.com/derp
-        tailscale.com/drive                                          from tailscale.com/client/tailscale+
-        tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/drive                                          from tailscale.com/client/local+
+        tailscale.com/envknob                                        from tailscale.com/client/local+
+        tailscale.com/feature                                        from tailscale.com/tsweb
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
-        tailscale.com/ipn                                            from tailscale.com/client/tailscale
-        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/ipn                                            from tailscale.com/client/local
+        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/local+
         tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/bakedroots                                 from tailscale.com/net/tlsdial
@@ -106,7 +110,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
      💣 tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
      💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
-        tailscale.com/net/netutil                                    from tailscale.com/client/tailscale
+        tailscale.com/net/netutil                                    from tailscale.com/client/local
         tailscale.com/net/sockstats                                  from tailscale.com/derp/derphttp
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
@@ -116,22 +120,22 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper
-        tailscale.com/paths                                          from tailscale.com/client/tailscale
-     💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
+        tailscale.com/paths                                          from tailscale.com/client/local
+     💣 tailscale.com/safesocket                                     from tailscale.com/client/local
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
-        tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
-        tailscale.com/tka                                            from tailscale.com/client/tailscale+
+        tailscale.com/tailcfg                                        from tailscale.com/client/local+
+        tailscale.com/tka                                            from tailscale.com/client/local+
    W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/derp
-        tailscale.com/tsweb                                          from tailscale.com/cmd/derper
-        tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb                                          from tailscale.com/cmd/derper+
+        tailscale.com/tsweb/promvarz                                 from tailscale.com/cmd/derper
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
-        tailscale.com/types/key                                      from tailscale.com/client/tailscale+
+        tailscale.com/types/key                                      from tailscale.com/client/local+
         tailscale.com/types/lazy                                     from tailscale.com/version+
         tailscale.com/types/logger                                   from tailscale.com/cmd/derper+
         tailscale.com/types/netmap                                   from tailscale.com/ipn
@@ -141,7 +145,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
         tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
-        tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
+        tailscale.com/types/tkatype                                  from tailscale.com/client/local+
         tailscale.com/types/views                                    from tailscale.com/ipn+
         tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/net/netmon+
@@ -188,11 +192,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls+
+        golang.org/x/crypto/internal/alias                           from golang.org/x/crypto/chacha20+
+        golang.org/x/crypto/internal/poly1305                        from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
         golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
@@ -201,9 +205,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/net/http/httpproxy                              from net/http+
         golang.org/x/net/http2/hpack                                 from net/http
         golang.org/x/net/idna                                        from golang.org/x/crypto/acme/autocert+
+        golang.org/x/net/internal/socks                              from golang.org/x/net/proxy
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
         golang.org/x/sync/errgroup                                   from github.com/mdlayher/socket+
+        golang.org/x/sync/singleflight                               from github.com/tailscale/setec/client/setec
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/argon2+
   LD    golang.org/x/sys/unix                                        from github.com/google/nftables+
    W    golang.org/x/sys/windows                                     from github.com/dblohm7/wingoes+
@@ -223,7 +229,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
         crypto                                                       from crypto/ecdh+
-        crypto/aes                                                   from crypto/ecdsa+
+        crypto/aes                                                   from crypto/internal/hpke+
         crypto/cipher                                                from crypto/aes+
         crypto/des                                                   from crypto/tls+
         crypto/dsa                                                   from crypto/x509
@@ -232,18 +238,58 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         crypto/ed25519                                               from crypto/tls+
         crypto/elliptic                                              from crypto/ecdsa+
         crypto/hmac                                                  from crypto/tls+
+        crypto/internal/boring                                       from crypto/aes+
+        crypto/internal/boring/bbig                                  from crypto/ecdsa+
+        crypto/internal/boring/sig                                   from crypto/internal/boring
+        crypto/internal/entropy                                      from crypto/internal/fips140/drbg
+        crypto/internal/fips140                                      from crypto/internal/fips140/aes+
+        crypto/internal/fips140/aes                                  from crypto/aes+
+        crypto/internal/fips140/aes/gcm                              from crypto/cipher+
+        crypto/internal/fips140/alias                                from crypto/cipher+
+        crypto/internal/fips140/bigmod                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/check                                from crypto/internal/fips140/aes+
+        crypto/internal/fips140/drbg                                 from crypto/internal/fips140/aes/gcm+
+        crypto/internal/fips140/ecdh                                 from crypto/ecdh
+        crypto/internal/fips140/ecdsa                                from crypto/ecdsa
+        crypto/internal/fips140/ed25519                              from crypto/ed25519
+        crypto/internal/fips140/edwards25519                         from crypto/internal/fips140/ed25519
+        crypto/internal/fips140/edwards25519/field                   from crypto/ecdh+
+        crypto/internal/fips140/hkdf                                 from crypto/internal/fips140/tls13+
+        crypto/internal/fips140/hmac                                 from crypto/hmac+
+        crypto/internal/fips140/mlkem                                from crypto/tls
+        crypto/internal/fips140/nistec                               from crypto/elliptic+
+        crypto/internal/fips140/nistec/fiat                          from crypto/internal/fips140/nistec
+        crypto/internal/fips140/rsa                                  from crypto/rsa
+        crypto/internal/fips140/sha256                               from crypto/internal/fips140/check+
+        crypto/internal/fips140/sha3                                 from crypto/internal/fips140/hmac+
+        crypto/internal/fips140/sha512                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/subtle                               from crypto/internal/fips140/aes+
+        crypto/internal/fips140/tls12                                from crypto/tls
+        crypto/internal/fips140/tls13                                from crypto/tls
+        crypto/internal/fips140deps/byteorder                        from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/cpu                              from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/godebug                          from crypto/internal/fips140+
+        crypto/internal/fips140hash                                  from crypto/ecdsa+
+        crypto/internal/fips140only                                  from crypto/cipher+
+        crypto/internal/hpke                                         from crypto/tls
+        crypto/internal/impl                                         from crypto/internal/fips140/aes+
+        crypto/internal/randutil                                     from crypto/dsa+
+        crypto/internal/sysrand                                      from crypto/internal/entropy+
         crypto/md5                                                   from crypto/tls+
         crypto/rand                                                  from crypto/ed25519+
         crypto/rc4                                                   from crypto/tls
         crypto/rsa                                                   from crypto/tls+
         crypto/sha1                                                  from crypto/tls+
         crypto/sha256                                                from crypto/tls+
+        crypto/sha3                                                  from crypto/internal/fips140hash
         crypto/sha512                                                from crypto/ecdsa+
-        crypto/subtle                                                from crypto/aes+
+        crypto/subtle                                                from crypto/cipher+
         crypto/tls                                                   from golang.org/x/crypto/acme+
+        crypto/tls/internal/fips140tls                               from crypto/tls
         crypto/x509                                                  from crypto/tls+
+   D    crypto/x509/internal/macos                                   from crypto/x509
         crypto/x509/pkix                                             from crypto/x509+
-        embed                                                        from crypto/internal/nistec+
+        embed                                                        from google.golang.org/protobuf/internal/editiondefaults+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base32                                              from github.com/fxamacker/cbor/v2+
@@ -263,6 +309,46 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
         html/template                                                from tailscale.com/cmd/derper
+        internal/abi                                                 from crypto/x509/internal/macos+
+        internal/asan                                                from internal/runtime/maps+
+        internal/bisect                                              from internal/godebug
+        internal/bytealg                                             from bytes+
+        internal/byteorder                                           from crypto/cipher+
+        internal/chacha8rand                                         from math/rand/v2+
+        internal/coverage/rtcov                                      from runtime
+        internal/cpu                                                 from crypto/internal/fips140deps/cpu+
+        internal/filepathlite                                        from os+
+        internal/fmtsort                                             from fmt+
+        internal/goarch                                              from crypto/internal/fips140deps/cpu+
+        internal/godebug                                             from crypto/internal/fips140deps/godebug+
+        internal/godebugs                                            from internal/godebug+
+        internal/goexperiment                                        from hash/maphash+
+        internal/goos                                                from crypto/x509+
+        internal/itoa                                                from internal/poll+
+        internal/msan                                                from internal/runtime/maps+
+        internal/nettrace                                            from net+
+        internal/oserror                                             from io/fs+
+        internal/poll                                                from net+
+        internal/profile                                             from net/http/pprof
+        internal/profilerecord                                       from runtime+
+        internal/race                                                from internal/poll+
+        internal/reflectlite                                         from context+
+        internal/runtime/atomic                                      from internal/runtime/exithook+
+        internal/runtime/exithook                                    from runtime
+        internal/runtime/maps                                        from reflect+
+        internal/runtime/math                                        from internal/runtime/maps+
+        internal/runtime/sys                                         from crypto/subtle+
+   L    internal/runtime/syscall                                     from runtime+
+        internal/singleflight                                        from net
+        internal/stringslite                                         from embed+
+        internal/sync                                                from sync+
+        internal/syscall/execenv                                     from os+
+  LD    internal/syscall/unix                                        from crypto/internal/sysrand+
+   W    internal/syscall/windows                                     from crypto/internal/sysrand+
+   W    internal/syscall/windows/registry                            from mime+
+   W    internal/syscall/windows/sysdll                              from internal/syscall/windows+
+        internal/testlog                                             from os
+        internal/unsafeheader                                        from internal/reflectlite+
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
    L    io/ioutil                                                    from github.com/mitchellh/go-ps+
@@ -274,7 +360,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from internal/concurrent+
+        math/rand/v2                                                 from crypto/ecdsa+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -282,11 +368,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
+        net/http/internal/ascii                                      from net/http
         net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
-        os                                                           from crypto/rand+
+        os                                                           from crypto/internal/sysrand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
    W    os/user                                                      from tailscale.com/util/winutil+
@@ -295,6 +382,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         reflect                                                      from crypto/x509+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
+        runtime                                                      from crypto/internal/fips140+
         runtime/debug                                                from github.com/prometheus/client_golang/prometheus+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof
@@ -305,7 +393,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         strings                                                      from bufio+
         sync                                                         from compress/flate+
         sync/atomic                                                  from context+
-        syscall                                                      from crypto/rand+
+        syscall                                                      from crypto/internal/sysrand+
         text/tabwriter                                               from runtime/pprof
         text/template                                                from html/template
         text/template/parse                                          from html/template+
@@ -314,3 +402,5 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
         unique                                                       from net/netip
+        unsafe                                                       from bytes+
+        weak                                                         from unique

cmd/derper/derper.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -36,6 +37,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/tailscale/setec/client/setec"
 	"golang.org/x/time/rate"
 	"tailscale.com/atomicfile"
 	"tailscale.com/derp"
@@ -47,6 +49,9 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (
@@ -61,15 +66,22 @@ var (
 	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks")
 	runSTUN     = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
 	runDERP     = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
+	flagHome    = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to")
 
 	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.")
+	secretsURL      = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key")
+	secretPrefix    = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key")
+	secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)")
 	bootstrapDNS    = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
 	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
+
 	verifyClients   = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 	verifyClientURL = flag.String("verify-client-url", "", "if non-empty, an admission controller URL for permitting client connections; see tailcfg.DERPAdmitClientRequest")
 	verifyFailOpen  = flag.Bool("verify-client-url-fail-open", true, "whether we fail open if --verify-client-url is unreachable")
 
+	socket = flag.String("socket", "", "optional alternate path to tailscaled socket (only relevant when using --verify-clients)")
+
 	acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection")
 	acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection")
 
@@ -77,13 +89,21 @@ var (
 	tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time")
 	// tcpUserTimeout is intentionally short, so that hung connections are cleaned up promptly. DERPs should be nearby users.
 	tcpUserTimeout = flag.Duration("tcp-user-timeout", 15*time.Second, "TCP user timeout")
+	// tcpWriteTimeout is the timeout for writing to client TCP connections. It does not apply to mesh connections.
+	tcpWriteTimeout = flag.Duration("tcp-write-timeout", derp.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes")
 )
 
 var (
 	tlsRequestVersion = &metrics.LabelMap{Label: "version"}
 	tlsActiveVersion  = &metrics.LabelMap{Label: "version"}
+
+	// Exactly 64 hexadecimal lowercase digits.
+	validMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`)
 )
 
+const setecMeshKeyName = "meshkey"
+const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY"
+
 func init() {
 	expvar.Publish("derper_tls_request_version", tlsRequestVersion)
 	expvar.Publish("gauge_derper_tls_active_version", tlsActiveVersion)
@@ -139,6 +159,14 @@ func writeNewConfig() config {
 	return cfg
 }
 
+func checkMeshKey(key string) (string, error) {
+	key = strings.TrimSpace(key)
+	if !validMeshKey.MatchString(key) {
+		return "", errors.New("key must contain exactly 64 hex digits")
+	}
+	return key, nil
+}
+
 func main() {
 	flag.Parse()
 	if *versionFlag {
@@ -171,26 +199,70 @@ func main() {
 
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)
+	s.SetTailscaledSocketPath(*socket)
 	s.SetVerifyClientURL(*verifyClientURL)
 	s.SetVerifyClientURLFailOpen(*verifyFailOpen)
+	s.SetTCPWriteTimeout(*tcpWriteTimeout)
 
-	if *meshPSKFile != "" {
-		b, err := os.ReadFile(*meshPSKFile)
+	var meshKey string
+	if *dev {
+		meshKey = os.Getenv(meshKeyEnvVar)
+		if meshKey == "" {
+			log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar)
+		} else {
+			log.Printf("Set mesh key from %s\n", meshKeyEnvVar)
+		}
+	} else if *secretsURL != "" {
+		meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName)
+		fc, err := setec.NewFileCache(*secretsCacheDir)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("NewFileCache: %v", err)
 		}
-		key := strings.TrimSpace(string(b))
-		if matched, _ := regexp.MatchString(`(?i)^[0-9a-f]{64,}$`, key); !matched {
-			log.Fatalf("key in %s must contain 64+ hex digits", *meshPSKFile)
+		log.Printf("Setting up setec store from %q", *secretsURL)
+		st, err := setec.NewStore(ctx,
+			setec.StoreConfig{
+				Client: setec.Client{Server: *secretsURL},
+				Secrets: []string{
+					meshKeySecret,
+				},
+				Cache: fc,
+			})
+		if err != nil {
+			log.Fatalf("NewStore: %v", err)
 		}
-		s.SetMeshKey(key)
-		log.Printf("DERP mesh key configured")
+		meshKey = st.Secret(meshKeySecret).GetString()
+		log.Println("Got mesh key from setec store")
+		st.Close()
+	} else if *meshPSKFile != "" {
+		b, err := setec.StaticFile(*meshPSKFile)
+		if err != nil {
+			log.Fatalf("StaticFile failed to get key: %v", err)
+		}
+		log.Println("Got mesh key from static file")
+		meshKey = b.GetString()
 	}
+
+	if meshKey == "" && *dev {
+		log.Printf("No mesh key configured for --dev mode")
+	} else if meshKey == "" {
+		log.Printf("No mesh key configured")
+	} else if key, err := checkMeshKey(meshKey); err != nil {
+		log.Fatalf("invalid mesh key: %v", err)
+	} else {
+		s.SetMeshKey(key)
+		log.Println("DERP mesh key configured")
+	}
+
 	if err := startMesh(s); err != nil {
 		log.Fatalf("startMesh: %v", err)
 	}
 	expvar.Publish("derp", s.ExpVar())
 
+	handleHome, ok := getHomeHandler(*flagHome)
+	if !ok {
+		log.Fatalf("unknown --home value %q", *flagHome)
+	}
+
 	mux := http.NewServeMux()
 	if *runDERP {
 		derpHandler := derphttp.Handler(s)
@@ -211,19 +283,7 @@ func main() {
 	mux.HandleFunc("/bootstrap-dns", tsweb.BrowserHeaderHandlerFunc(handleBootstrapDNS))
 	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tsweb.AddBrowserHeaders(w)
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		w.WriteHeader(200)
-		err := homePageTemplate.Execute(w, templateData{
-			ShowAbuseInfo: validProdHostname.MatchString(*hostname),
-			Disabled:      !*runDERP,
-			AllowDebug:    tsweb.AllowDebugAccess(r),
-		})
-		if err != nil {
-			if r.Context().Err() == nil {
-				log.Printf("homePageTemplate.Execute: %v", err)
-			}
-			return
-		}
+		handleHome.ServeHTTP(w, r)
 	}))
 	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tsweb.AddBrowserHeaders(w)
@@ -265,6 +325,9 @@ func main() {
 		Control:   ktimeout.UserTimeout(*tcpUserTimeout),
 		KeepAlive: *tcpKeepAlive,
 	}
+	// As of 2025-02-19, MPTCP does not support TCP_USER_TIMEOUT socket option
+	// set in ktimeout.UserTimeout above.
+	lc.SetMultipathTCP(false)
 
 	quietLogger := log.New(logger.HTTPServerLogFilter{Inner: log.Printf}, "", 0)
 	httpsrv := &http.Server{
@@ -379,6 +442,10 @@ func prodAutocertHostPolicy(_ context.Context, host string) error {
 	return errors.New("invalid hostname")
 }
 
+func defaultSetecCacheDir() string {
+	return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets")
+}
+
 func defaultMeshPSKFile() string {
 	try := []string{
 		"/home/derp/keys/derp-mesh.key",
@@ -509,3 +576,35 @@ var homePageTemplate = template.Must(template.New("home").Parse(`<html><body>
 </body>
 </html>
 `))
+
+// getHomeHandler returns a handler for the home page based on a flag string
+// as documented on the --home flag.
+func getHomeHandler(val string) (_ http.Handler, ok bool) {
+	if val == "" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.WriteHeader(200)
+			err := homePageTemplate.Execute(w, templateData{
+				ShowAbuseInfo: validProdHostname.MatchString(*hostname),
+				Disabled:      !*runDERP,
+				AllowDebug:    tsweb.AllowDebugAccess(r),
+			})
+			if err != nil {
+				if r.Context().Err() == nil {
+					log.Printf("homePageTemplate.Execute: %v", err)
+				}
+				return
+			}
+		}), true
+	}
+	if val == "blank" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.WriteHeader(200)
+		}), true
+	}
+	if strings.HasPrefix(val, "http://") || strings.HasPrefix(val, "https://") {
+		return http.RedirectHandler(val, http.StatusFound), true
+	}
+	return nil, false
+}

cmd/derper/derper_test.go

@@ -138,3 +138,46 @@ func TestTemplate(t *testing.T) {
 		t.Error("Output is missing debug info")
 	}
 }
+
+func TestCheckMeshKey(t *testing.T) {
+	testCases := []struct {
+		name    string
+		input   string
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "KeyOkay",
+			input:   "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			wantErr: false,
+		},
+		{
+			name:    "TrimKeyOkay",
+			input:   "  f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6  ",
+			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			wantErr: false,
+		},
+		{
+			name:    "NotAKey",
+			input:   "zzthisisnotakey",
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			k, err := checkMeshKey(tt.input)
+			if err != nil && !tt.wantErr {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if k != tt.want && err == nil {
+				t.Errorf("want: %s doesn't match expected: %s", tt.want, k)
+			}
+
+		})
+	}
+
+}

cmd/derpprobe/derpprobe.go

@@ -15,6 +15,9 @@ import (
 	"tailscale.com/prober"
 	"tailscale.com/tsweb"
 	"tailscale.com/version"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/get-authkey/main.go

@@ -16,14 +16,10 @@ import (
 	"strings"
 
 	"golang.org/x/oauth2/clientcredentials"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/internal/client/tailscale"
 )
 
 func main() {
-	// Required to use our client API. We're fine with the instability since the
-	// client lives in the same repo as this code.
-	tailscale.I_Acknowledge_This_API_Is_Unstable = true
-
 	reusable := flag.Bool("reusable", false, "allocate a reusable authkey")
 	ephemeral := flag.Bool("ephemeral", false, "allocate an ephemeral authkey")
 	preauth := flag.Bool("preauth", true, "set the authkey as pre-authorized")

cmd/gitops-pusher/gitops-pusher.go

@@ -13,6 +13,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -405,7 +406,8 @@ func getACLETag(ctx context.Context, client *http.Client, tailnet, apiKey string
 	got := resp.StatusCode
 	want := http.StatusOK
 	if got != want {
-		return "", fmt.Errorf("wanted HTTP status code %d but got %d", want, got)
+		errorDetails, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("wanted HTTP status code %d but got %d: %#q", want, got, string(errorDetails))
 	}
 
 	return Shuck(resp.Header.Get("ETag")), nil

cmd/hello/hello.go

@@ -18,8 +18,9 @@ import (
 	"strings"
 	"time"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/tailcfg"
 )
 
 var (
@@ -31,7 +32,7 @@ var (
 //go:embed hello.tmpl.html
 var embeddedTemplate string
 
-var localClient tailscale.LocalClient
+var localClient local.Client
 
 func main() {
 	flag.Parse()
@@ -134,6 +135,10 @@ func tailscaleIP(who *apitype.WhoIsResponse) string {
 	if who == nil {
 		return ""
 	}
+	vals, err := tailcfg.UnmarshalNodeCapJSON[string](who.Node.CapMap, tailcfg.NodeAttrNativeIPV4)
+	if err == nil && len(vals) > 0 {
+		return vals[0]
+	}
 	for _, nodeIP := range who.Node.Addresses {
 		if nodeIP.Addr().Is4() && nodeIP.IsSingleIP() {
 			return nodeIP.Addr().String()

cmd/k8s-operator/depaware.txt

@@ -9,7 +9,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/aws/defaults                    from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/aws/middleware                  from github.com/aws/aws-sdk-go-v2/aws/retry+
-   L    github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics  from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/query              from github.com/aws/aws-sdk-go-v2/service/sts
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/restjson           from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/xml                from github.com/aws/aws-sdk-go-v2/service/sts
@@ -31,10 +30,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/aws-sdk-go-v2/internal/auth                   from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
    L    github.com/aws/aws-sdk-go-v2/internal/auth/smithy            from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/internal/configsources          from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/context                from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/aws-sdk-go-v2/internal/endpoints              from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn   from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2           from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
    L    github.com/aws/aws-sdk-go-v2/internal/ini                    from github.com/aws/aws-sdk-go-v2/config
+   L    github.com/aws/aws-sdk-go-v2/internal/middleware             from github.com/aws/aws-sdk-go-v2/service/sso+
    L    github.com/aws/aws-sdk-go-v2/internal/rand                   from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdk                    from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdkio                  from github.com/aws/aws-sdk-go-v2/credentials/processcreds
@@ -69,16 +70,17 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/smithy-go/internal/sync/singleflight          from github.com/aws/smithy-go/auth/bearer
    L    github.com/aws/smithy-go/io                                  from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+
    L    github.com/aws/smithy-go/logging                             from github.com/aws/aws-sdk-go-v2/aws+
+   L    github.com/aws/smithy-go/metrics                             from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/smithy-go/middleware                          from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/smithy-go/private/requestcompression          from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/smithy-go/ptr                                 from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/smithy-go/rand                                from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/time                                from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/smithy-go/tracing                             from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/transport/http                      from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
-        github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
@@ -96,6 +98,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka+
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
+        github.com/gaissmai/bart/internal/bitset                     from github.com/gaissmai/bart+
+        github.com/gaissmai/bart/internal/sparse                     from github.com/gaissmai/bart
         github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
@@ -139,8 +143,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
-   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
+   L 💣 github.com/illarion/gonotify/v3                              from tailscale.com/net/dns
+   L    github.com/illarion/gonotify/v3/syscallf                     from github.com/illarion/gonotify/v3
+   L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/feature/tap
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/interfaces                      from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/rfc1035label                    from github.com/insomniacslk/dhcp/dhcpv4
@@ -156,7 +161,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
-        github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
+        github.com/kortschak/wol                                     from tailscale.com/feature/wakeonlan
         github.com/mailru/easyjson/buffer                            from github.com/mailru/easyjson/jwriter
      💣 github.com/mailru/easyjson/jlexer                            from github.com/go-openapi/swag
         github.com/mailru/easyjson/jwriter                           from github.com/go-openapi/swag
@@ -197,10 +202,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    W 💣 github.com/tailscale/go-winio/internal/socket                from github.com/tailscale/go-winio
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
-        github.com/tailscale/golang-x-crypto/acme                    from tailscale.com/ipn/ipnlocal
-  LD    github.com/tailscale/golang-x-crypto/internal/poly1305       from github.com/tailscale/golang-x-crypto/ssh
-  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal
-  LD    github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh
         github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2+
         github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
         github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
@@ -236,7 +237,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         go.uber.org/zap/internal/pool                                from go.uber.org/zap+
         go.uber.org/zap/internal/stacktrace                          from go.uber.org/zap
         go.uber.org/zap/zapcore                                      from github.com/go-logr/zapr+
-     💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
+     💣 go4.org/mem                                                  from tailscale.com/client/local+
         go4.org/netipx                                               from tailscale.com/ipn/ipnlocal+
    W 💣 golang.zx2c4.com/wintun                                      from github.com/tailscale/wireguard-go/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/dns+
@@ -297,12 +298,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/hash/jenkins                     from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
         gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
         gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
         gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/feature/tap+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
@@ -781,7 +782,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com                                                from tailscale.com/version
         tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
      💣 tailscale.com/atomicfile                                     from tailscale.com/ipn+
-        tailscale.com/client/tailscale                               from tailscale.com/client/web+
+        tailscale.com/client/local                                   from tailscale.com/client/tailscale+
+        tailscale.com/client/tailscale                               from tailscale.com/cmd/k8s-operator+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
@@ -798,19 +800,27 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/ethtool                                 from tailscale.com/ipn/ipnlocal
      💣 tailscale.com/doctor/permissions                             from tailscale.com/ipn/ipnlocal
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
-        tailscale.com/drive                                          from tailscale.com/client/tailscale+
-        tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/drive                                          from tailscale.com/client/local+
+        tailscale.com/envknob                                        from tailscale.com/client/local+
         tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
+        tailscale.com/feature                                        from tailscale.com/feature/wakeonlan+
+        tailscale.com/feature/capture                                from tailscale.com/feature/condregister
+        tailscale.com/feature/condregister                           from tailscale.com/tsnet
+   L    tailscale.com/feature/tap                                    from tailscale.com/feature/condregister
+        tailscale.com/feature/wakeonlan                              from tailscale.com/feature/condregister
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
+        tailscale.com/internal/client/tailscale                      from tailscale.com/cmd/k8s-operator
         tailscale.com/internal/noiseconn                             from tailscale.com/control/controlclient
-        tailscale.com/ipn                                            from tailscale.com/client/tailscale+
+        tailscale.com/ipn                                            from tailscale.com/client/local+
+        tailscale.com/ipn/auditlog                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn/conffile                                   from tailscale.com/ipn/ipnlocal+
+     💣 tailscale.com/ipn/desktop                                    from tailscale.com/ipn/ipnlocal+
      💣 tailscale.com/ipn/ipnauth                                    from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/localapi+
-        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
-        tailscale.com/ipn/localapi                                   from tailscale.com/tsnet
+        tailscale.com/ipn/ipnstate                                   from tailscale.com/client/local+
+        tailscale.com/ipn/localapi                                   from tailscale.com/tsnet+
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/ipn/store                                      from tailscale.com/ipn/ipnlocal+
    L    tailscale.com/ipn/store/awsstore                             from tailscale.com/ipn/store
@@ -856,7 +866,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/net/netmon                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
    W 💣 tailscale.com/net/netstat                                    from tailscale.com/portlist
-        tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
+        tailscale.com/net/netutil                                    from tailscale.com/client/local+
         tailscale.com/net/packet                                     from tailscale.com/net/connstats+
         tailscale.com/net/packet/checksum                            from tailscale.com/net/tstun
         tailscale.com/net/ping                                       from tailscale.com/net/netcheck+
@@ -874,17 +884,19 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/tsd+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
-        tailscale.com/paths                                          from tailscale.com/client/tailscale+
+        tailscale.com/paths                                          from tailscale.com/client/local+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
-     💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
+     💣 tailscale.com/safesocket                                     from tailscale.com/client/local+
         tailscale.com/sessionrecording                               from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
-        tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
+        tailscale.com/tailcfg                                        from tailscale.com/client/local+
         tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
+        tailscale.com/tempfork/acme                                  from tailscale.com/ipn/ipnlocal
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
-        tailscale.com/tka                                            from tailscale.com/client/tailscale+
+        tailscale.com/tempfork/httprec                               from tailscale.com/control/controlclient
+        tailscale.com/tka                                            from tailscale.com/client/local+
         tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
         tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
@@ -893,10 +905,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
         tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
+        tailscale.com/types/bools                                    from tailscale.com/tsnet
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
-        tailscale.com/types/key                                      from tailscale.com/client/tailscale+
+        tailscale.com/types/key                                      from tailscale.com/client/local+
         tailscale.com/types/lazy                                     from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/logger                                   from tailscale.com/appc+
         tailscale.com/types/logid                                    from tailscale.com/ipn/ipnlocal+
@@ -909,7 +922,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
         tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
-        tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
+        tailscale.com/types/tkatype                                  from tailscale.com/client/local+
         tailscale.com/types/views                                    from tailscale.com/appc+
         tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/cmd/k8s-operator+
@@ -965,7 +978,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
         tailscale.com/wgengine                                       from tailscale.com/ipn/ipnlocal+
-        tailscale.com/wgengine/capture                               from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap+
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
@@ -981,18 +993,21 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
-  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf
-        golang.org/x/crypto/chacha20                                 from github.com/tailscale/golang-x-crypto/ssh+
+  LD    golang.org/x/crypto/blowfish                                 from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf
+        golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/ssh+
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
-        golang.org/x/crypto/curve25519                               from github.com/tailscale/golang-x-crypto/ssh+
-        golang.org/x/crypto/hkdf                                     from crypto/tls+
+        golang.org/x/crypto/curve25519                               from golang.org/x/crypto/ssh+
+        golang.org/x/crypto/hkdf                                     from tailscale.com/control/controlbase
+        golang.org/x/crypto/internal/alias                           from golang.org/x/crypto/chacha20+
+        golang.org/x/crypto/internal/poly1305                        from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
+  LD    golang.org/x/crypto/ssh                                      from tailscale.com/ipn/ipnlocal
+  LD    golang.org/x/crypto/ssh/internal/bcrypt_pbkdf                from golang.org/x/crypto/ssh
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
@@ -1005,6 +1020,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/net/http2/hpack                                 from golang.org/x/net/http2+
         golang.org/x/net/icmp                                        from github.com/prometheus-community/pro-bing+
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
+        golang.org/x/net/internal/httpcommon                         from golang.org/x/net/http2
+        golang.org/x/net/internal/iana                               from golang.org/x/net/icmp+
+        golang.org/x/net/internal/socket                             from golang.org/x/net/icmp+
+        golang.org/x/net/internal/socks                              from golang.org/x/net/proxy
         golang.org/x/net/ipv4                                        from github.com/miekg/dns+
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
@@ -1037,7 +1056,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
         crypto                                                       from crypto/ecdh+
-        crypto/aes                                                   from crypto/ecdsa+
+        crypto/aes                                                   from crypto/internal/hpke+
         crypto/cipher                                                from crypto/aes+
         crypto/des                                                   from crypto/tls+
         crypto/dsa                                                   from crypto/x509+
@@ -1046,22 +1065,62 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         crypto/ed25519                                               from crypto/tls+
         crypto/elliptic                                              from crypto/ecdsa+
         crypto/hmac                                                  from crypto/tls+
+        crypto/internal/boring                                       from crypto/aes+
+        crypto/internal/boring/bbig                                  from crypto/ecdsa+
+        crypto/internal/boring/sig                                   from crypto/internal/boring
+        crypto/internal/entropy                                      from crypto/internal/fips140/drbg
+        crypto/internal/fips140                                      from crypto/internal/fips140/aes+
+        crypto/internal/fips140/aes                                  from crypto/aes+
+        crypto/internal/fips140/aes/gcm                              from crypto/cipher+
+        crypto/internal/fips140/alias                                from crypto/cipher+
+        crypto/internal/fips140/bigmod                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/check                                from crypto/internal/fips140/aes+
+        crypto/internal/fips140/drbg                                 from crypto/internal/fips140/aes/gcm+
+        crypto/internal/fips140/ecdh                                 from crypto/ecdh
+        crypto/internal/fips140/ecdsa                                from crypto/ecdsa
+        crypto/internal/fips140/ed25519                              from crypto/ed25519
+        crypto/internal/fips140/edwards25519                         from crypto/internal/fips140/ed25519
+        crypto/internal/fips140/edwards25519/field                   from crypto/ecdh+
+        crypto/internal/fips140/hkdf                                 from crypto/internal/fips140/tls13+
+        crypto/internal/fips140/hmac                                 from crypto/hmac+
+        crypto/internal/fips140/mlkem                                from crypto/tls
+        crypto/internal/fips140/nistec                               from crypto/elliptic+
+        crypto/internal/fips140/nistec/fiat                          from crypto/internal/fips140/nistec
+        crypto/internal/fips140/rsa                                  from crypto/rsa
+        crypto/internal/fips140/sha256                               from crypto/internal/fips140/check+
+        crypto/internal/fips140/sha3                                 from crypto/internal/fips140/hmac+
+        crypto/internal/fips140/sha512                               from crypto/internal/fips140/ecdsa+
+        crypto/internal/fips140/subtle                               from crypto/internal/fips140/aes+
+        crypto/internal/fips140/tls12                                from crypto/tls
+        crypto/internal/fips140/tls13                                from crypto/tls
+        crypto/internal/fips140deps/byteorder                        from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/cpu                              from crypto/internal/fips140/aes+
+        crypto/internal/fips140deps/godebug                          from crypto/internal/fips140+
+        crypto/internal/fips140hash                                  from crypto/ecdsa+
+        crypto/internal/fips140only                                  from crypto/cipher+
+        crypto/internal/hpke                                         from crypto/tls
+        crypto/internal/impl                                         from crypto/internal/fips140/aes+
+        crypto/internal/randutil                                     from crypto/dsa+
+        crypto/internal/sysrand                                      from crypto/internal/entropy+
         crypto/md5                                                   from crypto/tls+
         crypto/rand                                                  from crypto/ed25519+
         crypto/rc4                                                   from crypto/tls+
         crypto/rsa                                                   from crypto/tls+
         crypto/sha1                                                  from crypto/tls+
         crypto/sha256                                                from crypto/tls+
+        crypto/sha3                                                  from crypto/internal/fips140hash
         crypto/sha512                                                from crypto/ecdsa+
-        crypto/subtle                                                from crypto/aes+
+        crypto/subtle                                                from crypto/cipher+
         crypto/tls                                                   from github.com/aws/aws-sdk-go-v2/aws/transport/http+
+        crypto/tls/internal/fips140tls                               from crypto/tls
         crypto/x509                                                  from crypto/tls+
+   D    crypto/x509/internal/macos                                   from crypto/x509
         crypto/x509/pkix                                             from crypto/x509+
         database/sql                                                 from github.com/prometheus/client_golang/prometheus/collectors
         database/sql/driver                                          from database/sql+
    W    debug/dwarf                                                  from debug/pe
    W    debug/pe                                                     from github.com/dblohm7/wingoes/pe
-        embed                                                        from crypto/internal/nistec+
+        embed                                                        from github.com/tailscale/web-client-prebuilt+
         encoding                                                     from encoding/gob+
         encoding/asn1                                                from crypto/x509+
         encoding/base32                                              from github.com/fxamacker/cbor/v2+
@@ -1091,6 +1150,48 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         hash/maphash                                                 from go4.org/mem
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
+        internal/abi                                                 from crypto/x509/internal/macos+
+        internal/asan                                                from internal/runtime/maps+
+        internal/bisect                                              from internal/godebug
+        internal/bytealg                                             from bytes+
+        internal/byteorder                                           from crypto/cipher+
+        internal/chacha8rand                                         from math/rand/v2+
+        internal/coverage/rtcov                                      from runtime
+        internal/cpu                                                 from crypto/internal/fips140deps/cpu+
+        internal/filepathlite                                        from os+
+        internal/fmtsort                                             from fmt+
+        internal/goarch                                              from crypto/internal/fips140deps/cpu+
+        internal/godebug                                             from archive/tar+
+        internal/godebugs                                            from internal/godebug+
+        internal/goexperiment                                        from hash/maphash+
+        internal/goos                                                from crypto/x509+
+        internal/itoa                                                from internal/poll+
+        internal/lazyregexp                                          from go/doc
+        internal/msan                                                from internal/runtime/maps+
+        internal/nettrace                                            from net+
+        internal/oserror                                             from io/fs+
+        internal/poll                                                from net+
+        internal/profile                                             from net/http/pprof
+        internal/profilerecord                                       from runtime+
+        internal/race                                                from internal/poll+
+        internal/reflectlite                                         from context+
+        internal/runtime/atomic                                      from internal/runtime/exithook+
+        internal/runtime/exithook                                    from runtime
+        internal/runtime/maps                                        from reflect+
+        internal/runtime/math                                        from internal/runtime/maps+
+        internal/runtime/sys                                         from crypto/subtle+
+   L    internal/runtime/syscall                                     from runtime+
+        internal/saferio                                             from debug/pe+
+        internal/singleflight                                        from net
+        internal/stringslite                                         from embed+
+        internal/sync                                                from sync+
+        internal/syscall/execenv                                     from os+
+  LD    internal/syscall/unix                                        from crypto/internal/sysrand+
+   W    internal/syscall/windows                                     from crypto/internal/sysrand+
+   W    internal/syscall/windows/registry                            from mime+
+   W    internal/syscall/windows/sysdll                              from internal/syscall/windows+
+        internal/testlog                                             from os
+        internal/unsafeheader                                        from internal/reflectlite+
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
@@ -1099,6 +1200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         log/internal                                                 from log+
         log/slog                                                     from github.com/go-logr/logr+
         log/slog/internal                                            from log/slog
+        log/slog/internal/buffer                                     from log/slog
         maps                                                         from sigs.k8s.io/controller-runtime/pkg/predicate+
         math                                                         from archive/tar+
         math/big                                                     from crypto/dsa+
@@ -1110,15 +1212,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         mime/quotedprintable                                         from mime/multipart
         net                                                          from crypto/tls+
         net/http                                                     from expvar+
-        net/http/httptest                                            from tailscale.com/control/controlclient
         net/http/httptrace                                           from github.com/prometheus-community/pro-bing+
         net/http/httputil                                            from github.com/aws/smithy-go/transport/http+
         net/http/internal                                            from net/http+
+        net/http/internal/ascii                                      from net/http+
         net/http/pprof                                               from sigs.k8s.io/controller-runtime/pkg/manager+
         net/netip                                                    from github.com/gaissmai/bart+
         net/textproto                                                from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
         net/url                                                      from crypto/x509+
-        os                                                           from crypto/rand+
+        os                                                           from crypto/internal/sysrand+
         os/exec                                                      from github.com/aws/aws-sdk-go-v2/credentials/processcreds+
         os/signal                                                    from sigs.k8s.io/controller-runtime/pkg/manager/signals
         os/user                                                      from archive/tar+
@@ -1127,6 +1229,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints+
         regexp/syntax                                                from regexp
+        runtime                                                      from archive/tar+
         runtime/debug                                                from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof+
@@ -1146,3 +1249,5 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
         unique                                                       from net/netip
+        unsafe                                                       from bytes+
+        weak                                                         from unique

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -63,7 +63,10 @@ rules:
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["get","list","watch"]
+  verbs: ["get","list","watch", "update"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["update"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
@@ -72,7 +75,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
-  verbs: ["get", "create", "patch", "update", "list", "watch"]
+  verbs: ["get", "create", "patch", "update", "list", "watch", "deletecollection"]
 - apiGroups: ["monitoring.coreos.com"]
   resources: ["servicemonitors"]
   verbs: ["get", "list", "update", "create", "delete"]

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2215,6 +2215,22 @@ spec:
                         https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
+                useLetsEncryptStagingEnvironment:
+                  description: |-
+                    Set UseLetsEncryptStagingEnvironment to true to issue TLS
+                    certificates for any HTTPS endpoints exposed to the tailnet from
+                    LetsEncrypt's staging environment.
+                    https://letsencrypt.org/docs/staging-environment/
+                    This setting only affects Tailscale Ingress resources.
+                    By default Ingress TLS certificates are issued from LetsEncrypt's
+                    production environment.
+                    Changing this setting true -> false, will result in any
+                    existing certs being re-issued from the production environment.
+                    Changing this setting false (default) -> true, when certs have already
+                    been provisioned from production environment will NOT result in certs
+                    being re-issued from the staging environment before they need to be
+                    renewed.
+                  type: boolean
             status:
               description: |-
                 Status of the ProxyClass. This is set and managed automatically.

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -103,7 +103,7 @@ spec:
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
                   description: |-
-                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                    Type of the ProxyGroup proxies. Currently the only supported type is egress.
                     Type is immutable once a ProxyGroup is created.
                   type: string
                   enum:

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2685,6 +2685,22 @@ spec:
                                             Defaults to false.
                                         type: boolean
                                 type: object
+                            useLetsEncryptStagingEnvironment:
+                                description: |-
+                                    Set UseLetsEncryptStagingEnvironment to true to issue TLS
+                                    certificates for any HTTPS endpoints exposed to the tailnet from
+                                    LetsEncrypt's staging environment.
+                                    https://letsencrypt.org/docs/staging-environment/
+                                    This setting only affects Tailscale Ingress resources.
+                                    By default Ingress TLS certificates are issued from LetsEncrypt's
+                                    production environment.
+                                    Changing this setting true -> false, will result in any
+                                    existing certs being re-issued from the production environment.
+                                    Changing this setting false (default) -> true, when certs have already
+                                    been provisioned from production environment will NOT result in certs
+                                    being re-issued from the staging environment before they need to be
+                                    renewed.
+                                type: boolean
                         type: object
                     status:
                         description: |-
@@ -2860,7 +2876,7 @@ spec:
                                 type: array
                             type:
                                 description: |-
-                                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                                    Type of the ProxyGroup proxies. Currently the only supported type is egress.
                                     Type is immutable once a ProxyGroup is created.
                                 enum:
                                     - egress
@@ -4854,6 +4870,13 @@ rules:
         - get
         - list
         - watch
+        - update
+    - apiGroups:
+        - ""
+      resources:
+        - pods/status
+      verbs:
+        - update
     - apiGroups:
         - apps
       resources:
@@ -4891,6 +4914,7 @@ rules:
         - update
         - list
         - watch
+        - deletecollection
     - apiGroups:
         - monitoring.coreos.com
       resources:

cmd/k8s-operator/dnsrecords_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 )
@@ -163,10 +164,10 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 			Name:      o.GetName(),
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      o.GetName(),
-				LabelParentNamespace: o.GetNamespace(),
-				LabelParentType:      typ,
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        o.GetName(),
+				LabelParentNamespace:   o.GetNamespace(),
+				LabelParentType:        typ,
 			},
 		},
 		Spec: corev1.ServiceSpec{

cmd/k8s-operator/e2e/main_test.go

@@ -21,7 +21,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/internal/client/tailscale"
 )
 
 const (
@@ -64,7 +64,6 @@ func TestMain(m *testing.M) {
 func runTests(m *testing.M) (int, error) {
 	zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar()
 	logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
-	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" {
 		cleanup, err := setupClientAndACLs()

cmd/k8s-operator/egress-eps.go

@@ -20,7 +20,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
 	"tailscale.com/kube/egressservices"
 	"tailscale.com/types/ptr"
 )
@@ -71,25 +70,27 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	if err != nil {
 		return res, fmt.Errorf("error retrieving ExternalName Service: %w", err)
 	}
-	if !tsoperator.EgressServiceIsValidAndConfigured(svc) {
-		l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name)
-		return res, nil
-	}
 
 	// TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is
 	// wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to
 	// determine if a reconcile is needed.
 
 	oldEps := eps.DeepCopy()
-	proxyGroupName := eps.Labels[labelProxyGroup]
 	tailnetSvc := tailnetSvcName(svc)
 	l = l.With("tailnet-service-name", tailnetSvc)
 
 	// Retrieve the desired tailnet service configuration from the ConfigMap.
+	proxyGroupName := eps.Labels[labelProxyGroup]
 	_, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace)
 	if err != nil {
 		return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err)
 	}
+	if cfgs == nil {
+		// TODO(irbekrm): this path would be hit if egress service was once exposed on a ProxyGroup that later
+		// got deleted. Probably the EndpointSlices then need to be deleted too- need to rethink this flow.
+		l.Debugf("No egress config found, likely because ProxyGroup has not been created")
+		return res, nil
+	}
 	cfg, ok := (*cfgs)[tailnetSvc]
 	if !ok {
 		l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc)

cmd/k8s-operator/egress-pod-readiness.go

@@ -0,0 +1,274 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/tstime"
+	"tailscale.com/util/httpm"
+)
+
+const tsEgressReadinessGate = "tailscale.com/egress-services"
+
+// egressPodsReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods.
+// The condition is used as a readiness gate for the Pod, meaning that kubelet will not mark the Pod as ready before the
+// condition is set. The ProxyGroup StatefulSet updates are rolled out in such a way that no Pod is restarted, before
+// the previous Pod is marked as ready, so ensuring that the Pod does not get marked as ready when it is not yet able to
+// route traffic for egress service prevents downtime during restarts caused by no available endpoints left because
+// every Pod has been recreated and is not yet added to endpoints.
+// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate
+type egressPodsReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	tsNamespace string
+	clock       tstime.Clock
+	httpClient  doer          // http client that can be set to a mock client in tests
+	maxBackoff  time.Duration // max backoff period between health check calls
+}
+
+// Reconcile reconciles an egress ProxyGroup Pods on changes to those Pods and ProxyGroup EndpointSlices. It ensures
+// that for each Pod who is ready to route traffic to all egress services for the ProxyGroup, the Pod has a
+// tailscale.com/egress-services condition to set, so that kubelet will mark the Pod as ready.
+//
+// For the Pod to be ready
+// to route traffic to the egress service, the kube proxy needs to have set up the Pod's IP as an endpoint for the
+// ClusterIP Service corresponding to the egress service.
+//
+// Note that the endpoints for the ClusterIP Service are configured by the operator itself using custom
+// EndpointSlices(egress-eps-reconciler), so the routing is not blocked on Pod's readiness.
+//
+// Each egress service has a corresponding ClusterIP Service, that exposes all user configured
+// tailnet ports, as well as a health check port for the proxy.
+//
+// The reconciler calls the health check endpoint of each Service up to N number of times, where N is the number of
+// replicas for the ProxyGroup x 3, and checks if the received response is healthy response from the Pod being reconciled.
+//
+// The health check response contains a header with the
+// Pod's IP address- this is used to determine whether the response is received from this Pod.
+//
+// If the Pod does not appear to be serving the health check endpoint (pre-v1.80 proxies), the reconciler just sets the
+// readiness condition for backwards compatibility reasons.
+func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := er.logger.With("Pod", req.NamespacedName)
+	l.Debugf("starting reconcile")
+	defer l.Debugf("reconcile finished")
+
+	pod := new(corev1.Pod)
+	err = er.Get(ctx, req.NamespacedName, pod)
+	if apierrors.IsNotFound(err) {
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get Pod: %w", err)
+	}
+	if !pod.DeletionTimestamp.IsZero() {
+		l.Debugf("Pod is being deleted, do nothing")
+		return res, nil
+	}
+	if pod.Labels[LabelParentType] != proxyTypeProxyGroup {
+		l.Infof("[unexpected] reconciler called for a Pod that is not a ProxyGroup Pod")
+		return res, nil
+	}
+
+	// If the Pod does not have the readiness gate set, there is no need to add the readiness condition. In practice
+	// this will happen if the user has configured custom TS_LOCAL_ADDR_PORT, thus disabling the graceful failover.
+	if !slices.ContainsFunc(pod.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool {
+		return r.ConditionType == tsEgressReadinessGate
+	}) {
+		l.Debug("Pod does not have egress readiness gate set, skipping")
+		return res, nil
+	}
+
+	proxyGroupName := pod.Labels[LabelParentName]
+	pg := new(tsapi.ProxyGroup)
+	if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil {
+		return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err)
+	}
+	if pg.Spec.Type != typeEgress {
+		l.Infof("[unexpected] reconciler called for %q ProxyGroup Pod", pg.Spec.Type)
+		return res, nil
+	}
+	// Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup.
+	lbls := map[string]string{
+		kubetypes.LabelManaged: "true",
+		labelProxyGroup:        proxyGroupName,
+		labelSvcType:           typeEgress,
+	}
+	svcs := &corev1.ServiceList{}
+	if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil {
+		return res, fmt.Errorf("error listing ClusterIP Services")
+	}
+
+	idx := xslices.IndexFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
+		return c.Type == tsEgressReadinessGate
+	})
+	if idx != -1 {
+		l.Debugf("Pod is already ready, do nothing")
+		return res, nil
+	}
+
+	var routesMissing atomic.Bool
+	errChan := make(chan error, len(svcs.Items))
+	for _, svc := range svcs.Items {
+		s := svc
+		go func() {
+			ll := l.With("service_name", s.Name)
+			d := retrieveClusterDomain(er.tsNamespace, ll)
+			healthCheckAddr := healthCheckForSvc(&s, d)
+			if healthCheckAddr == "" {
+				ll.Debugf("ClusterIP Service does not expose a health check endpoint, unable to verify if routing is set up")
+				errChan <- nil
+				return
+			}
+
+			var routesSetup bool
+			bo := backoff.NewBackoff(s.Name, ll.Infof, er.maxBackoff)
+			for range numCalls(pgReplicas(pg)) {
+				if ctx.Err() != nil {
+					errChan <- nil
+					return
+				}
+				state, err := er.lookupPodRouteViaSvc(ctx, pod, healthCheckAddr, ll)
+				if err != nil {
+					errChan <- fmt.Errorf("error validating if routing has been set up for Pod: %w", err)
+					return
+				}
+				if state == healthy || state == cannotVerify {
+					routesSetup = true
+					break
+				}
+				if state == unreachable || state == unhealthy || state == podNotReady {
+					bo.BackOff(ctx, errors.New("backoff"))
+				}
+			}
+			if !routesSetup {
+				ll.Debugf("Pod is not yet configured as Service endpoint")
+				routesMissing.Store(true)
+			}
+			errChan <- nil
+		}()
+	}
+	for range len(svcs.Items) {
+		e := <-errChan
+		err = errors.Join(err, e)
+	}
+	if err != nil {
+		return res, fmt.Errorf("error verifying conectivity: %w", err)
+	}
+	if rm := routesMissing.Load(); rm {
+		l.Info("Pod is not yet added as an endpoint for all egress targets, waiting...")
+		return reconcile.Result{RequeueAfter: shortRequeue}, nil
+	}
+	if err := er.setPodReady(ctx, pod, l); err != nil {
+		return res, fmt.Errorf("error setting Pod as ready: %w", err)
+	}
+	return res, nil
+}
+
+func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, l *zap.SugaredLogger) error {
+	if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
+		return c.Type == tsEgressReadinessGate
+	}) {
+		return nil
+	}
+	l.Infof("Pod is ready to route traffic to all egress targets")
+	pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
+		Type:               tsEgressReadinessGate,
+		Status:             corev1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: er.clock.Now()},
+	})
+	return er.Status().Update(ctx, pod)
+}
+
+// healthCheckState is the result of a single request to an egress Service health check endpoint with a goal to hit a
+// specific backend Pod.
+type healthCheckState int8
+
+const (
+	cannotVerify healthCheckState = iota // not verifiable for this setup (i.e earlier proxy version)
+	unreachable                          // no backends or another network error
+	notFound                             // hit another backend
+	unhealthy                            // not 200
+	podNotReady                          // Pod is not ready, i.e does not have an IP address yet
+	healthy                              // 200
+)
+
+// lookupPodRouteViaSvc attempts to reach a Pod using a health check endpoint served by a Service and returns the state of the health check.
+func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, l *zap.SugaredLogger) (healthCheckState, error) {
+	if !slices.ContainsFunc(pod.Spec.Containers[0].Env, func(e corev1.EnvVar) bool {
+		return e.Name == "TS_ENABLE_HEALTH_CHECK" && e.Value == "true"
+	}) {
+		l.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service")
+		return cannotVerify, nil
+	}
+	wantsIP, err := podIPv4(pod)
+	if err != nil {
+		return -1, fmt.Errorf("error determining Pod's IP address: %w", err)
+	}
+	if wantsIP == "" {
+		return podNotReady, nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, time.Second*3)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, healthCheckAddr, nil)
+	if err != nil {
+		return -1, fmt.Errorf("error creating new HTTP request: %w", err)
+	}
+	// Do not re-use the same connection for the next request so to maximize the chance of hitting all backends equally.
+	req.Close = true
+	resp, err := er.httpClient.Do(req)
+	if err != nil {
+		// This is most likely because this is the first Pod and is not yet added to Service endoints. Other
+		// error types are possible, but checking for those would likely make the system too fragile.
+		return unreachable, nil
+	}
+	defer resp.Body.Close()
+	gotIP := resp.Header.Get(kubetypes.PodIPv4Header)
+	if gotIP == "" {
+		l.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service")
+		return cannotVerify, nil
+	}
+	if !strings.EqualFold(wantsIP, gotIP) {
+		return notFound, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return unhealthy, nil
+	}
+	return healthy, nil
+}
+
+// numCalls return the number of times an endpoint on a ProxyGroup Service should be called till it can be safely
+// assumed that, if none of the responses came back from a specific Pod then traffic for the Service is currently not
+// being routed to that Pod. This assumes that traffic for the Service is routed via round robin, so
+// InternalTrafficPolicy must be 'Cluster' and session affinity must be None.
+func numCalls(replicas int32) int32 {
+	return replicas * 3
+}
+
+// doer is an interface for HTTP client that can be set to a mock client in tests.
+type doer interface {
+	Do(*http.Request) (*http.Response, error)
+}

cmd/k8s-operator/egress-pod-readiness_test.go

@@ -0,0 +1,525 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
+)
+
+func TestEgressPodReadiness(t *testing.T) {
+	// We need to pass a Pod object to WithStatusSubresource because of some quirks in how the fake client
+	// works. Without this code we would not be able to update Pod's status further down.
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithStatusSubresource(&corev1.Pod{}).
+		Build()
+	zl, _ := zap.NewDevelopment()
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	rec := &egressPodsReconciler{
+		tsNamespace: "operator-ns",
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       cl,
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "dev",
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type:     "egress",
+			Replicas: ptr.To(int32(3)),
+		},
+	}
+	mustCreate(t, fc, pg)
+	podIP := "10.0.0.2"
+	podTemplate := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "operator-ns",
+			Name:      "pod",
+			Labels: map[string]string{
+				LabelParentType: "proxygroup",
+				LabelParentName: "dev",
+			},
+		},
+		Spec: corev1.PodSpec{
+			ReadinessGates: []corev1.PodReadinessGate{{
+				ConditionType: tsEgressReadinessGate,
+			}},
+			Containers: []corev1.Container{{
+				Name: "tailscale",
+				Env: []corev1.EnvVar{{
+					Name:  "TS_ENABLE_HEALTH_CHECK",
+					Value: "true",
+				}},
+			}},
+		},
+		Status: corev1.PodStatus{
+			PodIPs: []corev1.PodIP{{IP: podIP}},
+		},
+	}
+
+	t.Run("no_egress_services", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		mustCreate(t, fc, pod)
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod)
+	})
+	t.Run("one_svc_already_routed_to", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		resp := readyResps(podIP, 1)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resp},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+
+		// A subsequent reconcile should not change the Pod.
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+		expectEqual(t, fc, pod)
+
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_many_backends_eventually_routed_to", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only
+		// return with the right Pod IP.
+		resps := append(readyResps("10.0.0.3", 4), append(readyResps("10.0.0.4", 4), readyResps(podIP, 1)...)...)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resps},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_eventually_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only
+		// return with 200 status code.
+		resps := append(unreadyResps(podIP, 8), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resps},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_never_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times and Pod should be
+		// requeued if neither of those succeed.
+		resps := readyResps("10.0.0.3", 9)
+		httpCl := fakeHTTPClient{
+			t:     t,
+			state: map[string][]fakeResponse{hep: resps},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_many_backends_already_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		resps := readyResps(podIP, 1)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps,
+				hep3: resps,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_eventually_routable_and_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...)
+		resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...)
+		resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_never_routable_and_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
+		// will be requeued if neither succeeds.
+		resps := readyResps("10.0.0.3", 9)
+		resps2 := append(readyResps("10.0.0.3", 5), readyResps("10.0.0.4", 4)...)
+		resps3 := unreadyResps(podIP, 9)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_one_never_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
+		// will be requeued if any one never succeeds.
+		resps := readyResps(podIP, 9)
+		resps2 := readyResps(podIP, 9)
+		resps3 := append(readyResps("10.0.0.3", 5), readyResps("10.0.0.4", 4)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_one_never_healthy", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		svc2, hep2 := newSvc("svc-2", 9002)
+		svc3, hep3 := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
+		// will be requeued if any one never succeeds.
+		resps := readyResps(podIP, 9)
+		resps2 := unreadyResps(podIP, 9)
+		resps3 := readyResps(podIP, 9)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("one_svc_many_backends_different_ports_eventually_healthy_and_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9003)
+		svc2, hep2 := newSvc("svc-2", 9004)
+		svc3, hep3 := newSvc("svc-3", 9010)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried up to 9 times and
+		// marked as success as soon as one try succeeds.
+		resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...)
+		resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...)
+		resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep:  resps,
+				hep2: resps2,
+				hep3: resps3,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	// Proxies of 1.78 and earlier did not set the Pod IP header.
+	t.Run("pod_does_not_return_ip_header", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		pod.Name = "foo-bar"
+
+		svc, hep := newSvc("foo-bar", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// If a response does not contain Pod IP header, we assume that this is an earlier proxy version,
+		// readiness cannot be verified so the readiness gate is just set to true.
+		resps := unreadyResps("", 1)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep: resps,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_eventually_healthy_and_routable", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		svc, hep := newSvc("svc", 9002)
+		mustCreateAll(t, fc, svc, pod)
+		// If a response errors, it is probably because the Pod is not yet properly running, so retry.
+		resps := append(erroredResps(8), readyResps(podIP, 1)...)
+		httpCl := fakeHTTPClient{
+			t: t,
+			state: map[string][]fakeResponse{
+				hep: resps,
+			},
+		}
+		rec.httpClient = &httpCl
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("one_svc_one_backend_svc_does_not_have_health_port", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+
+		// If a Service does not have health port set, we assume that it is not possible to determine Pod's
+		// readiness and set it to ready.
+		svc, _ := newSvc("svc", -1)
+		mustCreateAll(t, fc, svc, pod)
+		rec.httpClient = nil
+		expectReconciled(t, rec, "operator-ns", pod.Name)
+
+		// Pod should have readiness gate condition set.
+		podSetReady(pod, cl)
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc)
+	})
+	t.Run("error_setting_up_healthcheck", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		// This is not a realistic reason for error, but we are just testing the behaviour of a healthcheck
+		// lookup failing.
+		pod.Status.PodIPs = []corev1.PodIP{{IP: "not-an-ip"}}
+
+		svc, _ := newSvc("svc", 9002)
+		svc2, _ := newSvc("svc-2", 9002)
+		svc3, _ := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		rec.httpClient = nil
+		expectError(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+	t.Run("pod_does_not_have_an_ip_address", func(t *testing.T) {
+		pod := podTemplate.DeepCopy()
+		pod.Status.PodIPs = nil
+
+		svc, _ := newSvc("svc", 9002)
+		svc2, _ := newSvc("svc-2", 9002)
+		svc3, _ := newSvc("svc-3", 9002)
+		mustCreateAll(t, fc, svc, svc2, svc3, pod)
+		rec.httpClient = nil
+		expectRequeue(t, rec, "operator-ns", pod.Name)
+
+		// Pod should not have readiness gate condition set.
+		expectEqual(t, fc, pod)
+		mustDeleteAll(t, fc, pod, svc, svc2, svc3)
+	})
+}
+
+func readyResps(ip string, num int) (resps []fakeResponse) {
+	for range num {
+		resps = append(resps, fakeResponse{statusCode: 200, podIP: ip})
+	}
+	return resps
+}
+
+func unreadyResps(ip string, num int) (resps []fakeResponse) {
+	for range num {
+		resps = append(resps, fakeResponse{statusCode: 503, podIP: ip})
+	}
+	return resps
+}
+
+func erroredResps(num int) (resps []fakeResponse) {
+	for range num {
+		resps = append(resps, fakeResponse{err: errors.New("timeout")})
+	}
+	return resps
+}
+
+func newSvc(name string, port int32) (*corev1.Service, string) {
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "operator-ns",
+			Name:      name,
+			Labels: map[string]string{
+				kubetypes.LabelManaged: "true",
+				labelProxyGroup:        "dev",
+				labelSvcType:           typeEgress,
+			},
+		},
+		Spec: corev1.ServiceSpec{},
+	}
+	if port != -1 {
+		svc.Spec.Ports = []corev1.ServicePort{
+			{
+				Name:       tsHealthCheckPortName,
+				Port:       port,
+				TargetPort: intstr.FromInt(9002),
+				Protocol:   "TCP",
+			},
+		}
+	}
+	return svc, fmt.Sprintf("http://%s.operator-ns.svc.cluster.local:%d/healthz", name, port)
+}
+
+func podSetReady(pod *corev1.Pod, cl *tstest.Clock) {
+	pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
+		Type:               tsEgressReadinessGate,
+		Status:             corev1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+	})
+}
+
+// fakeHTTPClient is a mock HTTP client with a preset map of request URLs to list of responses. When it receives a
+// request for a specific URL, it returns the preset response for that URL. It errors if an unexpected request is
+// received.
+type fakeHTTPClient struct {
+	t     *testing.T
+	mu    sync.Mutex // protects following
+	state map[string][]fakeResponse
+}
+
+func (f *fakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	f.mu.Lock()
+	resps := f.state[req.URL.String()]
+	if len(resps) == 0 {
+		f.mu.Unlock()
+		log.Printf("\n\n\nURL %q\n\n\n", req.URL)
+		f.t.Fatalf("fakeHTTPClient received an unexpected request for %q", req.URL)
+	}
+	defer func() {
+		if len(resps) == 1 {
+			delete(f.state, req.URL.String())
+			f.mu.Unlock()
+			return
+		}
+		f.state[req.URL.String()] = f.state[req.URL.String()][1:]
+		f.mu.Unlock()
+	}()
+
+	resp := resps[0]
+	if resp.err != nil {
+		return nil, resp.err
+	}
+	r := http.Response{
+		StatusCode: resp.statusCode,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(bytes.NewReader([]byte{})),
+	}
+	r.Header.Add(kubetypes.PodIPv4Header, resp.podIP)
+	return &r, nil
+}
+
+type fakeResponse struct {
+	err        error
+	statusCode int
+	podIP      string // for the Pod IP header
+}

cmd/k8s-operator/egress-services-readiness.go

@@ -48,11 +48,12 @@ type egressSvcsReadinessReconciler struct {
 // service to determine how many replicas are currently able to route traffic.
 func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	l := esrr.logger.With("Service", req.NamespacedName)
-	defer l.Info("reconcile finished")
+	l.Debugf("starting reconcile")
+	defer l.Debugf("reconcile finished")
 
 	svc := new(corev1.Service)
 	if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
-		l.Info("Service not found")
+		l.Debugf("Service not found")
 		return res, nil
 	} else if err != nil {
 		return res, fmt.Errorf("failed to get Service: %w", err)
@@ -127,16 +128,16 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
 			return res, err
 		}
 		if pod == nil {
-			l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
+			l.Warnf("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
 			reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
 			return res, nil
 		}
-		l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
+		l.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs)
 		ready := false
 		for _, ep := range eps.Endpoints {
-			l.Infof("looking at endpoint with addresses %v", ep.Addresses)
+			l.Debugf("looking at endpoint with addresses %v", ep.Addresses)
 			if endpointReadyForPod(&ep, pod, l) {
-				l.Infof("endpoint is ready for Pod")
+				l.Debugf("endpoint is ready for Pod")
 				ready = true
 				break
 			}
@@ -165,7 +166,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
 func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
 	podIP, err := podIPv4(pod)
 	if err != nil {
-		l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
+		l.Warnf("[unexpected] error retrieving Pod's IPv4 address: %v", err)
 		return false
 	}
 	// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.

cmd/k8s-operator/egress-services.go

@@ -59,6 +59,8 @@ const (
 	maxPorts = 1000
 
 	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
+
+	tsHealthCheckPortName = "tailscale-health-check"
 )
 
 var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
@@ -229,15 +231,16 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 		found := false
 		for _, wantsPM := range svc.Spec.Ports {
 			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
-				// We don't use the port name to distinguish this port internally, but Kubernetes
-				// require that, for Service ports with more than one name each port is uniquely named.
-				// So we can always pick the port name from the ExternalName Service as at this point we
-				// know that those are valid names because Kuberentes already validated it once. Note
-				// that users could have changed an unnamed port to a named port and might have changed
-				// port names- this should still work.
+				// We want to both preserve the user set port names for ease of debugging, but also
+				// ensure that we name all unnamed ports as the ClusterIP Service that we create will
+				// always have at least two ports.
 				// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
 				// See also https://github.com/tailscale/tailscale/issues/13406#issuecomment-2507230388
-				clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
+				if wantsPM.Name != "" {
+					clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
+				} else {
+					clusterIPSvc.Spec.Ports[i].Name = "tailscale-unnamed"
+				}
 				found = true
 				break
 			}
@@ -252,6 +255,12 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 	// ClusterIP Service produce new target port and add a portmapping to
 	// the ClusterIP Service.
 	for _, wantsPM := range svc.Spec.Ports {
+		// Because we add a healthcheck port of our own, we will always have at least two ports. That
+		// means that we cannot have ports with name not set.
+		// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
+		if wantsPM.Name == "" {
+			wantsPM.Name = "tailscale-unnamed"
+		}
 		found := false
 		for _, gotPM := range clusterIPSvc.Spec.Ports {
 			if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) {
@@ -278,6 +287,25 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 			})
 		}
 	}
+	var healthCheckPort int32 = defaultLocalAddrPort
+
+	for {
+		if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool {
+			return p.Port == healthCheckPort
+		}) {
+			break
+		}
+		healthCheckPort++
+		if healthCheckPort > 10002 {
+			return nil, false, fmt.Errorf("unable to find a free port for internal health check in range [9002, 10002]")
+		}
+	}
+	clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
+		Name:       tsHealthCheckPortName,
+		Port:       healthCheckPort,
+		TargetPort: intstr.FromInt(defaultLocalAddrPort),
+		Protocol:   "TCP",
+	})
 	if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
 		if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
 			svc.Labels = clusterIPSvc.Labels
@@ -320,7 +348,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 	}
 	tailnetSvc := tailnetSvcName(svc)
 	gotCfg := (*cfgs)[tailnetSvc]
-	wantsCfg := egressSvcCfg(svc, clusterIPSvc)
+	wantsCfg := egressSvcCfg(svc, clusterIPSvc, esr.tsNamespace, l)
 	if !reflect.DeepEqual(gotCfg, wantsCfg) {
 		l.Debugf("updating egress services ConfigMap %s", cm.Name)
 		mak.Set(cfgs, tailnetSvc, wantsCfg)
@@ -504,10 +532,8 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		return false, nil
 	}
 	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
 	}
 
 	l.Debugf("egress service is valid")
@@ -515,6 +541,24 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 	return true, nil
 }
 
+func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, l *zap.SugaredLogger) egressservices.Config {
+	d := retrieveClusterDomain(ns, l)
+	tt := tailnetTargetFromSvc(externalNameSvc)
+	hep := healthCheckForSvc(clusterIPSvc, d)
+	cfg := egressservices.Config{
+		TailnetTarget:       tt,
+		HealthCheckEndpoint: hep,
+	}
+	for _, svcPort := range clusterIPSvc.Spec.Ports {
+		if svcPort.Name == tsHealthCheckPortName {
+			continue // exclude healthcheck from egress svcs configs
+		}
+		pm := portMap(svcPort)
+		mak.Set(&cfg.Ports, pm, struct{}{})
+	}
+	return cfg
+}
+
 func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
 	violations := validateService(svc)
 
@@ -584,19 +628,13 @@ func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
 	}
 }
 
-func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config {
-	tt := tailnetTargetFromSvc(externalNameSvc)
-	cfg := egressservices.Config{TailnetTarget: tt}
-	for _, svcPort := range clusterIPSvc.Spec.Ports {
-		pm := portMap(svcPort)
-		mak.Set(&cfg.Ports, pm, struct{}{})
-	}
-	return cfg
-}
-
 func portMap(p corev1.ServicePort) egressservices.PortMap {
 	// TODO (irbekrm): out of bounds check?
-	return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)}
+	return egressservices.PortMap{
+		Protocol:   string(p.Protocol),
+		MatchPort:  uint16(p.TargetPort.IntVal),
+		TargetPort: uint16(p.Port),
+	}
 }
 
 func isEgressSvcForProxyGroup(obj client.Object) bool {
@@ -618,7 +656,11 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 			Namespace: tsNamespace,
 		},
 	}
-	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
+	err = cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)
+	if apierrors.IsNotFound(err) { // ProxyGroup resources have not been created (yet)
+		return nil, nil, nil
+	}
+	if err != nil {
 		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
 	}
 	cfgs = &egressservices.Configs{}
@@ -638,12 +680,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 // should probably validate and truncate (?) the names is they are too long.
 func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentType:      "svc",
-		LabelParentName:      svc.Name,
-		LabelParentNamespace: svc.Namespace,
-		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
-		labelSvcType:         typeEgress,
+		kubetypes.LabelManaged: "true",
+		LabelParentType:        "svc",
+		LabelParentName:        svc.Name,
+		LabelParentNamespace:   svc.Namespace,
+		labelProxyGroup:        svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:           typeEgress,
 	}
 }
 
@@ -740,3 +782,17 @@ func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.
 	svc.Status = *st
 	return err
 }
+
+// healthCheckForSvc return the URL of the containerboot's health check endpoint served by this Service or empty string.
+func healthCheckForSvc(svc *corev1.Service, clusterDomain string) string {
+	// This version of the operator always sets health check port on the egress Services. However, it is possible
+	// that this reconcile loops runs during a proxy upgrade from a version that did not set the health check port
+	// and parses a Service that does not have the port set yet.
+	i := slices.IndexFunc(svc.Spec.Ports, func(port corev1.ServicePort) bool {
+		return port.Name == tsHealthCheckPortName
+	})
+	if i == -1 {
+		return ""
+	}
+	return fmt.Sprintf("http://%s.%s.svc.%s:%d/healthz", svc.Name, svc.Namespace, clusterDomain, svc.Spec.Ports[i].Port)
+}

cmd/k8s-operator/egress-services_test.go

@@ -18,6 +18,7 @@ import (
 	discoveryv1 "k8s.io/api/discovery/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
@@ -78,42 +79,16 @@ func TestTailscaleEgressServices(t *testing.T) {
 			Selector:     nil,
 			Ports: []corev1.ServicePort{
 				{
-					Name:     "http",
 					Protocol: "TCP",
 					Port:     80,
 				},
-				{
-					Name:     "https",
-					Protocol: "TCP",
-					Port:     443,
-				},
 			},
 		},
 	}
 
-	t.Run("proxy_group_not_ready", func(t *testing.T) {
+	t.Run("service_one_unnamed_port", func(t *testing.T) {
 		mustCreate(t, fc, svc)
 		expectReconciled(t, esr, "default", "test")
-		// Service should have EgressSvcValid condition set to Unknown.
-		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
-		expectEqual(t, fc, svc)
-	})
-
-	t.Run("proxy_group_ready", func(t *testing.T) {
-		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
-			pg.Status.Conditions = []metav1.Condition{
-				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
-			}
-		})
-		expectReconciled(t, esr, "default", "test")
-		validateReadyService(t, fc, esr, svc, clock, zl, cm)
-	})
-	t.Run("service_retain_one_unnamed_port", func(t *testing.T) {
-		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80}}
-		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
-			s.Spec.Ports = svc.Spec.Ports
-		})
-		expectReconciled(t, esr, "default", "test")
 		validateReadyService(t, fc, esr, svc, clock, zl, cm)
 	})
 	t.Run("service_add_two_named_ports", func(t *testing.T) {
@@ -164,7 +139,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco
 	// Verify that an EndpointSlice has been created.
 	expectEqual(t, fc, endpointSlice(name, svc, clusterSvc))
 	// Verify that ConfigMap contains configuration for the new egress service.
-	mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
+	mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl)
 	r := svcConfiguredReason(svc, true, zl.Sugar())
 	// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
 	// CluterIP Service.
@@ -203,6 +178,23 @@ func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *c
 
 func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
 	labels := egressSvcChildResourceLabels(extNSvc)
+	ports := make([]corev1.ServicePort, len(extNSvc.Spec.Ports))
+	for i, port := range extNSvc.Spec.Ports {
+		ports[i] = corev1.ServicePort{ // Copy the port to avoid modifying the original.
+			Name:     port.Name,
+			Port:     port.Port,
+			Protocol: port.Protocol,
+		}
+		if port.Name == "" {
+			ports[i].Name = "tailscale-unnamed"
+		}
+	}
+	ports = append(ports, corev1.ServicePort{
+		Name:       "tailscale-health-check",
+		Port:       9002,
+		TargetPort: intstr.FromInt(9002),
+		Protocol:   "TCP",
+	})
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:         name,
@@ -212,7 +204,7 @@ func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
 		},
 		Spec: corev1.ServiceSpec{
 			Type:  corev1.ServiceTypeClusterIP,
-			Ports: extNSvc.Spec.Ports,
+			Ports: ports,
 		},
 	}
 }
@@ -257,9 +249,9 @@ func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
 	return ports
 }
 
-func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
+func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap, l *zap.Logger) {
 	t.Helper()
-	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
+	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc, clusterIPSvc.Namespace, l.Sugar())
 	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
 		t.Fatalf("Error retrieving ConfigMap: %v", err)
 	}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -8,6 +8,11 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"net/http"
+	"reflect"
 	"testing"
 
 	"slices"
@@ -15,84 +20,25 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"tailscale.com/internal/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 )
 
 func TestIngressPGReconciler(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{
-		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
-		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
-	}
+	ingPGR, fc, ft := setupIngressTest(t)
 
-	// Pre-create the ProxyGroup
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:       "test-pg",
-			Generation: 1,
-		},
-		Spec: tsapi.ProxyGroupSpec{
-			Type: tsapi.ProxyGroupTypeIngress,
-		},
-	}
-
-	// Pre-create the ConfigMap for the ProxyGroup
-	pgConfigMap := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-pg-ingress-config",
-			Namespace: "operator-ns",
-		},
-		BinaryData: map[string][]byte{
-			"serve-config.json": []byte(`{"Services":{}}`),
-		},
-	}
-
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pg, pgConfigMap, tsIngressClass).
-		WithStatusSubresource(pg).
-		Build()
-	mustUpdateStatus(t, fc, "", pg.Name, func(pg *tsapi.ProxyGroup) {
-		pg.Status.Conditions = []metav1.Condition{
-			{
-				Type:               string(tsapi.ProxyGroupReady),
-				Status:             metav1.ConditionTrue,
-				ObservedGeneration: 1,
-			},
-		}
-	})
-	ft := &fakeTSClient{}
-	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	lc := &fakeLocalClient{
-		status: &ipnstate.Status{
-			CurrentTailnet: &ipnstate.TailnetStatus{
-				MagicDNSSuffix: "ts.net",
-			},
-		},
-	}
-	ingPGR := &IngressPGReconciler{
-		Client:      fc,
-		tsClient:    ft,
-		tsnetServer: fakeTsnetServer,
-		defaultTags: []string{"tag:k8s"},
-		tsNamespace: "operator-ns",
-		logger:      zl.Sugar(),
-		recorder:    record.NewFakeRecorder(10),
-		lc:          lc,
-	}
-
-	// Test 1: Default tags
 	ing := &networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
 		ObjectMeta: metav1.ObjectMeta{
@@ -114,7 +60,7 @@ func TestIngressPGReconciler(t *testing.T) {
 				},
 			},
 			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
+				{Hosts: []string{"my-svc"}},
 			},
 		},
 	}
@@ -122,8 +68,91 @@ func TestIngressPGReconciler(t *testing.T) {
 
 	// Verify initial reconciliation
 	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
-	// Get and verify the ConfigMap was updated
+	// Verify that Role and RoleBinding have been created for the first Ingress.
+	// Do not verify the cert Secret as that was already verified implicitly above.
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
+
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
+	})
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+
+	// Verify VIPService uses custom tags
+	vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
+	if err != nil {
+		t.Fatalf("getting VIPService: %v", err)
+	}
+	if vipSvc == nil {
+		t.Fatal("VIPService not created")
+	}
+	wantTags := []string{"tag:custom", "tag:test"} // custom tags only
+	gotTags := slices.Clone(vipSvc.Tags)
+	slices.Sort(gotTags)
+	slices.Sort(wantTags)
+	if !slices.Equal(gotTags, wantTags) {
+		t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags)
+	}
+
+	// Create second Ingress
+	ing2 := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-other-ingress",
+			Namespace: "default",
+			UID:       types.UID("5678-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-other-svc.tailnetxyz.ts.net"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing2)
+
+	// Verify second Ingress reconciliation
+	expectReconciled(t, ingPGR, "default", "my-other-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-other-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "my-other-ingress")
+	verifyServeConfig(t, fc, "svc:my-other-svc", false)
+	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
+
+	// Verify that Role and RoleBinding have been created for the first Ingress.
+	// Do not verify the cert Secret as that was already verified implicitly above.
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
+
+	// Verify first Ingress is still working
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"})
+
+	// Delete second Ingress
+	if err := fc.Delete(context.Background(), ing2); err != nil {
+		t.Fatalf("deleting second Ingress: %v", err)
+	}
+	expectReconciled(t, ingPGR, "default", "my-other-ingress")
+
+	// Verify second Ingress cleanup
 	cm := &corev1.ConfigMap{}
 	if err := fc.Get(context.Background(), types.NamespacedName{
 		Name:      "test-pg-ingress-config",
@@ -137,46 +166,21 @@ func TestIngressPGReconciler(t *testing.T) {
 		t.Fatalf("unmarshaling serve config: %v", err)
 	}
 
-	if cfg.Services["my-svc"] == nil {
-		t.Error("expected serve config to contain VIPService configuration")
+	// Verify first Ingress is still configured
+	if cfg.Services["svc:my-svc"] == nil {
+		t.Error("first Ingress service config was incorrectly removed")
+	}
+	// Verify second Ingress was cleaned up
+	if cfg.Services["svc:my-other-svc"] != nil {
+		t.Error("second Ingress service config was not cleaned up")
 	}
 
-	// Verify VIPService uses default tags
-	vipSvc, err := ft.getVIPServiceByName(context.Background(), "my-svc")
-	if err != nil {
-		t.Fatalf("getting VIPService: %v", err)
-	}
-	if vipSvc == nil {
-		t.Fatal("VIPService not created")
-	}
-	wantTags := []string{"tag:k8s"} // default tags
-	if !slices.Equal(vipSvc.Tags, wantTags) {
-		t.Errorf("incorrect VIPService tags: got %v, want %v", vipSvc.Tags, wantTags)
-	}
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
 
-	// Test 2: Custom tags
-	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
-		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
-	})
-	expectReconciled(t, ingPGR, "default", "test-ingress")
-
-	// Verify VIPService uses custom tags
-	vipSvc, err = ft.getVIPServiceByName(context.Background(), "my-svc")
-	if err != nil {
-		t.Fatalf("getting VIPService: %v", err)
-	}
-	if vipSvc == nil {
-		t.Fatal("VIPService not created")
-	}
-	wantTags = []string{"tag:custom", "tag:test"} // custom tags only
-	gotTags := slices.Clone(vipSvc.Tags)
-	slices.Sort(gotTags)
-	slices.Sort(wantTags)
-	if !slices.Equal(gotTags, wantTags) {
-		t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags)
-	}
-
-	// Delete the Ingress and verify cleanup
+	// Delete the first Ingress and verify cleanup
 	if err := fc.Delete(context.Background(), ing); err != nil {
 		t.Fatalf("deleting Ingress: %v", err)
 	}
@@ -200,6 +204,71 @@ func TestIngressPGReconciler(t *testing.T) {
 	if len(cfg.Services) > 0 {
 		t.Error("serve config not cleaned up")
 	}
+	verifyTailscaledConfig(t, fc, nil)
+
+	// Add verification that cert resources were cleaned up
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-svc.ts.net")
+}
+
+func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+
+	// Verify initial reconciliation
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+
+	// Update the Ingress hostname and make sure the original VIPService is deleted.
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		ing.Spec.TLS[0].Hosts[0] = "updated-svc"
+	})
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "updated-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:updated-svc", false)
+	verifyVIPService(t, ft, "svc:updated-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})
+
+	_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc"))
+	if err == nil {
+		t.Fatalf("svc:my-svc not cleaned up")
+	}
+	var errResp *tailscale.ErrResponse
+	if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
+		t.Fatalf("unexpected error: %v", err)
+	}
 }
 
 func TestValidateIngress(t *testing.T) {
@@ -207,6 +276,15 @@ func TestValidateIngress(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-ingress",
 			Namespace: "default",
+			Annotations: map[string]string{
+				AnnotationProxyGroup: "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"test"}},
+			},
 		},
 	}
 
@@ -230,10 +308,11 @@ func TestValidateIngress(t *testing.T) {
 	}
 
 	tests := []struct {
-		name    string
-		ing     *networkingv1.Ingress
-		pg      *tsapi.ProxyGroup
-		wantErr string
+		name         string
+		ing          *networkingv1.Ingress
+		pg           *tsapi.ProxyGroup
+		existingIngs []networkingv1.Ingress
+		wantErr      string
 	}{
 		{
 			name: "valid_ingress_with_hostname",
@@ -323,15 +402,442 @@ func TestValidateIngress(t *testing.T) {
 			},
 			wantErr: "ProxyGroup \"test-pg\" is not ready",
 		},
+		{
+			name: "duplicate_hostname",
+			ing:  baseIngress,
+			pg:   readyProxyGroup,
+			existingIngs: []networkingv1.Ingress{{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "existing-ingress",
+					Namespace: "default",
+					Annotations: map[string]string{
+						AnnotationProxyGroup: "test-pg",
+					},
+				},
+				Spec: networkingv1.IngressSpec{
+					IngressClassName: ptr.To("tailscale"),
+					TLS: []networkingv1.IngressTLS{
+						{Hosts: []string{"test"}},
+					},
+				},
+			}},
+			wantErr: `found duplicate Ingress "existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := &IngressPGReconciler{}
-			err := r.validateIngress(tt.ing, tt.pg)
+			fc := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme).
+				WithObjects(tt.ing).
+				WithLists(&networkingv1.IngressList{Items: tt.existingIngs}).
+				Build()
+			r := &HAIngressReconciler{Client: fc}
+			err := r.validateIngress(context.Background(), tt.ing, tt.pg)
 			if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) {
 				t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})
 	}
 }
+
+func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+
+	// Create test Ingress with HTTP endpoint enabled
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group":   "test-pg",
+				"tailscale.com/http-endpoint": "enabled",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc"}},
+			},
+		},
+	}
+	if err := fc.Create(context.Background(), ing); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify initial reconciliation with HTTP enabled
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyVIPService(t, ft, "svc:my-svc", []string{"80", "443"})
+	verifyServeConfig(t, fc, "svc:my-svc", true)
+
+	// Verify Ingress status
+	ing = &networkingv1.Ingress{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-ingress",
+		Namespace: "default",
+	}, ing); err != nil {
+		t.Fatal(err)
+	}
+
+	// Status will be empty until the VIPService shows up in prefs.
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress, []networkingv1.IngressLoadBalancerIngress(nil)) {
+		t.Errorf("incorrect Ingress status: got %v, want empty",
+			ing.Status.LoadBalancer.Ingress)
+	}
+
+	// Add the VIPService to prefs to have the Ingress recognised as ready.
+	mustCreate(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg-0",
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "state"),
+		},
+		Data: map[string][]byte{
+			"_current-profile": []byte("profile-foo"),
+			"profile-foo":      []byte(`{"AdvertiseServices":["svc:my-svc"],"Config":{"NodeID":"node-foo"}}`),
+		},
+	})
+
+	// Reconcile and re-fetch Ingress.
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	if err := fc.Get(context.Background(), client.ObjectKeyFromObject(ing), ing); err != nil {
+		t.Fatal(err)
+	}
+
+	wantStatus := []networkingv1.IngressPortStatus{
+		{Port: 443, Protocol: "TCP"},
+		{Port: 80, Protocol: "TCP"},
+	}
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
+		t.Errorf("incorrect status ports: got %v, want %v",
+			ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
+	}
+
+	// Remove HTTP endpoint annotation
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		delete(ing.Annotations, "tailscale.com/http-endpoint")
+	})
+
+	// Verify reconciliation after removing HTTP
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+
+	// Verify Ingress status
+	ing = &networkingv1.Ingress{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-ingress",
+		Namespace: "default",
+	}, ing); err != nil {
+		t.Fatal(err)
+	}
+
+	wantStatus = []networkingv1.IngressPortStatus{
+		{Port: 443, Protocol: "TCP"},
+	}
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
+		t.Errorf("incorrect status ports: got %v, want %v",
+			ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
+	}
+}
+
+func verifyVIPService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) {
+	t.Helper()
+	vipSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName))
+	if err != nil {
+		t.Fatalf("getting VIPService %q: %v", serviceName, err)
+	}
+	if vipSvc == nil {
+		t.Fatalf("VIPService %q not created", serviceName)
+	}
+	gotPorts := slices.Clone(vipSvc.Ports)
+	slices.Sort(gotPorts)
+	slices.Sort(wantPorts)
+	if !slices.Equal(gotPorts, wantPorts) {
+		t.Errorf("incorrect ports for VIPService %q: got %v, want %v", serviceName, gotPorts, wantPorts)
+	}
+}
+
+func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantHTTP bool) {
+	t.Helper()
+
+	cm := &corev1.ConfigMap{}
+	if err := fc.Get(context.Background(), types.NamespacedName{
+		Name:      "test-pg-ingress-config",
+		Namespace: "operator-ns",
+	}, cm); err != nil {
+		t.Fatalf("getting ConfigMap: %v", err)
+	}
+
+	cfg := &ipn.ServeConfig{}
+	if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil {
+		t.Fatalf("unmarshaling serve config: %v", err)
+	}
+
+	t.Logf("Looking for service %q in config: %+v", serviceName, cfg)
+
+	svc := cfg.Services[tailcfg.ServiceName(serviceName)]
+	if svc == nil {
+		t.Fatalf("service %q not found in serve config, services: %+v", serviceName, maps.Keys(cfg.Services))
+	}
+
+	wantHandlers := 1
+	if wantHTTP {
+		wantHandlers = 2
+	}
+
+	// Check TCP handlers
+	if len(svc.TCP) != wantHandlers {
+		t.Errorf("incorrect number of TCP handlers for service %q: got %d, want %d", serviceName, len(svc.TCP), wantHandlers)
+	}
+	if wantHTTP {
+		if h, ok := svc.TCP[uint16(80)]; !ok {
+			t.Errorf("HTTP (port 80) handler not found for service %q", serviceName)
+		} else if !h.HTTP {
+			t.Errorf("HTTP not enabled for port 80 handler for service %q", serviceName)
+		}
+	}
+	if h, ok := svc.TCP[uint16(443)]; !ok {
+		t.Errorf("HTTPS (port 443) handler not found for service %q", serviceName)
+	} else if !h.HTTPS {
+		t.Errorf("HTTPS not enabled for port 443 handler for service %q", serviceName)
+	}
+
+	// Check Web handlers
+	if len(svc.Web) != wantHandlers {
+		t.Errorf("incorrect number of Web handlers for service %q: got %d, want %d", serviceName, len(svc.Web), wantHandlers)
+	}
+}
+
+func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) {
+	t.Helper()
+	var expected string
+	if expectedServices != nil {
+		expectedServicesJSON, err := json.Marshal(expectedServices)
+		if err != nil {
+			t.Fatalf("marshaling expected services: %v", err)
+		}
+		expected = fmt.Sprintf(`,"AdvertiseServices":%s`, expectedServicesJSON)
+	}
+	expectEqual(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgConfigSecretName("test-pg", 0),
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "config"),
+		},
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
+		},
+	})
+}
+
+func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) {
+
+	tsIngressClass := &networkingv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
+		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
+	}
+
+	// Pre-create the ProxyGroup
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-pg",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type: tsapi.ProxyGroupTypeIngress,
+		},
+	}
+
+	// Pre-create the ConfigMap for the ProxyGroup
+	pgConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg-ingress-config",
+			Namespace: "operator-ns",
+		},
+		BinaryData: map[string][]byte{
+			"serve-config.json": []byte(`{"Services":{}}`),
+		},
+	}
+
+	// Pre-create a config Secret for the ProxyGroup
+	pgCfgSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgConfigSecretName("test-pg", 0),
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "config"),
+		},
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): []byte("{}"),
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pg, pgCfgSecret, pgConfigMap, tsIngressClass).
+		WithStatusSubresource(pg).
+		Build()
+
+	// Set ProxyGroup status to ready
+	pg.Status.Conditions = []metav1.Condition{
+		{
+			Type:               string(tsapi.ProxyGroupReady),
+			Status:             metav1.ConditionTrue,
+			ObservedGeneration: 1,
+		},
+	}
+	if err := fc.Status().Update(context.Background(), pg); err != nil {
+		t.Fatal(err)
+	}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lc := &fakeLocalClient{
+		status: &ipnstate.Status{
+			CurrentTailnet: &ipnstate.TailnetStatus{
+				MagicDNSSuffix: "ts.net",
+			},
+		},
+	}
+
+	ingPGR := &HAIngressReconciler{
+		Client:      fc,
+		tsClient:    ft,
+		defaultTags: []string{"tag:k8s"},
+		tsNamespace: "operator-ns",
+		tsnetServer: fakeTsnetServer,
+		logger:      zl.Sugar(),
+		recorder:    record.NewFakeRecorder(10),
+		lc:          lc,
+	}
+
+	return ingPGR, fc, ft
+}
+
+func TestIngressPGReconciler_MultiCluster(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+	ingPGR.operatorID = "operator-1"
+
+	// Create initial Ingress
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+
+	// Simulate existing VIPService from another cluster
+	existingVIPSvc := &tailscale.VIPService{
+		Name: "svc:my-svc",
+		Annotations: map[string]string{
+			ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
+		},
+	}
+	ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{
+		"svc:my-svc": existingVIPSvc,
+	}
+
+	// Verify reconciliation adds our operator reference
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+
+	vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
+	if err != nil {
+		t.Fatalf("getting VIPService: %v", err)
+	}
+	if vipSvc == nil {
+		t.Fatal("VIPService not found")
+	}
+
+	o, err := parseOwnerAnnotation(vipSvc)
+	if err != nil {
+		t.Fatalf("parsing owner annotation: %v", err)
+	}
+
+	wantOwnerRefs := []OwnerRef{
+		{OperatorID: "operator-2"},
+		{OperatorID: "operator-1"},
+	}
+	if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
+	}
+
+	// Delete the Ingress and verify VIPService still exists with one owner ref
+	if err := fc.Delete(context.Background(), ing); err != nil {
+		t.Fatalf("deleting Ingress: %v", err)
+	}
+	expectRequeue(t, ingPGR, "default", "test-ingress")
+
+	vipSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc")
+	if err != nil {
+		t.Fatalf("getting VIPService after deletion: %v", err)
+	}
+	if vipSvc == nil {
+		t.Fatal("VIPService was incorrectly deleted")
+	}
+
+	o, err = parseOwnerAnnotation(vipSvc)
+	if err != nil {
+		t.Fatalf("parsing owner annotation: %v", err)
+	}
+
+	wantOwnerRefs = []OwnerRef{
+		{OperatorID: "operator-2"},
+	}
+	if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
+	}
+}
+
+func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: "operator-ns",
+			Labels: map[string]string{
+				kubetypes.LabelManaged:    "true",
+				labelProxyGroup:           pgName,
+				labelDomain:               domain,
+				kubetypes.LabelSecretType: "certs",
+			},
+		},
+		Type: corev1.SecretTypeTLS,
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       []byte("fake-cert"),
+			corev1.TLSPrivateKeyKey: []byte("fake-key"),
+		},
+	}
+
+	_, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) {
+		s.Data = secret.Data
+	})
+	return err
+}

cmd/k8s-operator/ingress.go

@@ -73,6 +73,7 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get ing: %w", err)
 	}
 	if !ing.DeletionTimestamp.IsZero() || !a.shouldExpose(ing) {
+		// TODO(irbekrm): this message is confusing if the Ingress is an HA Ingress
 		logger.Debugf("ingress is being deleted or should not be exposed, cleaning up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, ing)
 	}

cmd/k8s-operator/ingress_test.go

@@ -6,6 +6,7 @@
 package main
 
 import (
+	"context"
 	"testing"
 
 	"go.uber.org/zap"
@@ -15,17 +16,18 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
 
 func TestTailscaleIngress(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewFakeClient(tsIngressClass)
+	fc := fake.NewFakeClient(ingressClass())
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -46,45 +48,8 @@ func TestTailscaleIngress(t *testing.T) {
 	}
 
 	// 1. Resources get created for regular Ingress
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -114,6 +79,9 @@ func TestTailscaleIngress(t *testing.T) {
 		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
 	})
 	expectReconciled(t, ingR, "default", "test")
+
+	// Get the ingress and update it with expected changes
+	ing := ingress()
 	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 	ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
 		Ingress: []networkingv1.IngressLoadBalancerIngress{
@@ -143,8 +111,7 @@ func TestTailscaleIngress(t *testing.T) {
 }
 
 func TestTailscaleIngressHostname(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewFakeClient(tsIngressClass)
+	fc := fake.NewFakeClient(ingressClass())
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -165,45 +132,8 @@ func TestTailscaleIngressHostname(t *testing.T) {
 	}
 
 	// 1. Resources get created for regular Ingress
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -241,8 +171,10 @@ func TestTailscaleIngressHostname(t *testing.T) {
 		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
 	})
 	expectReconciled(t, ingR, "default", "test")
-	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 
+	// Get the ingress and update it with expected changes
+	ing := ingress()
+	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 	expectEqual(t, fc, ing)
 
 	// 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint
@@ -299,10 +231,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass).
+		WithObjects(pc, ingressClass()).
 		WithStatusSubresource(pc).
 		Build()
 	ft := &fakeTSClient{}
@@ -326,45 +257,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 
 	// 1. Ingress is created with no ProxyClass specified, default proxy
 	// resources get configured.
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -432,54 +326,19 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				ObservedGeneration: 1,
 			}}},
 	}
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-			Labels: map[string]string{
-				"tailscale.com/proxy-class": "metrics",
-			},
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	}
 	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+
+	// Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service
+	ing := ingress()
+	ing.Labels = map[string]string{
+		LabelProxyClass: "metrics",
+	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass, ing, svc).
+		WithObjects(pc, ingressClass(), ing, service()).
 		WithStatusSubresource(pc).
 		Build()
+
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -560,3 +419,118 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
 	// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
 }
+
+func TestIngressLetsEncryptStaging(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest()
+
+	testCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther)
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther)
+
+			fc := builder.Build()
+
+			if tt.proxyClassPerResource != "" || tt.defaultProxyClass != "" {
+				name := tt.proxyClassPerResource
+				if name == "" {
+					name = tt.defaultProxyClass
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			mustCreate(t, fc, ingressClass())
+			mustCreate(t, fc, service())
+			ing := ingress()
+			if tt.proxyClassPerResource != "" {
+				ing.Labels = map[string]string{
+					LabelProxyClass: tt.proxyClassPerResource,
+				}
+			}
+			mustCreate(t, fc, ing)
+
+			ingR := &IngressReconciler{
+				Client: fc,
+				ssr: &tailscaleSTSReconciler{
+					Client:            fc,
+					tsClient:          &fakeTSClient{},
+					tsnetServer:       &fakeTSNetServer{certDomains: []string{"test-host"}},
+					defaultTags:       []string{"tag:test"},
+					operatorNamespace: "operator-ns",
+					proxyImage:        "tailscale/tailscale:test",
+				},
+				logger:            zl.Sugar(),
+				defaultProxyClass: tt.defaultProxyClass,
+			}
+
+			expectReconciled(t, ingR, "default", "test")
+
+			_, shortName := findGenName(t, fc, "default", "test", "ingress")
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			if tt.useLEStagingEndpoint {
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			} else {
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			}
+		})
+	}
+}
+
+func ingressClass() *networkingv1.IngressClass {
+	return &networkingv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
+		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
+	}
+}
+
+func service() *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "1.2.3.4",
+			Ports: []corev1.ServicePort{{
+				Port: 8080,
+				Name: "http"},
+			},
+		},
+	}
+}
+
+func ingress() *networkingv1.Ingress {
+	return &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"default-test"}},
+			},
+		},
+	}
+}

cmd/k8s-operator/metrics_resources.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 )
 
 const (
@@ -222,7 +223,7 @@ func metricsResourceName(stsName string) string {
 // proxy.
 func metricsResourceLabels(opts *metricsOpts) map[string]string {
 	lbls := map[string]string{
-		LabelManaged:             "true",
+		kubetypes.LabelManaged:   "true",
 		labelMetricsTarget:       opts.proxyStsName,
 		labelPromProxyType:       opts.proxyType,
 		labelPromProxyParentName: opts.proxyLabels[LabelParentName],

cmd/k8s-operator/operator.go

@@ -9,6 +9,7 @@ package main
 
 import (
 	"context"
+	"net/http"
 	"os"
 	"regexp"
 	"strconv"
@@ -315,6 +316,7 @@ func runReconcilers(opts reconcilerOpts) {
 	err = builder.
 		ControllerManagedBy(mgr).
 		For(&networkingv1.Ingress{}).
+		Named("ingress-reconciler").
 		Watches(&appsv1.StatefulSet{}, ingressChildFilter).
 		Watches(&corev1.Secret{}, ingressChildFilter).
 		Watches(&corev1.Service{}, svcHandlerForIngress).
@@ -329,34 +331,13 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
 	}
-	lc, err := opts.tsServer.LocalClient()
-	if err != nil {
-		startlog.Fatalf("could not get local client: %v", err)
-	}
-	err = builder.
-		ControllerManagedBy(mgr).
-		For(&networkingv1.Ingress{}).
-		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
-		Complete(&IngressPGReconciler{
-			recorder:    eventRecorder,
-			tsClient:    opts.tsClient,
-			tsnetServer: opts.tsServer,
-			defaultTags: strings.Split(opts.proxyTags, ","),
-			Client:      mgr.GetClient(),
-			logger:      opts.log.Named("ingress-pg-reconciler"),
-			lc:          lc,
-			tsNamespace: opts.tailscaleNamespace,
-		})
-	if err != nil {
-		startlog.Fatalf("could not create ingress-pg-reconciler: %v", err)
-	}
-
 	connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector"))
 	// If a ProxyClassChanges, enqueue all Connectors that have
 	// .spec.proxyClass set to the name of this ProxyClass.
 	proxyClassFilterForConnector := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForConnector(mgr.GetClient(), startlog))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.Connector{}).
+		Named("connector-reconciler").
 		Watches(&appsv1.StatefulSet{}, connectorFilter).
 		Watches(&corev1.Secret{}, connectorFilter).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForConnector).
@@ -376,6 +357,7 @@ func runReconcilers(opts reconcilerOpts) {
 	nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver"))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.DNSConfig{}).
+		Named("nameserver-reconciler").
 		Watches(&appsv1.Deployment{}, nameserverFilter).
 		Watches(&corev1.ConfigMap{}, nameserverFilter).
 		Watches(&corev1.Service{}, nameserverFilter).
@@ -449,12 +431,31 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
 	}
 
+	podsForEps := handler.EnqueueRequestsFromMapFunc(podsFromEgressEps(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
+	podsER := handler.EnqueueRequestsFromMapFunc(egressPodsHandler)
+	err = builder.
+		ControllerManagedBy(mgr).
+		Named("egress-pods-readiness-reconciler").
+		Watches(&discoveryv1.EndpointSlice{}, podsForEps).
+		Watches(&corev1.Pod{}, podsER).
+		Complete(&egressPodsReconciler{
+			Client:      mgr.GetClient(),
+			tsNamespace: opts.tailscaleNamespace,
+			clock:       tstime.DefaultClock{},
+			logger:      opts.log.Named("egress-pods-readiness-reconciler"),
+			httpClient:  http.DefaultClient,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create egress Pods readiness reconciler: %v", err)
+	}
+
 	// ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that
 	// define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get
 	// reconciled if the CRD is applied at a later point.
 	serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
+		Named("proxyclass-reconciler").
 		Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter).
 		Complete(&ProxyClassReconciler{
 			Client:   mgr.GetClient(),
@@ -498,6 +499,7 @@ func runReconcilers(opts reconcilerOpts) {
 	recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.Recorder{}).
+		Named("recorder-reconciler").
 		Watches(&appsv1.StatefulSet{}, recorderFilter).
 		Watches(&corev1.ServiceAccount{}, recorderFilter).
 		Watches(&corev1.Secret{}, recorderFilter).
@@ -520,6 +522,7 @@ func runReconcilers(opts reconcilerOpts) {
 	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyGroup{}).
+		Named("proxygroup-reconciler").
 		Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
 		Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
 		Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
@@ -598,8 +601,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Service.
 		svcProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "svc",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "svc",
 		}
 		svcHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
@@ -612,8 +615,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Ingress.
 		ingProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "ingress",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "ingress",
 		}
 		ingHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
@@ -680,7 +683,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 func isManagedResource(o client.Object) bool {
 	ls := o.GetLabels()
-	return ls[LabelManaged] == "true"
+	return ls[kubetypes.LabelManaged] == "true"
 }
 
 func isManagedByType(o client.Object, typ string) bool {
@@ -770,7 +773,7 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass,
+// proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass,
 // returns a list of reconcile requests for all Connectors that have
 // .spec.proxyClass set.
 func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -899,11 +902,25 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
 	}
 }
 
+func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
+	if typ := o.GetLabels()[LabelParentType]; typ != proxyTypeProxyGroup {
+		return nil
+	}
+	return []reconcile.Request{
+		{
+			NamespacedName: types.NamespacedName{
+				Namespace: o.GetNamespace(),
+				Name:      o.GetName(),
+			},
+		},
+	}
+}
+
 // egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
 		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
@@ -923,15 +940,13 @@ func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
 		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
 			return nil
 		}
-		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
+		if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
 			return nil
 		}
 		pg, ok := o.GetLabels()[LabelParentName]
@@ -948,7 +963,7 @@ func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
 	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
 		return nil
 	}
-	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+	if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 		return nil
 	}
 	svcName, ok := o.GetLabels()[LabelParentName]
@@ -991,7 +1006,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
 // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
 // user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
 		pg, ok := o.(*tsapi.ProxyGroup)
 		if !ok {
 			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
@@ -1001,7 +1016,7 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 			return nil
 		}
 		svcList := &corev1.ServiceList{}
-		if err := cl.List(context.Background(), svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
+		if err := cl.List(ctx, svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
 			logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
 			return nil
 		}
@@ -1021,7 +1036,7 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 // epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
 // should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
 func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
 		svc, ok := o.(*corev1.Service)
 		if !ok {
 			logger.Infof("[unexpected] Service handler triggered for an object that is not a Service")
@@ -1031,7 +1046,7 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns
 			return nil
 		}
 		epsList := &discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
+		if err := cl.List(ctx, epsList, client.InNamespace(ns),
 			client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
 			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
 			return nil
@@ -1049,6 +1064,43 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns
 	}
 }
 
+func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		eps, ok := o.(*discoveryv1.EndpointSlice)
+		if !ok {
+			logger.Infof("[unexpected] EndpointSlice handler triggered for an object that is not a EndpointSlice")
+			return nil
+		}
+		if eps.Labels[labelProxyGroup] == "" {
+			return nil
+		}
+		if eps.Labels[labelSvcType] != "egress" {
+			return nil
+		}
+		podLabels := map[string]string{
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "proxygroup",
+			LabelParentName:        eps.Labels[labelProxyGroup],
+		}
+		podList := &corev1.PodList{}
+		if err := cl.List(ctx, podList, client.InNamespace(ns),
+			client.MatchingLabels(podLabels)); err != nil {
+			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, pod := range podList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: pod.Namespace,
+					Name:      pod.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // proxyClassesWithServiceMonitor returns an event handler that, given that the event is for the Prometheus
 // ServiceMonitor CRD, returns all ProxyClasses that define that a ServiceMonitor should be created.
 func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1101,42 +1153,6 @@ func indexEgressServices(o client.Object) []string {
 	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
 }
 
-// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service
-// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation,
-// the associated Ingress gets reconciled.
-func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(ctx context.Context, o client.Object) []reconcile.Request {
-		ingList := networkingv1.IngressList{}
-		if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil {
-			logger.Debugf("error listing Ingresses: %v", err)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, ing := range ingList.Items {
-			if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
-				continue
-			}
-			if !hasProxyGroupAnnotation(&ing) {
-				continue
-			}
-			if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
-				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
-			}
-			for _, rule := range ing.Spec.Rules {
-				if rule.HTTP == nil {
-					continue
-				}
-				for _, path := range rule.HTTP.Paths {
-					if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() {
-						reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
-					}
-				}
-			}
-		}
-		return reqs
-	}
-}
-
 func hasProxyGroupAnnotation(obj client.Object) bool {
 	ing := obj.(*networkingv1.Ingress)
 	return ing.Annotations[AnnotationProxyGroup] != ""

cmd/k8s-operator/operator_test.go

@@ -1339,71 +1339,6 @@ func TestProxyFirewallMode(t *testing.T) {
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
 }
 
-func TestTailscaledConfigfileHash(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:                zl.Sugar(),
-		clock:                 clock,
-		isDefaultLoadBalancer: true,
-	}
-
-	// Create a service that we should manage, and check that the initial round
-	// of objects looks right.
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "10.20.30.40",
-			Type:      corev1.ServiceTypeLoadBalancer,
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-	expectReconciled(t, sr, "default", "test")
-
-	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
-	o := configOpts{
-		stsName:         shortName,
-		secretName:      fullName,
-		namespace:       "default",
-		parentType:      "svc",
-		hostname:        "default-test",
-		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "848bff4b5ba83ac999e6984c8464e597156daba961ae045e7dbaef606d54ab5e",
-		app:             kubetypes.AppIngressProxy,
-	}
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
-
-	// 2. Hostname gets changed, configfile is updated and a new hash value
-	// is produced.
-	mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
-		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
-	})
-	o.hostname = "another-test"
-	o.confFileHash = "d4cc13f09f55f4f6775689004f9a466723325b84d2b590692796bfe22aeaa389"
-	expectReconciled(t, sr, "default", "test")
-	expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
-}
 func Test_isMagicDNSName(t *testing.T) {
 	tests := []struct {
 		in   string
@@ -1452,10 +1387,10 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 			Name:      "headless-1",
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      "ing-1",
-				LabelParentNamespace: "ns-1",
-				LabelParentType:      "ingress",
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        "ing-1",
+				LabelParentNamespace:   "ns-1",
+				LabelParentType:        "ingress",
 			},
 		},
 	}

cmd/k8s-operator/proxy.go

@@ -20,7 +20,7 @@ import (
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
 	ksr "tailscale.com/k8s-operator/sessionrecording"
 	"tailscale.com/kube/kubetypes"
@@ -189,7 +189,7 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
 // LocalAPI and then proxies them to the Kubernetes API.
 type apiserverProxy struct {
 	log *zap.SugaredLogger
-	lc  *tailscale.LocalClient
+	lc  *local.Client
 	rp  *httputil.ReverseProxy
 
 	mode        apiServerProxyMode

cmd/k8s-operator/proxygroup.go

@@ -32,6 +32,7 @@ import (
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
@@ -166,6 +167,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 			r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
 			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
 		}
+		validateProxyClassForPG(logger, pg, proxyClass)
 		if !tsoperator.ProxyClassIsReady(proxyClass) {
 			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName)
 			logger.Info(message)
@@ -204,6 +206,31 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady)
 }
 
+// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup.
+func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) {
+	if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
+		return
+	}
+	// Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check
+	// beig accessible on the replica Pod IP:9002. This address can also be modified by users, via
+	// TS_LOCAL_ADDR_PORT env var.
+	//
+	// Currently TS_LOCAL_ADDR_PORT controls Pod's health check and metrics address. _Probably_ there is no need for
+	// users to set this to a custom value. Users who want to consume metrics, should integrate with the metrics
+	// Service and/or ServiceMonitor, rather than Pods directly. The health check is likely not useful to integrate
+	// directly with for operator proxies (and we should aim for unified lifecycle logic in the operator, users
+	// shouldn't need to set their own).
+	//
+	// TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later).
+	if hasLocalAddrPortSet(pc) {
+		msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+
+			"This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+
+			"In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+
+			"Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name)
+		logger.Warn(msg)
+	}
+}
+
 func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
 	logger := r.logger(pg.Name)
 	r.mu.Lock()
@@ -253,10 +280,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 		return fmt.Errorf("error provisioning RoleBinding: %w", err)
 	}
 	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-		cm := pgEgressCM(pg, r.tsNamespace)
+		cm, hp := pgEgressCM(pg, r.tsNamespace)
 		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
 			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
 			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
+			mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp)
 		}); err != nil {
 			return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err)
 		}
@@ -270,11 +298,14 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err)
 		}
 	}
-	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode)
+	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass)
 	if err != nil {
 		return fmt.Errorf("error generating StatefulSet spec: %w", err)
 	}
-	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
+	cfg := &tailscaleSTSConfig{
+		proxyType: string(pg.Spec.Type),
+	}
+	ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger)
 	capver, err := r.capVerForPG(ctx, pg, logger)
 	if err != nil {
 		return fmt.Errorf("error getting device info: %w", err)
@@ -424,7 +455,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 	for i := range pgReplicas(pg) {
 		cfgSecret := &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:            fmt.Sprintf("%s-%d-config", pg.Name, i),
+				Name:            pgConfigSecretName(pg.Name, i),
 				Namespace:       r.tsNamespace,
 				Labels:          pgSecretLabels(pg.Name, "config"),
 				OwnerReferences: pgOwnerReference(pg),
@@ -433,7 +464,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 
 		var existingCfgSecret *corev1.Secret // unmodified copy of secret
 		if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
-			logger.Debugf("secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
+			logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
 			existingCfgSecret = cfgSecret.DeepCopy()
 		} else if !apierrors.IsNotFound(err) {
 			return "", err
@@ -441,7 +472,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 
 		var authKey string
 		if existingCfgSecret == nil {
-			logger.Debugf("creating authkey for new ProxyGroup proxy")
+			logger.Debugf("Creating authkey for new ProxyGroup proxy")
 			tags := pg.Spec.Tags.Stringify()
 			if len(tags) == 0 {
 				tags = r.defaultTags
@@ -462,7 +493,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			if err != nil {
 				return "", fmt.Errorf("error marshalling tailscaled config: %w", err)
 			}
-			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
+			mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
 		}
 
 		// The config sha256 sum is a value for a hash annotation used to trigger
@@ -492,12 +523,14 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		}
 
 		if existingCfgSecret != nil {
-			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
-			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
-				return "", err
+			if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) {
+				logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name)
+				if err := r.Update(ctx, cfgSecret); err != nil {
+					return "", err
+				}
 			}
 		} else {
-			logger.Debugf("creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
+			logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
 			if err := r.Create(ctx, cfgSecret); err != nil {
 				return "", err
 			}
@@ -568,10 +601,35 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
 		conf.AuthKey = key
 	}
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+
+	// AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
+	// don't overwrite it here.
+	if err := copyAdvertiseServicesConfig(conf, oldSecret, 106); err != nil {
+		return nil, err
+	}
 	capVerConfigs[106] = *conf
 	return capVerConfigs, nil
 }
 
+func copyAdvertiseServicesConfig(conf *ipn.ConfigVAlpha, oldSecret *corev1.Secret, capVer tailcfg.CapabilityVersion) error {
+	if oldSecret == nil {
+		return nil
+	}
+
+	oldConfB := oldSecret.Data[tsoperator.TailscaledConfigFileName(capVer)]
+	if len(oldConfB) == 0 {
+		return nil
+	}
+
+	var oldConf ipn.ConfigVAlpha
+	if err := json.Unmarshal(oldConfB, &oldConf); err != nil {
+		return fmt.Errorf("error unmarshalling existing config: %w", err)
+	}
+	conf.AdvertiseServices = oldConf.AdvertiseServices
+
+	return nil
+}
+
 func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error {
 	return nil
 }
@@ -592,7 +650,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 			return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err)
 		}
 
-		id, dnsName, ok, err := getNodeMetadata(ctx, &secret)
+		prefs, ok, err := getDevicePrefs(&secret)
 		if err != nil {
 			return nil, err
 		}
@@ -603,8 +661,8 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 		nm := nodeMetadata{
 			ordinal:     ordinal,
 			stateSecret: &secret,
-			tsID:        id,
-			dnsName:     dnsName,
+			tsID:        prefs.Config.NodeID,
+			dnsName:     prefs.Config.UserProfile.LoginName,
 		}
 		pod := &corev1.Pod{}
 		if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) {

cmd/k8s-operator/proxygroup_specs.go

@@ -7,11 +7,14 @@ package main
 
 import (
 	"fmt"
+	"slices"
+	"strconv"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/egressservices"
@@ -19,9 +22,12 @@ import (
 	"tailscale.com/types/ptr"
 )
 
+// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully.
+const deletionGracePeriodSeconds int64 = 360
+
 // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
 // applied over the top after.
-func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string) (*appsv1.StatefulSet, error) {
+func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
@@ -67,7 +73,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				Name: fmt.Sprintf("tailscaledconfig-%d", i),
 				VolumeSource: corev1.VolumeSource{
 					Secret: &corev1.SecretVolumeSource{
-						SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i),
+						SecretName: pgConfigSecretName(pg.Name, i),
 					},
 				},
 			})
@@ -145,15 +151,25 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 		}
 
 		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			envs = append(envs, corev1.EnvVar{
-				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
-				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
-			},
+			envs = append(envs,
+				// TODO(irbekrm): in 1.80 we deprecated TS_EGRESS_SERVICES_CONFIG_PATH in favour of
+				// TS_EGRESS_PROXIES_CONFIG_PATH. Remove it in 1.84.
+				corev1.EnvVar{
+					Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
+					Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
+				},
+				corev1.EnvVar{
+					Name:  "TS_EGRESS_PROXIES_CONFIG_PATH",
+					Value: "/etc/proxies",
+				},
 				corev1.EnvVar{
 					Name:  "TS_INTERNAL_APP",
 					Value: kubetypes.AppProxyGroupEgress,
 				},
-			)
+				corev1.EnvVar{
+					Name:  "TS_ENABLE_HEALTH_CHECK",
+					Value: "true",
+				})
 		} else { // ingress
 			envs = append(envs, corev1.EnvVar{
 				Name:  "TS_INTERNAL_APP",
@@ -162,11 +178,38 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				corev1.EnvVar{
 					Name:  "TS_SERVE_CONFIG",
 					Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),
-				})
+				},
+				corev1.EnvVar{
+					// Run proxies in cert share mode to
+					// ensure that only one TLS cert is
+					// issued for an HA Ingress.
+					Name:  "TS_EXPERIMENTAL_CERT_SHARE",
+					Value: "true",
+				},
+			)
 		}
 		return append(c.Env, envs...)
 	}()
 
+	// The pre-stop hook is used to ensure that a replica does not get terminated while cluster traffic for egress
+	// services is still being routed to it.
+	//
+	// This mechanism currently (2025-01-26) rely on the local health check being accessible on the Pod's
+	// IP, so they are not supported for ProxyGroups where users have configured TS_LOCAL_ADDR_PORT to a custom
+	// value.
+	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && !hasLocalAddrPortSet(proxyClass) {
+		c.Lifecycle = &corev1.Lifecycle{
+			PreStop: &corev1.LifecycleHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: kubetypes.EgessServicesPreshutdownEP,
+					Port: intstr.FromInt(defaultLocalAddrPort),
+				},
+			},
+		}
+		// Set the deletion grace period to 6 minutes to ensure that the pre-stop hook has enough time to terminate
+		// gracefully.
+		ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds)
+	}
 	return ss, nil
 }
 
@@ -190,6 +233,13 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 			OwnerReferences: pgOwnerReference(pg),
 		},
 		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"list",
+				},
+			},
 			{
 				APIGroups: []string{""},
 				Resources: []string{"secrets"},
@@ -201,8 +251,8 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 				ResourceNames: func() (secrets []string) {
 					for i := range pgReplicas(pg) {
 						secrets = append(secrets,
-							fmt.Sprintf("%s-%d-config", pg.Name, i), // Config with auth key.
-							fmt.Sprintf("%s-%d", pg.Name, i),        // State.
+							pgConfigSecretName(pg.Name, i),   // Config with auth key.
+							fmt.Sprintf("%s-%d", pg.Name, i), // State.
 						)
 					}
 					return secrets
@@ -258,7 +308,9 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
 	return secrets
 }
 
-func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
+func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) (*corev1.ConfigMap, []byte) {
+	hp := hepPings(pg)
+	hpBs := []byte(strconv.Itoa(hp))
 	return &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:            pgEgressCMName(pg.Name),
@@ -266,8 +318,10 @@ func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
 			Labels:          pgLabels(pg.Name, nil),
 			OwnerReferences: pgOwnerReference(pg),
 		},
-	}
+		BinaryData: map[string][]byte{egressservices.KeyHEPPings: hpBs},
+	}, hpBs
 }
+
 func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
@@ -279,9 +333,9 @@ func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
 	}
 }
 
-func pgSecretLabels(pgName, typ string) map[string]string {
+func pgSecretLabels(pgName, secretType string) map[string]string {
 	return pgLabels(pgName, map[string]string{
-		labelSecretType: typ, // "config" or "state".
+		kubetypes.LabelSecretType: secretType, // "config" or "state".
 	})
 }
 
@@ -291,7 +345,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
 		l[k] = v
 	}
 
-	l[LabelManaged] = "true"
+	l[kubetypes.LabelManaged] = "true"
 	l[LabelParentType] = "proxygroup"
 	l[LabelParentName] = pgName
 
@@ -310,6 +364,30 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 {
 	return 2
 }
 
+func pgConfigSecretName(pgName string, i int32) string {
+	return fmt.Sprintf("%s-%d-config", pgName, i)
+}
+
 func pgEgressCMName(pg string) string {
 	return fmt.Sprintf("%s-egress-config", pg)
 }
+
+// hasLocalAddrPortSet returns true if the proxyclass has the TS_LOCAL_ADDR_PORT env var set. For egress ProxyGroups,
+// currently (2025-01-26) this means that the ProxyGroup does not support graceful failover.
+func hasLocalAddrPortSet(proxyClass *tsapi.ProxyClass) bool {
+	if proxyClass == nil || proxyClass.Spec.StatefulSet == nil || proxyClass.Spec.StatefulSet.Pod == nil || proxyClass.Spec.StatefulSet.Pod.TailscaleContainer == nil {
+		return false
+	}
+	return slices.ContainsFunc(proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env, func(env tsapi.Env) bool {
+		return env.Name == envVarTSLocalAddrPort
+	})
+}
+
+// hepPings returns the number of times a health check endpoint exposed by a Service fronting ProxyGroup replicas should
+// be pinged to ensure that all currently configured backend replicas are hit.
+func hepPings(pg *tsapi.ProxyGroup) int {
+	rc := pgReplicas(pg)
+	// Assuming a Service implemented using round robin load balancing, number-of-replica-times should be enough, but in
+	// practice, we cannot assume that the requests will be load balanced perfectly.
+	return int(rc) * 3
+}

cmd/k8s-operator/proxygroup_test.go

@@ -19,13 +19,14 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
@@ -97,7 +98,7 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, false, "")
+		expectProxyGroupResources(t, fc, pg, false, "", pc)
 	})
 
 	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
@@ -118,11 +119,11 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, true, "")
+		expectProxyGroupResources(t, fc, pg, true, "", pc)
 		if expected := 1; reconciler.egressProxyGroups.Len() != expected {
 			t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
 		}
-		expectProxyGroupResources(t, fc, pg, true, "")
+		expectProxyGroupResources(t, fc, pg, true, "", pc)
 		keyReq := tailscale.KeyCapabilities{
 			Devices: tailscale.KeyDeviceCapabilities{
 				Create: tailscale.KeyDeviceCreateCapabilities{
@@ -154,7 +155,7 @@ func TestProxyGroup(t *testing.T) {
 		}
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc)
 	})
 
 	t.Run("scale_up_to_3", func(t *testing.T) {
@@ -165,7 +166,7 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc)
 
 		addNodeIDToStateSecrets(t, fc, pg)
 		expectReconciled(t, reconciler, "", pg.Name)
@@ -175,7 +176,7 @@ func TestProxyGroup(t *testing.T) {
 			TailnetIPs: []string{"1.2.3.4", "::1"},
 		})
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc)
 	})
 
 	t.Run("scale_down_to_1", func(t *testing.T) {
@@ -188,7 +189,7 @@ func TestProxyGroup(t *testing.T) {
 
 		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc)
 	})
 
 	t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) {
@@ -202,7 +203,7 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 
 		expectEqual(t, fc, pg)
-		expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74")
+		expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74", pc)
 	})
 
 	t.Run("enable_metrics", func(t *testing.T) {
@@ -249,9 +250,25 @@ func TestProxyGroup(t *testing.T) {
 }
 
 func TestProxyGroupTypes(t *testing.T) {
+	pc := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{},
+	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pc).
+		WithStatusSubresource(pc).
 		Build()
+	mustUpdateStatus(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+		p.Status.Conditions = []metav1.Condition{{
+			Type:               string(tsapi.ProxyClassReady),
+			Status:             metav1.ConditionTrue,
+			ObservedGeneration: 1,
+		}}
+	})
 
 	zl, _ := zap.NewDevelopment()
 	reconciler := &ProxyGroupReconciler{
@@ -274,9 +291,7 @@ func TestProxyGroupTypes(t *testing.T) {
 				Replicas: ptr.To[int32](0),
 			},
 		}
-		if err := fc.Create(context.Background(), pg); err != nil {
-			t.Fatal(err)
-		}
+		mustCreate(t, fc, pg)
 
 		expectReconciled(t, reconciler, "", pg.Name)
 		verifyProxyGroupCounts(t, reconciler, 0, 1)
@@ -286,7 +301,8 @@ func TestProxyGroupTypes(t *testing.T) {
 			t.Fatalf("failed to get StatefulSet: %v", err)
 		}
 		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress)
-		verifyEnvVar(t, sts, "TS_EGRESS_SERVICES_CONFIG_PATH", fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices))
+		verifyEnvVar(t, sts, "TS_EGRESS_PROXIES_CONFIG_PATH", "/etc/proxies")
+		verifyEnvVar(t, sts, "TS_ENABLE_HEALTH_CHECK", "true")
 
 		// Verify that egress configuration has been set up.
 		cm := &corev1.ConfigMap{}
@@ -323,6 +339,57 @@ func TestProxyGroupTypes(t *testing.T) {
 		if diff := cmp.Diff(expectedVolumeMounts, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" {
 			t.Errorf("unexpected volume mounts (-want +got):\n%s", diff)
 		}
+
+		expectedLifecycle := corev1.Lifecycle{
+			PreStop: &corev1.LifecycleHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: kubetypes.EgessServicesPreshutdownEP,
+					Port: intstr.FromInt(defaultLocalAddrPort),
+				},
+			},
+		}
+		if diff := cmp.Diff(expectedLifecycle, *sts.Spec.Template.Spec.Containers[0].Lifecycle); diff != "" {
+			t.Errorf("unexpected lifecycle (-want +got):\n%s", diff)
+		}
+		if *sts.Spec.Template.DeletionGracePeriodSeconds != deletionGracePeriodSeconds {
+			t.Errorf("unexpected deletion grace period seconds %d, want %d", *sts.Spec.Template.DeletionGracePeriodSeconds, deletionGracePeriodSeconds)
+		}
+	})
+	t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) {
+		pg := &tsapi.ProxyGroup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-egress-no-lifecycle",
+				UID:  "test-egress-no-lifecycle-uid",
+			},
+			Spec: tsapi.ProxyGroupSpec{
+				Type:       tsapi.ProxyGroupTypeEgress,
+				Replicas:   ptr.To[int32](0),
+				ProxyClass: "test",
+			},
+		}
+		mustCreate(t, fc, pg)
+		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+			p.Spec.StatefulSet = &tsapi.StatefulSet{
+				Pod: &tsapi.Pod{
+					TailscaleContainer: &tsapi.Container{
+						Env: []tsapi.Env{{
+							Name:  "TS_LOCAL_ADDR_PORT",
+							Value: "127.0.0.1:8080",
+						}},
+					},
+				},
+			}
+		})
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		sts := &appsv1.StatefulSet{}
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+			t.Fatalf("failed to get StatefulSet: %v", err)
+		}
+
+		if sts.Spec.Template.Spec.Containers[0].Lifecycle != nil {
+			t.Error("lifecycle hook was set when TS_LOCAL_ADDR_PORT was configured via ProxyClass")
+		}
 	})
 
 	t.Run("ingress_type", func(t *testing.T) {
@@ -341,7 +408,7 @@ func TestProxyGroupTypes(t *testing.T) {
 		}
 
 		expectReconciled(t, reconciler, "", pg.Name)
-		verifyProxyGroupCounts(t, reconciler, 1, 1)
+		verifyProxyGroupCounts(t, reconciler, 1, 2)
 
 		sts := &appsv1.StatefulSet{}
 		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -349,6 +416,7 @@ func TestProxyGroupTypes(t *testing.T) {
 		}
 		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
 		verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
+		verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
 
 		// Verify ConfigMap volume mount
 		cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
@@ -379,6 +447,131 @@ func TestProxyGroupTypes(t *testing.T) {
 	})
 }
 
+func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		Build()
+	reconciler := &ProxyGroupReconciler{
+		tsNamespace: tsNamespace,
+		proxyImage:  testProxyImage,
+		Client:      fc,
+		l:           zap.Must(zap.NewDevelopment()).Sugar(),
+		tsClient:    &fakeTSClient{},
+		clock:       tstest.NewClock(tstest.ClockOpts{}),
+	}
+
+	existingServices := []string{"svc1", "svc2"}
+	existingConfigBytes, err := json.Marshal(ipn.ConfigVAlpha{
+		AdvertiseServices: existingServices,
+		Version:           "should-get-overwritten",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const pgName = "test-ingress"
+	mustCreate(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgConfigSecretName(pgName, 0),
+			Namespace: tsNamespace,
+		},
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): existingConfigBytes,
+		},
+	})
+
+	mustCreate(t, fc, &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: pgName,
+			UID:  "test-ingress-uid",
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type:     tsapi.ProxyGroupTypeIngress,
+			Replicas: ptr.To[int32](1),
+		},
+	})
+	expectReconciled(t, reconciler, "", pgName)
+
+	expectedConfigBytes, err := json.Marshal(ipn.ConfigVAlpha{
+		// Preserved.
+		AdvertiseServices: existingServices,
+
+		// Everything else got updated in the reconcile:
+		Version:      "alpha0",
+		AcceptDNS:    "false",
+		AcceptRoutes: "false",
+		Locked:       "false",
+		Hostname:     ptr.To(fmt.Sprintf("%s-%d", pgName, 0)),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectEqual(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pgConfigSecretName(pgName, 0),
+			Namespace:       tsNamespace,
+			ResourceVersion: "2",
+		},
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): expectedConfigBytes,
+		},
+	})
+}
+
+func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) {
+	pcLEStaging := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "le-staging",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: true,
+		},
+	}
+
+	pcLEStagingFalse := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "le-staging-false",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: false,
+		},
+	}
+
+	pcOther := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "other",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{},
+	}
+
+	return pcLEStaging, pcLEStagingFalse, pcOther
+}
+
+func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name string) *tsapi.ProxyClass {
+	t.Helper()
+	pc := &tsapi.ProxyClass{}
+	if err := fc.Get(context.Background(), client.ObjectKey{Name: name}, pc); err != nil {
+		t.Fatal(err)
+	}
+	pc.Status = tsapi.ProxyClassStatus{
+		Conditions: []metav1.Condition{{
+			Type:               string(tsapi.ProxyClassReady),
+			Status:             metav1.ConditionTrue,
+			Reason:             reasonProxyClassValid,
+			Message:            reasonProxyClassValid,
+			LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+			ObservedGeneration: pc.Generation,
+		}},
+	}
+	if err := fc.Status().Update(context.Background(), pc); err != nil {
+		t.Fatal(err)
+	}
+	return pc
+}
+
 func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
 	t.Helper()
 	if r.ingressProxyGroups.Len() != wantIngress {
@@ -402,13 +595,23 @@ func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue str
 	t.Errorf("%s environment variable not found", name)
 }
 
-func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
+func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) {
+	t.Helper()
+	for _, env := range sts.Spec.Template.Spec.Containers[0].Env {
+		if env.Name == name {
+			t.Errorf("environment variable %s should not be present", name)
+			return
+		}
+	}
+}
+
+func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) {
 	t.Helper()
 
 	role := pgRole(pg, tsNamespace)
 	roleBinding := pgRoleBinding(pg, tsNamespace)
 	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto")
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -434,7 +637,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 		for i := range pgReplicas(pg) {
 			expectedSecrets = append(expectedSecrets,
 				fmt.Sprintf("%s-%d", pg.Name, i),
-				fmt.Sprintf("%s-%d-config", pg.Name, i),
+				pgConfigSecretName(pg.Name, i),
 			)
 		}
 	}
@@ -479,3 +682,146 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
 		})
 	}
 }
+
+func TestProxyGroupLetsEncryptStaging(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	// Set up test cases- most are shared with non-HA Ingress.
+	type proxyGroupLETestCase struct {
+		leStagingTestCase
+		pgType tsapi.ProxyGroupType
+	}
+	pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest()
+	sharedTestCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther)
+	var tests []proxyGroupLETestCase
+	for _, tt := range sharedTestCases {
+		tests = append(tests, proxyGroupLETestCase{
+			leStagingTestCase: tt,
+			pgType:            tsapi.ProxyGroupTypeIngress,
+		})
+	}
+	tests = append(tests, proxyGroupLETestCase{
+		leStagingTestCase: leStagingTestCase{
+			name:                  "egress_pg_with_staging_proxyclass",
+			proxyClassPerResource: "le-staging",
+			useLEStagingEndpoint:  false,
+		},
+		pgType: tsapi.ProxyGroupTypeEgress,
+	})
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			// Pre-populate the fake client with ProxyClasses.
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther)
+
+			fc := builder.Build()
+
+			// If the test case needs a ProxyClass to exist, ensure it is set to Ready.
+			if tt.proxyClassPerResource != "" || tt.defaultProxyClass != "" {
+				name := tt.proxyClassPerResource
+				if name == "" {
+					name = tt.defaultProxyClass
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			// Create ProxyGroup
+			pg := &tsapi.ProxyGroup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+				},
+				Spec: tsapi.ProxyGroupSpec{
+					Type:       tt.pgType,
+					Replicas:   ptr.To[int32](1),
+					ProxyClass: tt.proxyClassPerResource,
+				},
+			}
+			mustCreate(t, fc, pg)
+
+			reconciler := &ProxyGroupReconciler{
+				tsNamespace:       tsNamespace,
+				proxyImage:        testProxyImage,
+				defaultTags:       []string{"tag:test"},
+				defaultProxyClass: tt.defaultProxyClass,
+				Client:            fc,
+				tsClient:          &fakeTSClient{},
+				l:                 zl.Sugar(),
+				clock:             cl,
+			}
+
+			expectReconciled(t, reconciler, "", pg.Name)
+
+			// Verify that the StatefulSet created for ProxyGrup has
+			// the expected setting for the staging endpoint.
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			if tt.useLEStagingEndpoint {
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			} else {
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			}
+		})
+	}
+}
+
+type leStagingTestCase struct {
+	name string
+	// ProxyClass set on ProxyGroup or Ingress resource.
+	proxyClassPerResource string
+	// Default ProxyClass.
+	defaultProxyClass    string
+	useLEStagingEndpoint bool
+}
+
+// Shared test cases for LE staging endpoint configuration for ProxyGroup and
+// non-HA Ingress.
+func testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther *tsapi.ProxyClass) []leStagingTestCase {
+	return []leStagingTestCase{
+		{
+			name:                  "with_staging_proxyclass",
+			proxyClassPerResource: "le-staging",
+			useLEStagingEndpoint:  true,
+		},
+		{
+			name:                  "with_staging_proxyclass_false",
+			proxyClassPerResource: "le-staging-false",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_other_proxyclass",
+			proxyClassPerResource: "other",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "no_proxyclass",
+			proxyClassPerResource: "",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_default_staging_proxyclass",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "le-staging",
+			useLEStagingEndpoint:  true,
+		},
+		{
+			name:                  "with_default_other_proxyclass",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "other",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_default_staging_proxyclass_false",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "le-staging-false",
+			useLEStagingEndpoint:  false,
+		},
+	}
+}

cmd/k8s-operator/sts.go

@@ -44,11 +44,9 @@ const (
 	// Labels that the operator sets on StatefulSets and Pods. If you add a
 	// new label here, do also add it to tailscaleManagedLabels var to
 	// ensure that it does not get overwritten by ProxyClass configuration.
-	LabelManaged         = "tailscale.com/managed"
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
 	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
 	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
@@ -101,11 +99,16 @@ const (
 	proxyTypeIngressResource = "ingress_resource"
 	proxyTypeConnector       = "connector"
 	proxyTypeProxyGroup      = "proxygroup"
+
+	envVarTSLocalAddrPort = "TS_LOCAL_ADDR_PORT"
+	defaultLocalAddrPort  = 9002 // metrics and health check port
+
+	letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory"
 )
 
 var (
 	// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
-	tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
+	tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
 	// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
 	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
 )
@@ -694,7 +697,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 		// being created, there is no need for a restart.
 		// TODO(irbekrm): remove this in 1.84.
 		hash := tsConfigHash
-		if dev != nil && dev.capver >= 110 {
+		if dev == nil || dev.capver >= 110 {
 			hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash]
 		}
 		s.Spec = ss.Spec
@@ -782,6 +785,17 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 			enableEndpoints(ss, metricsEnabled, debugEnabled)
 		}
 	}
+	if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) {
+		for i, c := range ss.Spec.Template.Spec.Containers {
+			if c.Name == "tailscale" {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{
+					Name:  "TS_DEBUG_ACME_DIRECTORY_URL",
+					Value: letsEncryptStagingEndpoint,
+				})
+				break
+			}
+		}
+	}
 
 	if pc.Spec.StatefulSet == nil {
 		return ss

cmd/k8s-operator/sts_test.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
 
@@ -156,8 +157,8 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// Set a couple additional fields so we can test that we don't
 	// mistakenly override those.
 	labels := map[string]string{
-		LabelManaged:    "true",
-		LabelParentName: "foo",
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        "foo",
 	}
 	annots := map[string]string{
 		podAnnotationLastSetClusterIP: "1.2.3.4",
@@ -303,28 +304,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 	}{
 		{
 			name:    "no custom labels specified and none present in current labels, return current labels",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels",
-			current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels only contain tailscale managed labels, return a union of both",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels",
-			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{

cmd/k8s-operator/svc.go

@@ -84,10 +84,10 @@ func childResourceLabels(name, ns, typ string) map[string]string {
 	// proxying. Instead, we have to do our own filtering and tracking with
 	// labels.
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 }
 

cmd/k8s-operator/testutils_test.go

@@ -28,10 +28,12 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/internal/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
@@ -562,10 +564,10 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
 	t.Helper()
 	labels := map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 	s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
 	if err != nil {
@@ -583,6 +585,21 @@ func mustCreate(t *testing.T, client client.Client, obj client.Object) {
 		t.Fatalf("creating %q: %v", obj.GetName(), err)
 	}
 }
+func mustCreateAll(t *testing.T, client client.Client, objs ...client.Object) {
+	t.Helper()
+	for _, obj := range objs {
+		mustCreate(t, client, obj)
+	}
+}
+
+func mustDeleteAll(t *testing.T, client client.Client, objs ...client.Object) {
+	t.Helper()
+	for _, obj := range objs {
+		if err := client.Delete(context.Background(), obj); err != nil {
+			t.Fatalf("deleting %q: %v", obj.GetName(), err)
+		}
+	}
+}
 
 func mustUpdate[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string, update func(O)) {
 	t.Helper()
@@ -706,6 +723,19 @@ func expectRequeue(t *testing.T, sr reconcile.Reconciler, ns, name string) {
 		t.Fatalf("expected timed requeue, got success")
 	}
 }
+func expectError(t *testing.T, sr reconcile.Reconciler, ns, name string) {
+	t.Helper()
+	req := reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      name,
+			Namespace: ns,
+		},
+	}
+	_, err := sr.Reconcile(context.Background(), req)
+	if err == nil {
+		t.Error("Reconcile: expected error but did not get one")
+	}
+}
 
 // expectEvents accepts a test recorder and a list of events, tests that expected
 // events are sent down the recorder's channel. Waits for 5s for each event.
@@ -739,7 +769,7 @@ type fakeTSClient struct {
 	sync.Mutex
 	keyRequests []tailscale.KeyCapabilities
 	deleted     []string
-	vipServices map[string]*VIPService
+	vipServices map[tailcfg.ServiceName]*tailscale.VIPService
 }
 type fakeTSNetServer struct {
 	certDomains []string
@@ -846,7 +876,7 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	}
 }
 
-func (c *fakeTSClient) getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) {
+func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error) {
 	c.Lock()
 	defer c.Unlock()
 	if c.vipServices == nil {
@@ -859,17 +889,17 @@ func (c *fakeTSClient) getVIPServiceByName(ctx context.Context, name string) (*V
 	return svc, nil
 }
 
-func (c *fakeTSClient) createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error {
+func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error {
 	c.Lock()
 	defer c.Unlock()
 	if c.vipServices == nil {
-		c.vipServices = make(map[string]*VIPService)
+		c.vipServices = make(map[tailcfg.ServiceName]*tailscale.VIPService)
 	}
 	c.vipServices[svc.Name] = svc
 	return nil
 }
 
-func (c *fakeTSClient) deleteVIPServiceByName(ctx context.Context, name string) error {
+func (c *fakeTSClient) DeleteVIPService(ctx context.Context, name tailcfg.ServiceName) error {
 	c.Lock()
 	defer c.Unlock()
 	if c.vipServices != nil {

cmd/k8s-operator/tsclient.go

@@ -6,18 +6,13 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"os"
 
 	"golang.org/x/oauth2/clientcredentials"
-	"tailscale.com/client/tailscale"
-	"tailscale.com/util/httpm"
+	"tailscale.com/internal/client/tailscale"
+	"tailscale.com/tailcfg"
 )
 
 // defaultTailnet is a value that can be used in Tailscale API calls instead of tailnet name to indicate that the API
@@ -44,142 +39,14 @@ func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (ts
 	c := tailscale.NewClient(defaultTailnet, nil)
 	c.UserAgent = "tailscale-k8s-operator"
 	c.HTTPClient = credentials.Client(ctx)
-	tsc := &tsClientImpl{
-		Client:  c,
-		baseURL: defaultBaseURL,
-		tailnet: defaultTailnet,
-	}
-	return tsc, nil
+	return c, nil
 }
 
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
 	Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error
-	getVIPServiceByName(ctx context.Context, name string) (*VIPService, error)
-	createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error
-	deleteVIPServiceByName(ctx context.Context, name string) error
-}
-
-type tsClientImpl struct {
-	*tailscale.Client
-	baseURL string
-	tailnet string
-}
-
-// VIPService is a Tailscale VIPService with Tailscale API JSON representation.
-type VIPService struct {
-	// Name is the leftmost label of the DNS name of the VIP service.
-	// Name is required.
-	Name string `json:"name,omitempty"`
-	// Addrs are the IP addresses of the VIP Service. There are two addresses:
-	// the first is IPv4 and the second is IPv6.
-	// When creating a new VIP Service, the IP addresses are optional: if no
-	// addresses are specified then they will be selected. If an IPv4 address is
-	// specified at index 0, then that address will attempt to be used. An IPv6
-	// address can not be specified upon creation.
-	Addrs []string `json:"addrs,omitempty"`
-	// Comment is an optional text string for display in the admin panel.
-	Comment string `json:"comment,omitempty"`
-	// Ports are the ports of a VIPService that will be configured via Tailscale serve config.
-	// If set, any node wishing to advertise this VIPService must have this port configured via Tailscale serve.
-	Ports []string `json:"ports,omitempty"`
-	// Tags are optional ACL tags that will be applied to the VIPService.
-	Tags []string `json:"tags,omitempty"`
-}
-
-// GetVIPServiceByName retrieves a VIPService by its name. It returns 404 if the VIPService is not found.
-func (c *tsClientImpl) getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(name))
-	req, err := http.NewRequestWithContext(ctx, httpm.GET, path, nil)
-	if err != nil {
-		return nil, fmt.Errorf("error creating new HTTP request: %w", err)
-	}
-	b, resp, err := c.sendRequest(req)
-	if err != nil {
-		return nil, fmt.Errorf("error making Tailsale API request: %w", err)
-	}
-	// If status code was not successful, return the error.
-	// TODO: Change the check for the StatusCode to include other 2XX success codes.
-	if resp.StatusCode != http.StatusOK {
-		return nil, handleErrorResponse(b, resp)
-	}
-	svc := &VIPService{}
-	if err := json.Unmarshal(b, svc); err != nil {
-		return nil, err
-	}
-	return svc, nil
-}
-
-// CreateOrUpdateVIPServiceByName creates or updates a VIPService by its name. Caller must ensure that, if the
-// VIPService already exists, the VIPService is fetched first to ensure that any auto-allocated IP addresses are not
-// lost during the update. If the VIPService was created without any IP addresses explicitly set (so that they were
-// auto-allocated by Tailscale) any subsequent request to this function that does not set any IP addresses will error.
-func (c *tsClientImpl) createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error {
-	data, err := json.Marshal(svc)
-	if err != nil {
-		return err
-	}
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(svc.Name))
-	req, err := http.NewRequestWithContext(ctx, httpm.PUT, path, bytes.NewBuffer(data))
-	if err != nil {
-		return fmt.Errorf("error creating new HTTP request: %w", err)
-	}
-	b, resp, err := c.sendRequest(req)
-	if err != nil {
-		return fmt.Errorf("error making Tailscale API request: %w", err)
-	}
-	// If status code was not successful, return the error.
-	// TODO: Change the check for the StatusCode to include other 2XX success codes.
-	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
-	}
-	return nil
-}
-
-// DeleteVIPServiceByName deletes a VIPService by its name. It returns an error if the VIPService
-// does not exist or if the deletion fails.
-func (c *tsClientImpl) deleteVIPServiceByName(ctx context.Context, name string) error {
-	path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(name))
-	req, err := http.NewRequestWithContext(ctx, httpm.DELETE, path, nil)
-	if err != nil {
-		return fmt.Errorf("error creating new HTTP request: %w", err)
-	}
-	b, resp, err := c.sendRequest(req)
-	if err != nil {
-		return fmt.Errorf("error making Tailscale API request: %w", err)
-	}
-	// If status code was not successful, return the error.
-	if resp.StatusCode != http.StatusOK {
-		return handleErrorResponse(b, resp)
-	}
-	return nil
-}
-
-// sendRequest add the authentication key to the request and sends it. It
-// receives the response and reads up to 10MB of it.
-func (c *tsClientImpl) sendRequest(req *http.Request) ([]byte, *http.Response, error) {
-	resp, err := c.Do(req)
-	if err != nil {
-		return nil, resp, fmt.Errorf("error actually doing request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	// Read response
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		err = fmt.Errorf("error reading response body: %v", err)
-	}
-	return b, resp, err
-}
-
-// handleErrorResponse decodes the error message from the server and returns
-// an ErrResponse from it.
-func handleErrorResponse(b []byte, resp *http.Response) error {
-	var errResp tailscale.ErrResponse
-	if err := json.Unmarshal(b, &errResp); err != nil {
-		return err
-	}
-	errResp.Status = resp.StatusCode
-	return errResp
+	GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error)
+	CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error
+	DeleteVIPService(ctx context.Context, name tailcfg.ServiceName) error
 }

cmd/k8s-operator/tsrecorder.go

@@ -230,7 +230,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
 func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
 	logger := r.logger(tsr.Name)
 
-	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
+	prefs, ok, err := r.getDevicePrefs(ctx, tsr.Name)
 	if err != nil {
 		return false, err
 	}
@@ -243,6 +243,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
 		return true, nil
 	}
 
+	id := string(prefs.Config.NodeID)
 	logger.Debugf("deleting device %s from control", string(id))
 	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
 		errResp := &tailscale.ErrResponse{}
@@ -327,34 +328,33 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string)
 	return secret, nil
 }
 
-func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+func (r *RecorderReconciler) getDevicePrefs(ctx context.Context, tsrName string) (prefs prefs, ok bool, err error) {
 	secret, err := r.getStateSecret(ctx, tsrName)
 	if err != nil || secret == nil {
-		return "", "", false, err
+		return prefs, false, err
 	}
 
-	return getNodeMetadata(ctx, secret)
+	return getDevicePrefs(secret)
 }
 
-// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// getDevicePrefs returns 'ok == true' iff the node ID is found. The dnsName
 // is expected to always be non-empty if the node ID is, but not required.
-func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) {
 	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
 	currentProfile, ok := secret.Data[currentProfileKey]
 	if !ok {
-		return "", "", false, nil
+		return prefs, false, nil
 	}
 	profileBytes, ok := secret.Data[string(currentProfile)]
 	if !ok {
-		return "", "", false, nil
+		return prefs, false, nil
 	}
-	var profile profile
-	if err := json.Unmarshal(profileBytes, &profile); err != nil {
-		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
+	if err := json.Unmarshal(profileBytes, &prefs); err != nil {
+		return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
 	}
 
-	ok = profile.Config.NodeID != ""
-	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
+	ok = prefs.Config.NodeID != ""
+	return prefs, ok, nil
 }
 
 func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
@@ -367,14 +367,14 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string)
 }
 
 func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	nodeID, dnsName, ok, err := getNodeMetadata(ctx, secret)
+	prefs, ok, err := getDevicePrefs(secret)
 	if !ok || err != nil {
 		return tsapi.RecorderTailnetDevice{}, false, err
 	}
 
 	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
 	// need the API. Should we instead update the profile to include addresses?
-	device, err := tsClient.Device(ctx, string(nodeID), nil)
+	device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil)
 	if err != nil {
 		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
 	}
@@ -383,20 +383,25 @@ func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret
 		Hostname:   device.Hostname,
 		TailnetIPs: device.Addresses,
 	}
-	if dnsName != "" {
+	if dnsName := prefs.Config.UserProfile.LoginName; dnsName != "" {
 		d.URL = fmt.Sprintf("https://%s", dnsName)
 	}
 
 	return d, true, nil
 }
 
-type profile struct {
+// [prefs] is a subset of the ipn.Prefs struct used for extracting information
+// from the state Secret of Tailscale devices.
+type prefs struct {
 	Config struct {
-		NodeID      string `json:"NodeID"`
+		NodeID      tailcfg.StableNodeID `json:"NodeID"`
 		UserProfile struct {
+			// LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net.
 			LoginName string `json:"LoginName"`
 		} `json:"UserProfile"`
 	} `json:"Config"`
+
+	AdvertiseServices []string `json:"AdvertiseServices"`
 }
 
 func markedForDeletion(obj metav1.Object) bool {

cmd/natc/natc.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"expvar"
 	"flag"
 	"fmt"
 	"log"
@@ -26,7 +27,7 @@ import (
 	"github.com/inetaf/tcpproxy"
 	"github.com/peterbourgon/ff/v3"
 	"golang.org/x/net/dns/dnsmessage"
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/envknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -37,8 +38,11 @@ import (
 	"tailscale.com/tsweb"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
+	"tailscale.com/wgengine/netstack"
 )
 
+var ErrNoIPsAvailable = errors.New("no IPs available")
+
 func main() {
 	hostinfo.SetApp("natc")
 	if !envknob.UseWIPCode() {
@@ -90,18 +94,24 @@ func main() {
 		}
 		ignoreDstTable.Insert(pfx, true)
 	}
-	var v4Prefixes []netip.Prefix
+	var (
+		v4Prefixes    []netip.Prefix
+		numV4DNSAddrs int
+	)
 	for _, s := range strings.Split(*v4PfxStr, ",") {
 		p := netip.MustParsePrefix(strings.TrimSpace(s))
 		if p.Masked() != p {
 			log.Fatalf("v4 prefix %v is not a masked prefix", p)
 		}
 		v4Prefixes = append(v4Prefixes, p)
+		numIPs := 1 << (32 - p.Bits())
+		numV4DNSAddrs += numIPs
 	}
 	if len(v4Prefixes) == 0 {
 		log.Fatalf("no v4 prefixes specified")
 	}
 	dnsAddr := v4Prefixes[0].Addr()
+	numV4DNSAddrs -= 1 // Subtract the dnsAddr allocated above.
 	ts := &tsnet.Server{
 		Hostname: *hostname,
 	}
@@ -112,6 +122,7 @@ func main() {
 		ts.Port = uint16(*wgPort)
 	}
 	defer ts.Close()
+
 	if *verboseTSNet {
 		ts.Logf = log.Printf
 	}
@@ -129,6 +140,16 @@ func main() {
 			log.Fatalf("debug serve: %v", http.Serve(dln, mux))
 		}()
 	}
+
+	if err := ts.Start(); err != nil {
+		log.Fatalf("ts.Start: %v", err)
+	}
+	// TODO(raggi): this is not a public interface or guarantee.
+	ns := ts.Sys().Netstack.Get().(*netstack.Impl)
+	if *debugPort != 0 {
+		expvar.Publish("netstack", ns.ExpVar())
+	}
+
 	lc, err := ts.LocalClient()
 	if err != nil {
 		log.Fatalf("LocalClient() failed: %v", err)
@@ -138,12 +159,13 @@ func main() {
 	}
 
 	c := &connector{
-		ts:         ts,
-		lc:         lc,
-		dnsAddr:    dnsAddr,
-		v4Ranges:   v4Prefixes,
-		v6ULA:      ula(uint16(*siteID)),
-		ignoreDsts: ignoreDstTable,
+		ts:            ts,
+		lc:            lc,
+		dnsAddr:       dnsAddr,
+		v4Ranges:      v4Prefixes,
+		numV4DNSAddrs: numV4DNSAddrs,
+		v6ULA:         ula(uint16(*siteID)),
+		ignoreDsts:    ignoreDstTable,
 	}
 	c.run(ctx)
 }
@@ -151,9 +173,9 @@ func main() {
 type connector struct {
 	// ts is the tsnet.Server used to host the connector.
 	ts *tsnet.Server
-	// lc is the LocalClient used to interact with the tsnet.Server hosting this
+	// lc is the local.Client used to interact with the tsnet.Server hosting this
 	// connector.
-	lc *tailscale.LocalClient
+	lc *local.Client
 
 	// dnsAddr is the IPv4 address to listen on for DNS requests. It is used to
 	// prevent the app connector from assigning it to a domain.
@@ -162,6 +184,11 @@ type connector struct {
 	// v4Ranges is the list of IPv4 ranges to advertise and assign addresses from.
 	// These are masked prefixes.
 	v4Ranges []netip.Prefix
+
+	// numV4DNSAddrs is the total size of the IPv4 ranges in addresses, minus the
+	// dnsAddr allocation.
+	numV4DNSAddrs int
+
 	// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
 	v6ULA netip.Prefix
 
@@ -264,14 +291,14 @@ func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDP
 	defer cancel()
 	who, err := c.lc.WhoIs(ctx, remoteAddr.String())
 	if err != nil {
-		log.Printf("HandleDNS: WhoIs failed: %v\n", err)
+		log.Printf("HandleDNS(remote=%s): WhoIs failed: %v\n", remoteAddr.String(), err)
 		return
 	}
 
 	var msg dnsmessage.Message
 	err = msg.Unpack(buf)
 	if err != nil {
-		log.Printf("HandleDNS: dnsmessage unpack failed: %v\n ", err)
+		log.Printf("HandleDNS(remote=%s): dnsmessage unpack failed: %v\n", remoteAddr.String(), err)
 		return
 	}
 
@@ -284,19 +311,19 @@ func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDP
 			case dnsmessage.TypeAAAA, dnsmessage.TypeA:
 				dstAddrs, err := lookupDestinationIP(q.Name.String())
 				if err != nil {
-					log.Printf("HandleDNS: lookup destination failed: %v\n ", err)
+					log.Printf("HandleDNS(remote=%s): lookup destination failed: %v\n", remoteAddr.String(), err)
 					return
 				}
 				if c.ignoreDestination(dstAddrs) {
 					bs, err := dnsResponse(&msg, dstAddrs)
 					// TODO (fran): treat as SERVFAIL
 					if err != nil {
-						log.Printf("HandleDNS: generate ignore response failed: %v\n", err)
+						log.Printf("HandleDNS(remote=%s): generate ignore response failed: %v\n", remoteAddr.String(), err)
 						return
 					}
 					_, err = pc.WriteTo(bs, remoteAddr)
 					if err != nil {
-						log.Printf("HandleDNS: write failed: %v\n", err)
+						log.Printf("HandleDNS(remote=%s): write failed: %v\n", remoteAddr.String(), err)
 					}
 					return
 				}
@@ -309,7 +336,7 @@ func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDP
 	resp, err := c.generateDNSResponse(&msg, who.Node.ID)
 	// TODO (fran): treat as SERVFAIL
 	if err != nil {
-		log.Printf("HandleDNS: connector handling failed: %v\n", err)
+		log.Printf("HandleDNS(remote=%s): connector handling failed: %v\n", remoteAddr.String(), err)
 		return
 	}
 	// TODO (fran): treat as NXDOMAIN
@@ -319,7 +346,7 @@ func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDP
 	// This connector handled the DNS request
 	_, err = pc.WriteTo(resp, remoteAddr)
 	if err != nil {
-		log.Printf("HandleDNS: write failed: %v\n", err)
+		log.Printf("HandleDNS(remote=%s): write failed: %v\n", remoteAddr.String(), err)
 	}
 }
 
@@ -487,6 +514,7 @@ type perPeerState struct {
 	mu           sync.Mutex
 	domainToAddr map[string][]netip.Addr
 	addrToDomain *bart.Table[string]
+	numV4Allocs  int
 }
 
 // domainForIP returns the domain name assigned to the given IP address and
@@ -516,6 +544,9 @@ func (ps *perPeerState) ipForDomain(domain string) ([]netip.Addr, error) {
 		return addrs, nil
 	}
 	addrs := ps.assignAddrsLocked(domain)
+	if addrs == nil {
+		return nil, ErrNoIPsAvailable
+	}
 	return addrs, nil
 }
 
@@ -529,17 +560,25 @@ func (ps *perPeerState) isIPUsedLocked(ip netip.Addr) bool {
 
 // unusedIPv4Locked returns an unused IPv4 address from the available ranges.
 func (ps *perPeerState) unusedIPv4Locked() netip.Addr {
+	// All addresses have been allocated.
+	if ps.numV4Allocs >= ps.c.numV4DNSAddrs {
+		return netip.Addr{}
+	}
+
 	// TODO: skip ranges that have been exhausted
-	for _, r := range ps.c.v4Ranges {
-		ip := randV4(r)
-		for r.Contains(ip) {
+	// TODO: implement a much more efficient algorithm for finding unused IPs,
+	// this is fairly crazy.
+	for {
+		for _, r := range ps.c.v4Ranges {
+			ip := randV4(r)
+			if !r.Contains(ip) {
+				panic("error: randV4 returned invalid address")
+			}
 			if !ps.isIPUsedLocked(ip) && ip != ps.c.dnsAddr {
 				return ip
 			}
-			ip = ip.Next()
 		}
 	}
-	return netip.Addr{}
 }
 
 // randV4 returns a random IPv4 address within the given prefix.
@@ -562,6 +601,10 @@ func (ps *perPeerState) assignAddrsLocked(domain string) []netip.Addr {
 		ps.addrToDomain = &bart.Table[string]{}
 	}
 	v4 := ps.unusedIPv4Locked()
+	if !v4.IsValid() {
+		return nil
+	}
+	ps.numV4Allocs++
 	as16 := ps.c.v6ULA.Addr().As16()
 	as4 := v4.As4()
 	copy(as16[12:], as4[:])

cmd/natc/natc_test.go

@@ -0,0 +1,429 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"slices"
+	"testing"
+
+	"github.com/gaissmai/bart"
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/tailcfg"
+)
+
+func prefixEqual(a, b netip.Prefix) bool {
+	return a.Bits() == b.Bits() && a.Addr() == b.Addr()
+}
+
+func TestULA(t *testing.T) {
+	tests := []struct {
+		name     string
+		siteID   uint16
+		expected string
+	}{
+		{"zero", 0, "fd7a:115c:a1e0:a99c:0000::/80"},
+		{"one", 1, "fd7a:115c:a1e0:a99c:0001::/80"},
+		{"max", 65535, "fd7a:115c:a1e0:a99c:ffff::/80"},
+		{"random", 12345, "fd7a:115c:a1e0:a99c:3039::/80"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ula(tc.siteID)
+			expected := netip.MustParsePrefix(tc.expected)
+			if !prefixEqual(got, expected) {
+				t.Errorf("ula(%d) = %s; want %s", tc.siteID, got, expected)
+			}
+		})
+	}
+}
+
+func TestRandV4(t *testing.T) {
+	pfx := netip.MustParsePrefix("100.64.1.0/24")
+
+	for i := 0; i < 512; i++ {
+		ip := randV4(pfx)
+		if !pfx.Contains(ip) {
+			t.Errorf("randV4(%s) = %s; not contained in prefix", pfx, ip)
+		}
+	}
+}
+
+func TestDNSResponse(t *testing.T) {
+	tests := []struct {
+		name        string
+		questions   []dnsmessage.Question
+		addrs       []netip.Addr
+		wantEmpty   bool
+		wantAnswers []struct {
+			name  string
+			qType dnsmessage.Type
+			addr  netip.Addr
+		}
+	}{
+		{
+			name:        "empty_request",
+			questions:   []dnsmessage.Question{},
+			addrs:       []netip.Addr{},
+			wantEmpty:   false,
+			wantAnswers: nil,
+		},
+		{
+			name: "a_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs: []netip.Addr{netip.MustParseAddr("100.64.1.5")},
+			wantAnswers: []struct {
+				name  string
+				qType dnsmessage.Type
+				addr  netip.Addr
+			}{
+				{
+					name:  "example.com.",
+					qType: dnsmessage.TypeA,
+					addr:  netip.MustParseAddr("100.64.1.5"),
+				},
+			},
+		},
+		{
+			name: "aaaa_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeAAAA,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs: []netip.Addr{netip.MustParseAddr("fd7a:115c:a1e0:a99c:0001:0505:0505:0505")},
+			wantAnswers: []struct {
+				name  string
+				qType dnsmessage.Type
+				addr  netip.Addr
+			}{
+				{
+					name:  "example.com.",
+					qType: dnsmessage.TypeAAAA,
+					addr:  netip.MustParseAddr("fd7a:115c:a1e0:a99c:0001:0505:0505:0505"),
+				},
+			},
+		},
+		{
+			name: "soa_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeSOA,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs:       []netip.Addr{},
+			wantAnswers: nil,
+		},
+		{
+			name: "ns_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeNS,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs:       []netip.Addr{},
+			wantAnswers: nil,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			req := &dnsmessage.Message{
+				Header: dnsmessage.Header{
+					ID: 1234,
+				},
+				Questions: tc.questions,
+			}
+
+			resp, err := dnsResponse(req, tc.addrs)
+			if err != nil {
+				t.Fatalf("dnsResponse() error = %v", err)
+			}
+
+			if tc.wantEmpty && len(resp) != 0 {
+				t.Errorf("dnsResponse() returned non-empty response when expected empty")
+			}
+
+			if !tc.wantEmpty && len(resp) == 0 {
+				t.Errorf("dnsResponse() returned empty response when expected non-empty")
+			}
+
+			if len(resp) > 0 {
+				var msg dnsmessage.Message
+				err = msg.Unpack(resp)
+				if err != nil {
+					t.Fatalf("Failed to unpack response: %v", err)
+				}
+
+				if !msg.Header.Response {
+					t.Errorf("Response header is not set")
+				}
+
+				if msg.Header.ID != req.Header.ID {
+					t.Errorf("Response ID = %d, want %d", msg.Header.ID, req.Header.ID)
+				}
+
+				if len(tc.wantAnswers) > 0 {
+					if len(msg.Answers) != len(tc.wantAnswers) {
+						t.Errorf("got %d answers, want %d", len(msg.Answers), len(tc.wantAnswers))
+					} else {
+						for i, want := range tc.wantAnswers {
+							ans := msg.Answers[i]
+
+							gotName := ans.Header.Name.String()
+							if gotName != want.name {
+								t.Errorf("answer[%d] name = %s, want %s", i, gotName, want.name)
+							}
+
+							if ans.Header.Type != want.qType {
+								t.Errorf("answer[%d] type = %v, want %v", i, ans.Header.Type, want.qType)
+							}
+
+							var gotIP netip.Addr
+							switch want.qType {
+							case dnsmessage.TypeA:
+								if ans.Body.(*dnsmessage.AResource) == nil {
+									t.Errorf("answer[%d] not an A record", i)
+									continue
+								}
+								resource := ans.Body.(*dnsmessage.AResource)
+								gotIP = netip.AddrFrom4([4]byte(resource.A))
+							case dnsmessage.TypeAAAA:
+								if ans.Body.(*dnsmessage.AAAAResource) == nil {
+									t.Errorf("answer[%d] not an AAAA record", i)
+									continue
+								}
+								resource := ans.Body.(*dnsmessage.AAAAResource)
+								gotIP = netip.AddrFrom16([16]byte(resource.AAAA))
+							}
+
+							if gotIP != want.addr {
+								t.Errorf("answer[%d] IP = %s, want %s", i, gotIP, want.addr)
+							}
+						}
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestPerPeerState(t *testing.T) {
+	c := &connector{
+		v4Ranges:      []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
+		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
+		numV4DNSAddrs: (1<<(32-24) - 1),
+	}
+
+	ps := &perPeerState{c: c}
+
+	addrs, err := ps.ipForDomain("example.com")
+	if err != nil {
+		t.Fatalf("ipForDomain() error = %v", err)
+	}
+
+	if len(addrs) != 2 {
+		t.Fatalf("ipForDomain() returned %d addresses, want 2", len(addrs))
+	}
+
+	v4 := addrs[0]
+	v6 := addrs[1]
+
+	if !v4.Is4() {
+		t.Errorf("First address is not IPv4: %s", v4)
+	}
+
+	if !v6.Is6() {
+		t.Errorf("Second address is not IPv6: %s", v6)
+	}
+
+	if !c.v4Ranges[0].Contains(v4) {
+		t.Errorf("IPv4 address %s not in range %s", v4, c.v4Ranges[0])
+	}
+
+	domain, ok := ps.domainForIP(v4)
+	if !ok {
+		t.Errorf("domainForIP(%s) not found", v4)
+	} else if domain != "example.com" {
+		t.Errorf("domainForIP(%s) = %s, want %s", v4, domain, "example.com")
+	}
+
+	domain, ok = ps.domainForIP(v6)
+	if !ok {
+		t.Errorf("domainForIP(%s) not found", v6)
+	} else if domain != "example.com" {
+		t.Errorf("domainForIP(%s) = %s, want %s", v6, domain, "example.com")
+	}
+
+	addrs2, err := ps.ipForDomain("example.com")
+	if err != nil {
+		t.Fatalf("ipForDomain() second call error = %v", err)
+	}
+
+	if !slices.Equal(addrs, addrs2) {
+		t.Errorf("ipForDomain() second call = %v, want %v", addrs2, addrs)
+	}
+}
+
+func TestIgnoreDestination(t *testing.T) {
+	ignoreDstTable := &bart.Table[bool]{}
+	ignoreDstTable.Insert(netip.MustParsePrefix("192.168.1.0/24"), true)
+	ignoreDstTable.Insert(netip.MustParsePrefix("10.0.0.0/8"), true)
+
+	c := &connector{
+		ignoreDsts: ignoreDstTable,
+	}
+
+	tests := []struct {
+		name     string
+		addrs    []netip.Addr
+		expected bool
+	}{
+		{
+			name:     "no_match",
+			addrs:    []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("1.1.1.1")},
+			expected: false,
+		},
+		{
+			name:     "one_match",
+			addrs:    []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("192.168.1.5")},
+			expected: true,
+		},
+		{
+			name:     "all_match",
+			addrs:    []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("192.168.1.5")},
+			expected: true,
+		},
+		{
+			name:     "empty_addrs",
+			addrs:    []netip.Addr{},
+			expected: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := c.ignoreDestination(tc.addrs)
+			if got != tc.expected {
+				t.Errorf("ignoreDestination(%v) = %v, want %v", tc.addrs, got, tc.expected)
+			}
+		})
+	}
+}
+
+func TestConnectorGenerateDNSResponse(t *testing.T) {
+	c := &connector{
+		v4Ranges:      []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
+		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
+		numV4DNSAddrs: (1<<(32-24) - 1),
+	}
+
+	req := &dnsmessage.Message{
+		Header: dnsmessage.Header{ID: 1234},
+		Questions: []dnsmessage.Question{
+			{
+				Name:  dnsmessage.MustNewName("example.com."),
+				Type:  dnsmessage.TypeA,
+				Class: dnsmessage.ClassINET,
+			},
+		},
+	}
+
+	nodeID := tailcfg.NodeID(12345)
+
+	resp1, err := c.generateDNSResponse(req, nodeID)
+	if err != nil {
+		t.Fatalf("generateDNSResponse() error = %v", err)
+	}
+	if len(resp1) == 0 {
+		t.Fatalf("generateDNSResponse() returned empty response")
+	}
+
+	resp2, err := c.generateDNSResponse(req, nodeID)
+	if err != nil {
+		t.Fatalf("generateDNSResponse() second call error = %v", err)
+	}
+
+	if !cmp.Equal(resp1, resp2) {
+		t.Errorf("generateDNSResponse() responses differ between calls")
+	}
+}
+
+func TestIPPoolExhaustion(t *testing.T) {
+	smallPrefix := netip.MustParsePrefix("100.64.1.0/30") // Only 4 IPs: .0, .1, .2, .3
+	c := &connector{
+		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		v4Ranges:      []netip.Prefix{smallPrefix},
+		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
+		numV4DNSAddrs: 3,
+	}
+
+	ps := &perPeerState{c: c}
+
+	assignedIPs := make(map[netip.Addr]string)
+
+	domains := []string{"a.example.com", "b.example.com", "c.example.com", "d.example.com"}
+
+	var errs []error
+
+	for i := 0; i < 5; i++ {
+		for _, domain := range domains {
+			addrs, err := ps.ipForDomain(domain)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to get IP for domain %q: %w", domain, err))
+				continue
+			}
+
+			for _, addr := range addrs {
+				if d, ok := assignedIPs[addr]; ok {
+					if d != domain {
+						t.Errorf("IP %s reused for domain %q, previously assigned to %q", addr, domain, d)
+					}
+				} else {
+					assignedIPs[addr] = domain
+				}
+			}
+		}
+	}
+
+	for addr, domain := range assignedIPs {
+		if addr.Is4() && !smallPrefix.Contains(addr) {
+			t.Errorf("IP %s for domain %q not in expected range %s", addr, domain, smallPrefix)
+		}
+		if addr.Is6() && !c.v6ULA.Contains(addr) {
+			t.Errorf("IP %s for domain %q not in expected range %s", addr, domain, c.v6ULA)
+		}
+		if addr == c.dnsAddr {
+			t.Errorf("IP %s for domain %q is the reserved DNS address", addr, domain)
+		}
+	}
+
+	// expect one error for each iteration with the 4th domain
+	if len(errs) != 5 {
+		t.Errorf("Expected 5 errors, got %d: %v", len(errs), errs)
+	}
+	for _, err := range errs {
+		if !errors.Is(err, ErrNoIPsAvailable) {
+			t.Errorf("generateDNSResponse() error = %v, want ErrNoIPsAvailable", err)
+		}
+	}
+}

cmd/pgproxy/pgproxy.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 	"time"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/metrics"
 	"tailscale.com/tsnet"
 	"tailscale.com/tsweb"
@@ -105,7 +105,7 @@ type proxy struct {
 	upstreamHost     string // "my.database.com"
 	upstreamCertPool *x509.CertPool
 	downstreamCert   []tls.Certificate
-	client           *tailscale.LocalClient
+	client           *local.Client
 
 	activeSessions  expvar.Int
 	startedSessions expvar.Int
@@ -115,7 +115,7 @@ type proxy struct {
 // newProxy returns a proxy that forwards connections to
 // upstreamAddr. The upstream's TLS session is verified using the CA
 // cert(s) in upstreamCAPath.
-func newProxy(upstreamAddr, upstreamCAPath string, client *tailscale.LocalClient) (*proxy, error) {
+func newProxy(upstreamAddr, upstreamCAPath string, client *local.Client) (*proxy, error) {
 	bs, err := os.ReadFile(upstreamCAPath)
 	if err != nil {
 		return nil, err

cmd/proxy-to-grafana/proxy-to-grafana.go

@@ -19,8 +19,25 @@
 //	header_property = username
 //	auto_sign_up = true
 //	whitelist = 127.0.0.1
-//	headers = Name:X-WEBAUTH-NAME
+//	headers = Email:X-Webauth-User, Name:X-Webauth-Name, Role:X-Webauth-Role
 //	enable_login_token = true
+//
+// You can use grants in Tailscale ACL to give users different roles in Grafana.
+// For example, to give group:eng the Editor role, add the following to your ACLs:
+//
+//	 "grants": [
+//			{
+//				"src": ["group:eng"],
+//				"dst": ["tag:grafana"],
+//				"app": {
+//					"tailscale.com/cap/proxy-to-grafana": [{
+//						"role": "editor",
+//					}],
+//				},
+//			},
+//	 ],
+//
+// If multiple roles are specified, the most permissive role is used.
 package main
 
 import (
@@ -36,7 +53,7 @@ import (
 	"strings"
 	"time"
 
-	"tailscale.com/client/tailscale"
+	"tailscale.com/client/local"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 )
@@ -49,6 +66,57 @@ var (
 	loginServer  = flag.String("login-server", "", "URL to alternative control server. If empty, the default Tailscale control is used.")
 )
 
+// aclCap is the Tailscale ACL capability used to configure proxy-to-grafana.
+const aclCap tailcfg.PeerCapability = "tailscale.com/cap/proxy-to-grafana"
+
+// aclGrant is an access control rule that assigns Grafana permissions
+// while provisioning a user.
+type aclGrant struct {
+	// Role is one of: "viewer", "editor", "admin".
+	Role string `json:"role"`
+}
+
+// grafanaRole defines possible Grafana roles.
+type grafanaRole int
+
+const (
+	// Roles are ordered by their permissions, with the least permissive role first.
+	// If a user has multiple roles, the most permissive role is used.
+	ViewerRole grafanaRole = iota
+	EditorRole
+	AdminRole
+)
+
+// String returns the string representation of a grafanaRole.
+// It is used as a header value in the HTTP request to Grafana.
+func (r grafanaRole) String() string {
+	switch r {
+	case ViewerRole:
+		return "Viewer"
+	case EditorRole:
+		return "Editor"
+	case AdminRole:
+		return "Admin"
+	default:
+		// A safe default.
+		return "Viewer"
+	}
+}
+
+// roleFromString converts a string to a grafanaRole.
+// It is used to parse the role from the ACL grant.
+func roleFromString(s string) (grafanaRole, error) {
+	switch strings.ToLower(s) {
+	case "viewer":
+		return ViewerRole, nil
+	case "editor":
+		return EditorRole, nil
+	case "admin":
+		return AdminRole, nil
+	}
+	return ViewerRole, fmt.Errorf("unknown role: %q", s)
+}
+
 func main() {
 	flag.Parse()
 	if *hostname == "" || strings.Contains(*hostname, ".") {
@@ -127,14 +195,22 @@ func main() {
 	log.Fatal(http.Serve(ln, proxy))
 }
 
-func modifyRequest(req *http.Request, localClient *tailscale.LocalClient) {
+func modifyRequest(req *http.Request, localClient *local.Client) {
 	// with enable_login_token set to true, we get a cookie that handles
 	// auth for paths that are not /login
 	if req.URL.Path != "/login" {
 		return
 	}
 
-	user, err := getTailscaleUser(req.Context(), localClient, req.RemoteAddr)
+	// Delete any existing X-Webauth-* headers to prevent possible spoofing
+	// if getting Tailnet identity fails.
+	for h := range req.Header {
+		if strings.HasPrefix(h, "X-Webauth-") {
+			req.Header.Del(h)
+		}
+	}
+
+	user, role, err := getTailscaleIdentity(req.Context(), localClient, req.RemoteAddr)
 	if err != nil {
 		log.Printf("error getting Tailscale user: %v", err)
 		return
@@ -142,19 +218,33 @@ func modifyRequest(req *http.Request, localClient *tailscale.LocalClient) {
 
 	req.Header.Set("X-Webauth-User", user.LoginName)
 	req.Header.Set("X-Webauth-Name", user.DisplayName)
+	req.Header.Set("X-Webauth-Role", role.String())
 }
 
-func getTailscaleUser(ctx context.Context, localClient *tailscale.LocalClient, ipPort string) (*tailcfg.UserProfile, error) {
+func getTailscaleIdentity(ctx context.Context, localClient *local.Client, ipPort string) (*tailcfg.UserProfile, grafanaRole, error) {
 	whois, err := localClient.WhoIs(ctx, ipPort)
 	if err != nil {
-		return nil, fmt.Errorf("failed to identify remote host: %w", err)
+		return nil, ViewerRole, fmt.Errorf("failed to identify remote host: %w", err)
 	}
 	if whois.Node.IsTagged() {
-		return nil, fmt.Errorf("tagged nodes are not users")
+		return nil, ViewerRole, fmt.Errorf("tagged nodes are not users")
 	}
 	if whois.UserProfile == nil || whois.UserProfile.LoginName == "" {
-		return nil, fmt.Errorf("failed to identify remote user")
+		return nil, ViewerRole, fmt.Errorf("failed to identify remote user")
 	}
 
-	return whois.UserProfile, nil
+	role := ViewerRole
+	grants, err := tailcfg.UnmarshalCapJSON[aclGrant](whois.CapMap, aclCap)
+	if err != nil {
+		return nil, ViewerRole, fmt.Errorf("failed to unmarshal ACL grants: %w", err)
+	}
+	for _, g := range grants {
+		r, err := roleFromString(g.Role)
+		if err != nil {
+			return nil, ViewerRole, fmt.Errorf("failed to parse role: %w", err)
+		}
+		role = max(role, r)
+	}
+
+	return whois.UserProfile, role, nil
 }