control/keyfallback: add baked-in fallback for control key

Similar to how we bake in the DERPMap to ensure that we can reach the
DERP servers if DNS isn't working, also bake in the control key for the
default control server that we use if the control server is down.

Updates #13890

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I18ef0381e266bd3db10063685993bc3cb76b2f42

.github/workflows/checklocks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: "actions/checkout@v4"
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,18 +23,18 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: go.mod
           cache: false
 
       - name: golangci-lint
-        # Note: this is the 'v3' tag as of 2023-08-14
-        uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299
+        # Note: this is the 'v6.1.0' tag as of 2024-08-21
+        uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86
         with:
-          version: v1.56
+          version: v1.60
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -24,7 +24,7 @@ jobs:
 
       - name: Post to slack
         if: failure() && github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
         with:

.github/workflows/installer.yml

@@ -98,7 +98,7 @@ jobs:
       # We cannot use v4, as it requires a newer glibc version than some of the
       # tested images provide. See
       # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@v3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -78,9 +78,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -150,16 +150,16 @@ jobs:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -190,7 +190,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +202,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +214,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +258,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -295,7 +295,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -317,9 +317,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -350,7 +350,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -365,9 +365,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@v3
+      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -399,7 +399,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -456,18 +456,22 @@ jobs:
         fuzz-seconds: 300
         dry-run: false
         language: go
+    - name: Set artifacts_path in env (workaround for actions/upload-artifact#176)
+      if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
+      run: |
+        echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
-        path: ./out/artifacts
+        path: ${{ env.artifacts_path }}/out/artifacts
 
   depaware:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
@@ -477,7 +481,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -490,7 +494,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -502,7 +506,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -518,7 +522,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck
@@ -559,7 +563,7 @@ jobs:
       # By having the job always run, but skipping its only step as needed, we
       # let the CI output collapse nicely in PRs.
       if: failure() && github.event_name == 'push'
-      uses: ruby/action-slack@v3.2.1
+      uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
       with:
         payload: |
           {
@@ -574,6 +578,7 @@ jobs:
           }
       env:
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   check_mergeability:
     if: always()
@@ -596,6 +601,6 @@ jobs:
     steps:
     - name: Decide if change is okay to merge
       if: github.event_name != 'push'
-      uses: re-actors/alls-green@release/v1
+      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
       with:
         jobs: ${{ toJSON(needs) }}

.github/workflows/update-flake.yml

@@ -21,21 +21,22 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run update-flakes
         run: ./update-flake.sh
 
       - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         id: generate-token
         with:
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_retrieval_mode: "id"
+          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run go get
         run: |
@@ -23,18 +23,19 @@ jobs:
           ./tool/go mod tidy
 
       - name: Get access token
-        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         id: generate-token
         with:
           # TODO(will): this should use the code updater app rather than licensing.
           # It has the same permissions, so not a big deal, but still.
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_retrieval_mode: "id"
+          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

.gitignore

@@ -43,3 +43,9 @@ client/web/build/assets
 
 /gocross
 /dist
+
+# Ignore xcode userstate and workspace data
+*.xcuserstate
+*.xcworkspacedata
+/tstest/tailmac/bin
+/tstest/tailmac/build

Dockerfile

@@ -27,7 +27,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.22-alpine AS build-env
+FROM golang:1.23-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
@@ -42,7 +42,7 @@ RUN go install \
     gvisor.dev/gvisor/pkg/tcpip/stack \
     golang.org/x/crypto/ssh \
     golang.org/x/crypto/acme \
-    nhooyr.io/websocket \
+    github.com/coder/websocket \
     github.com/mdlayher/netlink
 
 COPY . .

Makefile

@@ -117,7 +117,8 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
+	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
 
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"

README.md

@@ -37,7 +37,7 @@ not open source.
 
 ## Building
 
-We always require the latest Go release, currently Go 1.22. (While we build
+We always require the latest Go release, currently Go 1.23. (While we build
 releases with our [Go fork](https://github.com/tailscale/go/), its use is not
 required.)
 

VERSION.txt

@@ -1 +1 @@
-1.71.0
+1.77.0

api.md

@@ -1,104 +1,2 @@
 > [!IMPORTANT]
 > The Tailscale API documentation has moved to https://tailscale.com/api
-
-# Tailscale API
-
-The Tailscale API documentation is located in **[tailscale/publicapi](./publicapi/readme.md#tailscale-api)**.
-
-# APIs
-
-**[Overview](./publicapi/readme.md)**
-
-**[Device](./publicapi/device.md#device)**
-
-<a href="device-delete"></a>
-<a href="expire-device-key"></a>
-<a href="device-routes-get">
-<a href="device-routes-post"></a>
-<a href="#device-authorized-post"></a>
-<a href="device-tags-post"></a>
-<a href="device-key-post"></a>
-<a href="tailnet-acl-get"></a>
-
-- Get a device: [`GET /api/v2/device/{deviceid}`](./publicapi/device.md#get-device)
-- Delete a device: [`DELETE /api/v2/device/{deviceID}`](./publicapi/device.md#delete-device)
-- Expire device key: [`POST /api/v2/device/{deviceID}/expire`](./publicapi/device.md#expire-device-key)
-- [**Routes**](./publicapi/device.md#routes)
-  - Get device routes: [`GET /api/v2/device/{deviceID}/routes`](./publicapi/device.md#get-device-routes)
-  - Set device routes: [`POST /api/v2/device/{deviceID}/routes`](./publicapi/device.md#set-device-routes)
-- [**Authorize**](./publicapi/device.md#authorize)
-  - Authorize a device: [`POST /api/v2/device/{deviceID}/authorized`](./publicapi/device.md#authorize-device)
-- [**Tags**](./publicapi/device.md#tags)
-  - Update tags: [`POST /api/v2/device/{deviceID}/tags`](./publicapi/device.md#update-device-tags)
-- [**Keys**](./publicapi/device.md#keys)
-  - Update device key: [`POST /api/v2/device/{deviceID}/key`](./publicapi/device.md#update-device-key)
-- [**IP Addresses**](./publicapi/device.md#ip-addresses)
-  - Set device IPv4 address: [`POST /api/v2/device/{deviceID}/ip`](./publicapi/device.md#set-device-ipv4-address)
-- [**Device posture attributes**](./publicapi/device.md#device-posture-attributes)
-  - Get device posture attributes: [`GET /api/v2/device/{deviceID}/attributes`](./publicapi/device.md#get-device-posture-attributes)
-  - Set custom device posture attributes: [`POST /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#set-device-posture-attributes)
-  - Delete custom device posture attributes: [`DELETE /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#delete-custom-device-posture-attributes)
-- [**Device invites**](./publicapi/device.md#invites-to-a-device)
-  - List device invites: [`GET /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#list-device-invites)
-  - Create device invites: [`POST /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#create-device-invites)
-
-**[Tailnet](./publicapi/tailnet.md#tailnet)**
-
-<a href="tailnet-acl-post"></a>
-<a href="tailnet-acl-preview-post"></a>
-<a href="tailnet-acl-validate-post"></a>
-<a href="tailnet-devices"></a>
-<a href="tailnet-keys-get"></a>
-<a href="tailnet-keys-post"></a>
-<a href="tailnet-keys-key-get"></a>
-<a href="tailnet-keys-key-delete"></a>
-<a href="tailnet-dns"></a>
-<a href="tailnet-dns-nameservers-get"></a>
-<a href="tailnet-dns-nameservers-post"></a>
-<a href="tailnet-dns-preferences-get"></a>
-<a href="tailnet-dns-preferences-post"></a>
-<a href="tailnet-dns-searchpaths-get"></a>
-<a href="tailnet-dns-searchpaths-post"></a>
-
-- [**Policy File**](./publicapi/tailnet.md#policy-file)
-  - Get policy file: [`GET /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#get-policy-file)
-  - Update policy file: [`POST /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#update-policy-file)
-  - Preview rule matches: [`POST /api/v2/tailnet/{tailnet}/acl/preview`](./publicapi/tailnet.md#preview-policy-file-rule-matches)
-  - Validate and test policy file: [`POST /api/v2/tailnet/{tailnet}/acl/validate`](./publicapi/tailnet.md#validate-and-test-policy-file)
-- [**Devices**](./publicapi/tailnet.md#devices)
-  - List tailnet devices: [`GET /api/v2/tailnet/{tailnet}/devices`](./publicapi/tailnet.md#list-tailnet-devices)
-- [**Keys**](./publicapi/tailnet.md#tailnet-keys)
-  - List tailnet keys: [`GET /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#list-tailnet-keys)
-  - Create an auth key: [`POST /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#create-auth-key)
-  - Get a key: [`GET /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#get-key)
-  - Delete a key: [`DELETE /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#delete-key)
-- [**DNS**](./publicapi/tailnet.md#dns)
-  - [**Nameservers**](./publicapi/tailnet.md#nameservers)
-    - Get nameservers: [`GET /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#get-nameservers)
-    - Set nameservers: [`POST /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#set-nameservers)
-  - [**Preferences**](./publicapi/tailnet.md#preferences)
-    - Get DNS preferences: [`GET /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#get-dns-preferences)
-    - Set DNS preferences: [`POST /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#set-dns-preferences)
-  - [**Search Paths**](./publicapi/tailnet.md#search-paths)
-    - Get search paths: [`GET /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#get-search-paths)
-    - Set search paths: [`POST /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#set-search-paths)
-  - [**Split DNS**](./publicapi/tailnet.md#split-dns)
-    - Get split DNS: [`GET /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#get-split-dns)
-    - Update split DNS: [`PATCH /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#update-split-dns)
-    - Set split DNS: [`PUT /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#set-split-dns)
-- [**User invites**](./publicapi/tailnet.md#tailnet-user-invites)
-  - List user invites: [`GET /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#list-user-invites)
-  - Create user invites: [`POST /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#create-user-invites)
-
-**[User invites](./publicapi/userinvites.md#user-invites)**
-
-- Get user invite: [`GET /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#get-user-invite)
-- Delete user invite: [`DELETE /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#delete-user-invite)
-- Resend user invite (by email): [`POST /api/v2/user-invites/{userInviteId}/resend`](#resend-user-invite)
-
-**[Device invites](./publicapi/deviceinvites.md#device-invites)**
-
-- Get device invite: [`GET /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#get-device-invite)
-- Delete device invite: [`DELETE /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#delete-device-invite)
-- Resend device invite (by email): [`POST /api/v2/device-invites/{deviceInviteId}/resend`](./publicapi/deviceinvites.md#resend-device-invite)
-- Accept device invite [`POST /api/v2/device-invites/-/accept`](#accept-device-invite)

assert_ts_toolchain_match.go

@@ -0,0 +1,27 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build tailscale_go
+
+package tailscaleroot
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func init() {
+	tsRev, ok := tailscaleToolchainRev()
+	if !ok {
+		panic("binary built with tailscale_go build tag but failed to read build info or find tailscale.toolchain.rev in build info")
+	}
+	want := strings.TrimSpace(GoToolchainRev)
+	if tsRev != want {
+		if os.Getenv("TS_PERMIT_TOOLCHAIN_MISMATCH") == "1" {
+			fmt.Fprintf(os.Stderr, "tailscale.toolchain.rev = %q, want %q; but ignoring due to TS_PERMIT_TOOLCHAIN_MISMATCH=1\n", tsRev, want)
+			return
+		}
+		panic(fmt.Sprintf("binary built with tailscale_go build tag but Go toolchain %q doesn't match github.com/tailscale/tailscale expected value %q; override this failure with TS_PERMIT_TOOLCHAIN_MISMATCH=1", tsRev, want))
+	}
+}

build_docker.sh

@@ -56,6 +56,7 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
@@ -72,6 +73,7 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \

client/tailscale/acl.go

@@ -19,6 +19,7 @@ import (
 // Only one of Src/Dst or Users/Ports may be specified.
 type ACLRow struct {
 	Action string   `json:"action,omitempty"` // valid values: "accept"
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Users  []string `json:"users,omitempty"`  // old name for src
 	Ports  []string `json:"ports,omitempty"`  // old name for dst
 	Src    []string `json:"src,omitempty"`
@@ -31,6 +32,7 @@ type ACLRow struct {
 type ACLTest struct {
 	Src    string   `json:"src,omitempty"`    // source
 	User   string   `json:"user,omitempty"`   // old name for source
+	Proto  string   `json:"proto,omitempty"`  // protocol
 	Accept []string `json:"accept,omitempty"` // expected destination ip:port that user can access
 	Deny   []string `json:"deny,omitempty"`   // expected destination ip:port that user cannot access
 
@@ -286,6 +288,9 @@ type UserRuleMatch struct {
 	Users      []string `json:"users"`
 	Ports      []string `json:"ports"`
 	LineNumber int      `json:"lineNumber"`
+	// Via is the list of targets through which Users can access Ports.
+	// See https://tailscale.com/kb/1378/via for more information.
+	Via []string `json:"via,omitempty"`
 
 	// Postures is a list of posture policies that are
 	// associated with this match. The rules can be looked

client/tailscale/apitype/apitype.go

@@ -4,7 +4,10 @@
 // Package apitype contains types for the Tailscale LocalAPI and control plane API.
 package apitype
 
-import "tailscale.com/tailcfg"
+import (
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/dnstype"
+)
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
@@ -57,3 +60,19 @@ type ExitNodeSuggestionResponse struct {
 	Name     string
 	Location tailcfg.LocationView `json:",omitempty"`
 }
+
+// DNSOSConfig mimics dns.OSConfig without forcing us to import the entire dns package
+// into the CLI.
+type DNSOSConfig struct {
+	Nameservers   []string
+	SearchDomains []string
+	MatchDomains  []string
+}
+
+// DNSQueryResponse is the response to a DNS query request sent via LocalAPI.
+type DNSQueryResponse struct {
+	// Bytes is the raw DNS response bytes.
+	Bytes []byte
+	// Resolvers is the list of resolvers that the forwarder deemed able to resolve the query.
+	Resolvers []*dnstype.Resolver
+}

client/tailscale/localclient.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
+	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
 )
@@ -69,6 +70,14 @@ type LocalClient struct {
 	// connecting to the GUI client variants.
 	UseSocketOnly bool
 
+	// OmitAuth, if true, omits sending the local Tailscale daemon any
+	// authentication token that might be required by the platform.
+	//
+	// As of 2024-08-12, only macOS uses an authentication token. OmitAuth is
+	// meant for when Dial is set and the LocalAPI is being proxied to a
+	// different operating system, such as in integration tests.
+	OmitAuth bool
+
 	// tsClient does HTTP requests to the local Tailscale daemon.
 	// It's lazily initialized on first use.
 	tsClient     *http.Client
@@ -124,8 +133,10 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 			},
 		}
 	})
-	if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
-		req.SetBasicAuth("", token)
+	if !lc.OmitAuth {
+		if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
+			req.SetBasicAuth("", token)
+		}
 	}
 	return lc.tsClient.Do(req)
 }
@@ -343,6 +354,12 @@ func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
+// UserMetrics returns the user metrics in
+// the Prometheus text exposition format.
+func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
+	return lc.get200(ctx, "/localapi/v0/usermetrics")
+}
+
 // IncrementCounter increments the value of a Tailscale daemon's counter
 // metric by the given delta. If the metric has yet to exist, a new counter
 // metric is created and initialized to delta.
@@ -797,6 +814,35 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
+// GetDNSOSConfig returns the system DNS configuration for the current device.
+// That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
+func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
+	if err != nil {
+		return nil, err
+	}
+	var osCfg apitype.DNSOSConfig
+	if err := json.Unmarshal(body, &osCfg); err != nil {
+		return nil, fmt.Errorf("invalid dns.OSConfig: %w", err)
+	}
+	return &osCfg, nil
+}
+
+// QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
+// It returns the raw DNS response bytes and the resolvers that were used to answer the query
+// (often just one, but can be more if we raced multiple resolvers).
+func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
+	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
+	if err != nil {
+		return nil, nil, err
+	}
+	var res apitype.DNSQueryResponse
+	if err := json.Unmarshal(body, &res); err != nil {
+		return nil, nil, fmt.Errorf("invalid query response: %w", err)
+	}
+	return res.Bytes, res.Resolvers, nil
+}
+
 // StartLoginInteractive starts an interactive login.
 func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)

client/tailscale/required_version.go

@@ -1,10 +1,10 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !go1.21
+//go:build !go1.23
 
 package tailscale
 
 func init() {
-	you_need_Go_1_21_to_compile_Tailscale()
+	you_need_Go_1_23_to_compile_Tailscale()
 }

client/tailscale/tailscale.go

@@ -51,6 +51,9 @@ type Client struct {
 	// HTTPClient optionally specifies an alternate HTTP client to use.
 	// If nil, http.DefaultClient is used.
 	HTTPClient *http.Client
+
+	// UserAgent optionally specifies an alternate User-Agent header
+	UserAgent string
 }
 
 func (c *Client) httpClient() *http.Client {
@@ -97,8 +100,9 @@ func (c *Client) setAuth(r *http.Request) {
 // and can be changed manually by the user.
 func NewClient(tailnet string, auth AuthMethod) *Client {
 	return &Client{
-		tailnet: tailnet,
-		auth:    auth,
+		tailnet:   tailnet,
+		auth:      auth,
+		UserAgent: "tailscale-client-oss",
 	}
 }
 
@@ -110,17 +114,16 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 		return nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
 	}
 	c.setAuth(req)
+	if c.UserAgent != "" {
+		req.Header.Set("User-Agent", c.UserAgent)
+	}
 	return c.httpClient().Do(req)
 }
 
 // sendRequest add the authentication key to the request and sends it. It
 // receives the response and reads up to 10MB of it.
 func (c *Client) sendRequest(req *http.Request) ([]byte, *http.Response, error) {
-	if !I_Acknowledge_This_API_Is_Unstable {
-		return nil, nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
-	}
-	c.setAuth(req)
-	resp, err := c.httpClient().Do(req)
+	resp, err := c.Do(req)
 	if err != nil {
 		return nil, resp, err
 	}

client/web/web.go

@@ -17,7 +17,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -27,6 +26,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
+	"tailscale.com/envknob/featureknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -35,6 +35,7 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/views"
 	"tailscale.com/util/httpm"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -113,11 +114,6 @@ const (
 	ManageServerMode ServerMode = "manage"
 )
 
-var (
-	exitNodeRouteV4 = netip.MustParsePrefix("0.0.0.0/0")
-	exitNodeRouteV6 = netip.MustParsePrefix("::/0")
-)
-
 // ServerOpts contains options for constructing a new Server.
 type ServerOpts struct {
 	// Mode specifies the mode of web client being constructed.
@@ -283,6 +279,12 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if r.URL.Path == "/metrics" {
+		r.URL.Path = "/api/local/v0/usermetrics"
+		s.proxyRequestToLocalAPI(w, r)
+		return
+	}
+
 	if strings.HasPrefix(r.URL.Path, "/api/") {
 		switch {
 		case r.URL.Path == "/api/auth" && r.Method == httpm.GET:
@@ -921,10 +923,10 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 			return p == route
 		})
 	}
-	data.AdvertisingExitNodeApproved = routeApproved(exitNodeRouteV4) || routeApproved(exitNodeRouteV6)
+	data.AdvertisingExitNodeApproved = routeApproved(tsaddr.AllIPv4()) || routeApproved(tsaddr.AllIPv6())
 
 	for _, r := range prefs.AdvertiseRoutes {
-		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
+		if tsaddr.IsExitRoute(r) {
 			data.AdvertisingExitNode = true
 		} else {
 			data.AdvertisedRoutes = append(data.AdvertisedRoutes, subnetRoute{
@@ -959,37 +961,16 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 }
 
 func availableFeatures() map[string]bool {
-	env := hostinfo.GetEnvType()
 	features := map[string]bool{
 		"advertise-exit-node": true, // available on all platforms
 		"advertise-routes":    true, // available on all platforms
-		"use-exit-node":       canUseExitNode(env) == nil,
-		"ssh":                 envknob.CanRunTailscaleSSH() == nil,
+		"use-exit-node":       featureknob.CanUseExitNode() == nil,
+		"ssh":                 featureknob.CanRunTailscaleSSH() == nil,
 		"auto-update":         version.IsUnstableBuild() && clientupdate.CanAutoUpdate(),
 	}
-	if env == hostinfo.HomeAssistantAddOn {
-		// Setting SSH on Home Assistant causes trouble on startup
-		// (since the flag is not being passed to `tailscale up`).
-		// Although Tailscale SSH does work here,
-		// it's not terribly useful since it's running in a separate container.
-		features["ssh"] = false
-	}
 	return features
 }
 
-func canUseExitNode(env hostinfo.EnvType) error {
-	switch dist := distro.Get(); dist {
-	case distro.Synology, // see https://github.com/tailscale/tailscale/issues/1995
-		distro.QNAP,
-		distro.Unraid:
-		return fmt.Errorf("Tailscale exit nodes cannot be used on %s.", dist)
-	}
-	if env == hostinfo.HomeAssistantAddOn {
-		return errors.New("Tailscale exit nodes cannot be used on Home Assistant.")
-	}
-	return nil
-}
-
 // aclsAllowAccess returns whether tailnet ACLs (as expressed in the provided filter rules)
 // permit any devices to access the local web client.
 // This does not currently check whether a specific device can connect, just any device.
@@ -1065,7 +1046,7 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 	var currNonExitRoutes []string
 	var currAdvertisingExitNode bool
 	for _, r := range prefs.AdvertiseRoutes {
-		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
+		if tsaddr.IsExitRoute(r) {
 			currAdvertisingExitNode = true
 			continue
 		}
@@ -1086,12 +1067,7 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 		return err
 	}
 
-	hasExitNodeRoute := func(all []netip.Prefix) bool {
-		return slices.Contains(all, exitNodeRouteV4) ||
-			slices.Contains(all, exitNodeRouteV6)
-	}
-
-	if !data.UseExitNode.IsZero() && hasExitNodeRoute(routes) {
+	if !data.UseExitNode.IsZero() && tsaddr.ContainsExitRoutes(views.SliceOf(routes)) {
 		return errors.New("cannot use and advertise exit node at same time")
 	}
 

client/web/yarn.lock

@@ -5382,9 +5382,9 @@ wrappy@1:
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
 ws@^8.14.2:
-  version "8.14.2"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f"
-  integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
+  integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
 
 xml-name-validator@^5.0.0:
   version "5.0.0"

clientupdate/clientupdate.go

@@ -27,11 +27,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/uuid"
-	"tailscale.com/clientupdate/distsign"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
-	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
@@ -756,164 +753,6 @@ func (up *Updater) updateMacAppStore() error {
 	return nil
 }
 
-const (
-	// winMSIEnv is the environment variable that, if set, is the MSI file for
-	// the update command to install. It's passed like this so we can stop the
-	// tailscale.exe process from running before the msiexec process runs and
-	// tries to overwrite ourselves.
-	winMSIEnv = "TS_UPDATE_WIN_MSI"
-	// winExePathEnv is the environment variable that is set along with
-	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
-	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
-	// install is complete.
-	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
-)
-
-var (
-	verifyAuthenticode func(string) error // set non-nil only on Windows
-	markTempFileFunc   func(string) error // set non-nil only on Windows
-)
-
-func (up *Updater) updateWindows() error {
-	if msi := os.Getenv(winMSIEnv); msi != "" {
-		// stdout/stderr from this part of the install could be lost since the
-		// parent tailscaled is replaced. Create a temp log file to have some
-		// output to debug with in case update fails.
-		close, err := up.switchOutputToFile()
-		if err != nil {
-			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
-		} else {
-			defer close.Close()
-		}
-
-		up.Logf("installing %v ...", msi)
-		if err := up.installMSI(msi); err != nil {
-			up.Logf("MSI install failed: %v", err)
-			return err
-		}
-
-		up.Logf("success.")
-		return nil
-	}
-
-	if !winutil.IsCurrentProcessElevated() {
-		return errors.New(`update must be run as Administrator
-
-you can run the command prompt as Administrator one of these ways:
-* right-click cmd.exe, select 'Run as administrator'
-* press Windows+x, then press a
-* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
-	}
-	ver, err := requestedTailscaleVersion(up.Version, up.Track)
-	if err != nil {
-		return err
-	}
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	if !up.confirm(ver) {
-		return nil
-	}
-
-	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
-	msiDir := filepath.Join(tsDir, "MSICache")
-	if fi, err := os.Stat(tsDir); err != nil {
-		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
-	} else if !fi.IsDir() {
-		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
-	}
-	if err := os.MkdirAll(msiDir, 0700); err != nil {
-		return err
-	}
-	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
-	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
-	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
-	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
-		return err
-	}
-
-	up.Logf("verifying MSI authenticode...")
-	if err := verifyAuthenticode(msiTarget); err != nil {
-		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
-	}
-	up.Logf("authenticode verification succeeded")
-
-	up.Logf("making tailscale.exe copy to switch to...")
-	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
-	selfOrig, selfCopy, err := makeSelfCopy()
-	if err != nil {
-		return err
-	}
-	defer os.Remove(selfCopy)
-	up.Logf("running tailscale.exe copy for final install...")
-
-	cmd := exec.Command(selfCopy, "update")
-	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
-	cmd.Stdout = up.Stderr
-	cmd.Stderr = up.Stderr
-	cmd.Stdin = os.Stdin
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	// Once it's started, exit ourselves, so the binary is free
-	// to be replaced.
-	os.Exit(0)
-	panic("unreachable")
-}
-
-func (up *Updater) switchOutputToFile() (io.Closer, error) {
-	var logFilePath string
-	exePath, err := os.Executable()
-	if err != nil {
-		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
-	} else {
-		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
-	}
-
-	up.Logf("writing update output to %q", logFilePath)
-	logFile, err := os.Create(logFilePath)
-	if err != nil {
-		return nil, err
-	}
-
-	up.Logf = func(m string, args ...any) {
-		fmt.Fprintf(logFile, m+"\n", args...)
-	}
-	up.Stdout = logFile
-	up.Stderr = logFile
-	return logFile, nil
-}
-
-func (up *Updater) installMSI(msi string) error {
-	var err error
-	for tries := 0; tries < 2; tries++ {
-		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
-		cmd.Dir = filepath.Dir(msi)
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		if err == nil {
-			break
-		}
-		up.Logf("Install attempt failed: %v", err)
-		uninstallVersion := up.currentVersion
-		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
-			uninstallVersion = v
-		}
-		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
-		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
-		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		up.Logf("msiexec uninstall: %v", err)
-	}
-	return err
-}
-
 // cleanupOldDownloads removes all files matching glob (see filepath.Glob).
 // Only regular files are removed, so the glob must match specific files and
 // not directories.
@@ -938,53 +777,6 @@ func (up *Updater) cleanupOldDownloads(glob string) {
 	}
 }
 
-func msiUUIDForVersion(ver string) string {
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	track, err := versionToTrack(ver)
-	if err != nil {
-		track = UnstableTrack
-	}
-	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
-	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
-}
-
-func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
-	selfExe, err := os.Executable()
-	if err != nil {
-		return "", "", err
-	}
-	f, err := os.Open(selfExe)
-	if err != nil {
-		return "", "", err
-	}
-	defer f.Close()
-	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
-	if err != nil {
-		return "", "", err
-	}
-	if f := markTempFileFunc; f != nil {
-		if err := f(f2.Name()); err != nil {
-			return "", "", err
-		}
-	}
-	if _, err := io.Copy(f2, f); err != nil {
-		f2.Close()
-		return "", "", err
-	}
-	return selfExe, f2.Name(), f2.Close()
-}
-
-func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
-	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
-	if err != nil {
-		return err
-	}
-	return c.Download(context.Background(), pathSrc, fileDst)
-}
-
 func (up *Updater) updateFreeBSD() (err error) {
 	if up.Version != "" {
 		return errors.New("installing a specific version on FreeBSD is not supported")

clientupdate/clientupdate_downloads.go

@@ -0,0 +1,20 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build (linux && !android) || windows
+
+package clientupdate
+
+import (
+	"context"
+
+	"tailscale.com/clientupdate/distsign"
+)
+
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
+	if err != nil {
+		return err
+	}
+	return c.Download(context.Background(), pathSrc, fileDst)
+}

clientupdate/clientupdate_not_downloads.go

@@ -0,0 +1,10 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !((linux && !android) || windows)
+
+package clientupdate
+
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	panic("unreachable")
+}

clientupdate/clientupdate_notwindows.go

@@ -0,0 +1,10 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !windows
+
+package clientupdate
+
+func (up *Updater) updateWindows() error {
+	panic("unreachable")
+}

clientupdate/clientupdate_windows.go

@@ -7,13 +7,57 @@
 package clientupdate
 
 import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/google/uuid"
 	"golang.org/x/sys/windows"
+	"tailscale.com/util/winutil"
 	"tailscale.com/util/winutil/authenticode"
 )
 
-func init() {
-	markTempFileFunc = markTempFileWindows
-	verifyAuthenticode = verifyTailscale
+const (
+	// winMSIEnv is the environment variable that, if set, is the MSI file for
+	// the update command to install. It's passed like this so we can stop the
+	// tailscale.exe process from running before the msiexec process runs and
+	// tries to overwrite ourselves.
+	winMSIEnv = "TS_UPDATE_WIN_MSI"
+	// winExePathEnv is the environment variable that is set along with
+	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
+	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
+	// install is complete.
+	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
+)
+
+func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
+	selfExe, err := os.Executable()
+	if err != nil {
+		return "", "", err
+	}
+	f, err := os.Open(selfExe)
+	if err != nil {
+		return "", "", err
+	}
+	defer f.Close()
+	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
+	if err != nil {
+		return "", "", err
+	}
+	if err := markTempFileWindows(f2.Name()); err != nil {
+		return "", "", err
+	}
+	if _, err := io.Copy(f2, f); err != nil {
+		f2.Close()
+		return "", "", err
+	}
+	return selfExe, f2.Name(), f2.Close()
 }
 
 func markTempFileWindows(name string) error {
@@ -23,6 +67,159 @@ func markTempFileWindows(name string) error {
 
 const certSubjectTailscale = "Tailscale Inc."
 
-func verifyTailscale(path string) error {
+func verifyAuthenticode(path string) error {
 	return authenticode.Verify(path, certSubjectTailscale)
 }
+
+func (up *Updater) updateWindows() error {
+	if msi := os.Getenv(winMSIEnv); msi != "" {
+		// stdout/stderr from this part of the install could be lost since the
+		// parent tailscaled is replaced. Create a temp log file to have some
+		// output to debug with in case update fails.
+		close, err := up.switchOutputToFile()
+		if err != nil {
+			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
+		} else {
+			defer close.Close()
+		}
+
+		up.Logf("installing %v ...", msi)
+		if err := up.installMSI(msi); err != nil {
+			up.Logf("MSI install failed: %v", err)
+			return err
+		}
+
+		up.Logf("success.")
+		return nil
+	}
+
+	if !winutil.IsCurrentProcessElevated() {
+		return errors.New(`update must be run as Administrator
+
+you can run the command prompt as Administrator one of these ways:
+* right-click cmd.exe, select 'Run as administrator'
+* press Windows+x, then press a
+* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
+	}
+	ver, err := requestedTailscaleVersion(up.Version, up.Track)
+	if err != nil {
+		return err
+	}
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	if !up.confirm(ver) {
+		return nil
+	}
+
+	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
+	msiDir := filepath.Join(tsDir, "MSICache")
+	if fi, err := os.Stat(tsDir); err != nil {
+		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
+	}
+	if err := os.MkdirAll(msiDir, 0700); err != nil {
+		return err
+	}
+	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
+	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
+	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
+		return err
+	}
+
+	up.Logf("verifying MSI authenticode...")
+	if err := verifyAuthenticode(msiTarget); err != nil {
+		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
+	}
+	up.Logf("authenticode verification succeeded")
+
+	up.Logf("making tailscale.exe copy to switch to...")
+	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
+	selfOrig, selfCopy, err := makeSelfCopy()
+	if err != nil {
+		return err
+	}
+	defer os.Remove(selfCopy)
+	up.Logf("running tailscale.exe copy for final install...")
+
+	cmd := exec.Command(selfCopy, "update")
+	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
+	cmd.Stdout = up.Stderr
+	cmd.Stderr = up.Stderr
+	cmd.Stdin = os.Stdin
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	// Once it's started, exit ourselves, so the binary is free
+	// to be replaced.
+	os.Exit(0)
+	panic("unreachable")
+}
+
+func (up *Updater) installMSI(msi string) error {
+	var err error
+	for tries := 0; tries < 2; tries++ {
+		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
+		cmd.Dir = filepath.Dir(msi)
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		if err == nil {
+			break
+		}
+		up.Logf("Install attempt failed: %v", err)
+		uninstallVersion := up.currentVersion
+		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
+			uninstallVersion = v
+		}
+		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
+		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
+		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		up.Logf("msiexec uninstall: %v", err)
+	}
+	return err
+}
+
+func msiUUIDForVersion(ver string) string {
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	track, err := versionToTrack(ver)
+	if err != nil {
+		track = UnstableTrack
+	}
+	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
+	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
+}
+
+func (up *Updater) switchOutputToFile() (io.Closer, error) {
+	var logFilePath string
+	exePath, err := os.Executable()
+	if err != nil {
+		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
+	} else {
+		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
+	}
+
+	up.Logf("writing update output to %q", logFilePath)
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	up.Logf = func(m string, args ...any) {
+		fmt.Fprintf(logFile, m+"\n", args...)
+	}
+	up.Stdout = logFile
+	up.Stderr = logFile
+	return logFile, nil
+}

cmd/cloner/cloner.go

@@ -47,7 +47,7 @@ func main() {
 	it := codegen.NewImportTracker(pkg.Types)
 	buf := new(bytes.Buffer)
 	for _, typeName := range typeNames {
-		typ, ok := namedTypes[typeName]
+		typ, ok := namedTypes[typeName].(*types.Named)
 		if !ok {
 			log.Fatalf("could not find type %s", typeName)
 		}
@@ -115,7 +115,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		if !codegen.ContainsPointers(ft) || codegen.HasNoClone(t.Tag(i)) {
 			continue
 		}
-		if named, _ := ft.(*types.Named); named != nil {
+		if named, _ := codegen.NamedTypeOf(ft); named != nil {
 			if codegen.IsViewType(ft) {
 				writef("dst.%s = src.%s", fname, fname)
 				continue
@@ -161,7 +161,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		case *types.Pointer:
 			base := ft.Elem()
 			hasPtrs := codegen.ContainsPointers(base)
-			if named, _ := base.(*types.Named); named != nil && hasPtrs {
+			if named, _ := codegen.NamedTypeOf(base); named != nil && hasPtrs {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}

cmd/containerboot/forwarding.go

@@ -0,0 +1,262 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"tailscale.com/util/linuxfw"
+)
+
+// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
+	var (
+		v4Forwarding, v6Forwarding bool
+	)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	// Currently we only proxy traffic to the IPv4 address of the tailnet
+	// target.
+	if tailnetTargetFQDN != "" {
+		v4Forwarding = true
+	}
+	if routes != nil && *routes != "" {
+		for _, route := range strings.Split(*routes, ",") {
+			cidr, err := netip.ParsePrefix(route)
+			if err != nil {
+				return fmt.Errorf("invalid subnet route: %v", err)
+			}
+			if cidr.Addr().Is4() {
+				v4Forwarding = true
+			} else {
+				v6Forwarding = true
+			}
+		}
+	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
+
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
+	var paths []string
+	if v4Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
+	}
+	if v6Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
+	}
+
+	// In some common configurations (e.g. default docker,
+	// kubernetes), the container environment denies write access to
+	// most sysctls, including IP forwarding controls. Check the
+	// sysctl values before trying to change them, so that we
+	// gracefully do nothing if the container's already been set up
+	// properly by e.g. a k8s initContainer.
+	for _, path := range paths {
+		bs, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %q: %w", path, err)
+		}
+		if v := strings.TrimSpace(string(bs)); v != "1" {
+			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
+				return fmt.Errorf("enabling %q: %w", path, err)
+			}
+		}
+	}
+	return nil
+}
+
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.EnsureSNATForDst(local, dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	return nil
+}
+
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(_ context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	proxyHasIPv4Address := false
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.AddDNATRule(local, dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRuleForDNSTarget(_ context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}

cmd/containerboot/healthz.go

@@ -0,0 +1,51 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"log"
+	"net"
+	"net/http"
+	"sync"
+)
+
+// healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type healthz struct {
+	sync.Mutex
+	hasAddrs bool
+}
+
+func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+	if h.hasAddrs {
+		w.Write([]byte("ok"))
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
+	}
+}
+
+// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
+// provided address. A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func runHealthz(addr string, h *healthz) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", h)
+	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
+	hs := &http.Server{Handler: mux}
+
+	go func() {
+		if err := hs.Serve(lis); err != nil {
+			log.Fatalf("failed running health endpoint: %v", err)
+		}
+	}()
+}

cmd/containerboot/kube.go

@@ -8,21 +8,21 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 	"tailscale.com/tailcfg"
 )
 
 // storeDeviceID writes deviceID to 'device_id' data field of the named
 // Kubernetes Secret.
 func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
 			"device_id": []byte(deviceID),
 		},
@@ -42,7 +42,7 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 		return err
 	}
 
-	s := &kube.Secret{
+	s := &kubeapi.Secret{
 		Data: map[string][]byte{
 			"device_fqdn": []byte(fqdn),
 			"device_ips":  deviceIPs,
@@ -55,14 +55,14 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 // secret. No-op if there is no authkey in the secret.
 func deleteAuthKey(ctx context.Context, secretName string) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
-	m := []kube.JSONPatch{
+	m := []kubeclient.JSONPatch{
 		{
 			Op:   "remove",
 			Path: "/data/authkey",
 		},
 	}
 	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
-		if s, ok := err.(*kube.Status); ok && s.Code == http.StatusUnprocessableEntity {
+		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
 			return nil
@@ -72,66 +72,16 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kube.Client
-
-// setupKube is responsible for doing any necessary configuration and checks to
-// ensure that tailscale state storage and authentication mechanism will work on
-// Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
-	if cfg.KubeSecret == "" {
-		return nil
-	}
-	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
-	if err != nil {
-		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-	}
-	cfg.KubernetesCanPatch = canPatch
-
-	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
-	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
-		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
-			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
-			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
-			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
-	} else if err != nil && !kube.IsNotFoundErr(err) {
-		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
-	}
-
-	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-		if s == nil {
-			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
-			return nil
-		}
-		keyBytes, _ := s.Data["authkey"]
-		key := string(keyBytes)
-
-		if key != "" {
-			// This behavior of pulling authkeys from kube secrets was added
-			// at the same time as the patch permission, so we can enforce
-			// that we must be able to patch out the authkey after
-			// authenticating if you want to use this feature. This avoids
-			// us having to deal with the case where we might leave behind
-			// an unnecessary reusable authkey in a secret, like a rake in
-			// the grass.
-			if !cfg.KubernetesCanPatch {
-				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-			}
-			cfg.AuthKey = key
-		} else {
-			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-		}
-	}
-	return nil
-}
+var kc kubeclient.Client
 
 func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.
-		kube.SetRootPathForTesting(root)
+		kubeclient.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kube.New()
+	kc, err = kubeclient.New()
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/kube_test.go

@@ -11,7 +11,8 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/kube"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/kube/kubeclient"
 )
 
 func TestSetupKube(t *testing.T) {
@@ -20,7 +21,7 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kube.Client
+		kc      kubeclient.Client
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
@@ -28,11 +29,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, nil
 				},
 			},
@@ -47,12 +48,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -66,12 +67,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -86,12 +87,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 403}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 403}
 				},
 			},
 			wantCfg: &settings{
@@ -110,7 +111,7 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
@@ -126,12 +127,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return nil, &kube.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return nil, &kubeapi.Status{Code: 404}
 				},
 			},
 		},
@@ -144,12 +145,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{}, nil
 				},
 			},
 		},
@@ -158,12 +159,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{
@@ -176,12 +177,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kube.FakeClient{
+			kc: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
-					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{

cmd/containerboot/main.go

@@ -52,6 +52,12 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
+//   - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
+//     served at /healthz at the provided address, which should be in form [<address>]:<port>.
+//     If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
+//     The health endpoint will return 200 OK if this node has at least one tailnet IP address,
+//     otherwise returns 503.
+//     NB: the health criteria might change in the future.
 //   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
 //     directory that containers tailscaled config in file. The config file needs to be
 //     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
@@ -86,9 +92,7 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -97,24 +101,19 @@ import (
 	"net"
 	"net/netip"
 	"os"
-	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
-	"reflect"
 	"slices"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"golang.org/x/sys/unix"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
-	"tailscale.com/ipn/conffile"
 	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -133,34 +132,9 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) {
 func main() {
 	log.SetPrefix("boot: ")
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
-	cfg := &settings{
-		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
-		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
-		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
-		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
-		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
-		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
-		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
-		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
-		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
-		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
-		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
-		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
-		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
-		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
-		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
-		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
-		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
-		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
-		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
-		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
-		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
-		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
-		PodIP:                                 defaultEnv("POD_IP", ""),
-		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
-	}
 
-	if err := cfg.validate(); err != nil {
+	cfg, err := configFromEnv()
+	if err != nil {
 		log.Fatalf("invalid configuration: %v", err)
 	}
 
@@ -275,10 +249,8 @@ authLoop:
 			switch *n.State {
 			case ipn.NeedsLogin:
 				if isOneStepConfig(cfg) {
-					// This could happen if this is the
-					// first time tailscaled was run for
-					// this device and the auth key was not
-					// passed via the configfile.
+					// This could happen if this is the first time tailscaled was run for this
+					// device and the auth key was not passed via the configfile.
 					log.Fatalf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
 				}
 				if err := authTailscale(); err != nil {
@@ -349,6 +321,9 @@ authLoop:
 
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
+
+		h             = &healthz{} // http server for the healthz endpoint
+		healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
 	)
 	if cfg.ServeConfigPath != "" {
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
@@ -373,6 +348,9 @@ authLoop:
 			}
 		})
 	)
+	// egressSvcsErrorChan will get an error sent to it if this containerboot instance is configured to expose 1+
+	// egress services in HA mode and errored.
+	var egressSvcsErrorChan = make(chan error)
 	defer t.Stop()
 	// resetTimer resets timer for when to next attempt to resolve the DNS
 	// name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The
@@ -398,6 +376,7 @@ authLoop:
 		failedResolveAttempts++
 	}
 
+	var egressSvcsNotify chan ipn.Notify
 	notifyChan := make(chan ipn.Notify)
 	errChan := make(chan error)
 	go func() {
@@ -475,7 +454,11 @@ runLoop:
 					egressAddrs = node.Addresses().AsSlice()
 					newCurentEgressIPs = deephash.Hash(&egressAddrs)
 					egressIPsHaveChanged = newCurentEgressIPs != currentEgressIPs
-					if egressIPsHaveChanged && len(egressAddrs) != 0 {
+					// The firewall rules get (re-)installed:
+					// - on startup
+					// - when the tailnet IPs of the tailnet target have changed
+					// - when the tailnet IPs of this node have changed
+					if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 {
 						var rulesInstalled bool
 						for _, egressAddr := range egressAddrs {
 							ea := egressAddr.Addr()
@@ -565,31 +548,57 @@ runLoop:
 						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
+
+				if cfg.HealthCheckAddrPort != "" {
+					h.Lock()
+					h.hasAddrs = len(addrs) != 0
+					h.Unlock()
+					healthzRunner()
+				}
+				if egressSvcsNotify != nil {
+					egressSvcsNotify <- n
+				}
 			}
 			if !startupTasksDone {
-				// For containerboot instances that act as TCP
-				// proxies (proxying traffic to an endpoint
-				// passed via one of the env vars that
-				// containerbot reads) and store state in a
-				// Kubernetes Secret, we consider startup tasks
-				// done at the point when device info has been
-				// successfully stored to state Secret.
-				// For all other containerboot instances, if we
-				// just get to this point the startup tasks can
-				// be considered done.
+				// For containerboot instances that act as TCP proxies (proxying traffic to an endpoint
+				// passed via one of the env vars that containerboot reads) and store state in a
+				// Kubernetes Secret, we consider startup tasks done at the point when device info has
+				// been successfully stored to state Secret. For all other containerboot instances, if
+				// we just get to this point the startup tasks can be considered done.
 				if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) {
 					// This log message is used in tests to detect when all
 					// post-auth configuration is done.
 					log.Println("Startup complete, waiting for shutdown signal")
 					startupTasksDone = true
 
-					// Wait on tailscaled process. It won't
-					// be cleaned up by default when the
-					// container exits as it is not PID1.
-					// TODO (irbekrm): perhaps we can
-					// replace the reaper by a running
-					// cmd.Wait in a goroutine immediately
-					// after starting tailscaled?
+					// Configure egress proxy. Egress proxy will set up firewall rules to proxy
+					// traffic to tailnet targets configured in the provided configuration file. It
+					// will then continuously monitor the config file and netmap updates and
+					// reconfigure the firewall rules as needed. If any of its operations fail, it
+					// will crash this node.
+					if cfg.EgressSvcsCfgPath != "" {
+						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressSvcsCfgPath)
+						egressSvcsNotify = make(chan ipn.Notify)
+						ep := egressProxy{
+							cfgPath:      cfg.EgressSvcsCfgPath,
+							nfr:          nfr,
+							kc:           kc,
+							stateSecret:  cfg.KubeSecret,
+							netmapChan:   egressSvcsNotify,
+							podIPv4:      cfg.PodIPv4,
+							tailnetAddrs: addrs,
+						}
+						go func() {
+							if err := ep.run(ctx, n); err != nil {
+								egressSvcsErrorChan <- err
+							}
+						}()
+					}
+
+					// Wait on tailscaled process. It won't be cleaned up by default when the
+					// container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the
+					// reaper by a running cmd.Wait in a goroutine immediately after starting
+					// tailscaled?
 					reaper := func() {
 						defer wg.Done()
 						for {
@@ -627,226 +636,13 @@ runLoop:
 			}
 			backendAddrs = newBackendAddrs
 			resetTimer(false)
+		case e := <-egressSvcsErrorChan:
+			log.Fatalf("egress proxy failed: %v", e)
 		}
 	}
 	wg.Wait()
 }
 
-// watchServeConfigChanges watches path for changes, and when it sees one, reads
-// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
-// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
-// is written to when the certDomain changes, causing the serve config to be
-// re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
-	if certDomainAtomic == nil {
-		panic("cd must not be nil")
-	}
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
-			log.Fatalf("failed to add fsnotify watch: %v", err)
-		}
-		eventChan = w.Events
-	}
-
-	var certDomain string
-	var prevServeConfig *ipn.ServeConfig
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-cdChanged:
-			certDomain = *certDomainAtomic.Load()
-		case <-tickChan:
-		case <-eventChan:
-			// We can't do any reasonable filtering on the event because of how
-			// k8s handles these mounts. So just re-read the file and apply it
-			// if it's changed.
-		}
-		if certDomain == "" {
-			continue
-		}
-		sc, err := readServeConfig(path, certDomain)
-		if err != nil {
-			log.Fatalf("failed to read serve config: %v", err)
-		}
-		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
-			continue
-		}
-		log.Printf("Applying serve config")
-		if err := lc.SetServeConfig(ctx, sc); err != nil {
-			log.Fatalf("failed to set serve config: %v", err)
-		}
-		prevServeConfig = sc
-	}
-}
-
-// readServeConfig reads the ipn.ServeConfig from path, replacing
-// ${TS_CERT_DOMAIN} with certDomain.
-func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
-	if path == "" {
-		return nil, nil
-	}
-	j, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
-	var sc ipn.ServeConfig
-	if err := json.Unmarshal(j, &sc); err != nil {
-		return nil, err
-	}
-	return &sc, nil
-}
-
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
-	args := tailscaledArgs(cfg)
-	// tailscaled runs without context, since it needs to persist
-	// beyond the startup timeout in ctx.
-	cmd := exec.Command("tailscaled", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-	log.Printf("Starting tailscaled")
-	if err := cmd.Start(); err != nil {
-		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
-	}
-
-	// Wait for the socket file to appear, otherwise API ops will racily fail.
-	log.Printf("Waiting for tailscaled socket")
-	for {
-		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
-		}
-		_, err := os.Stat(cfg.Socket)
-		if errors.Is(err, fs.ErrNotExist) {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
-		}
-		break
-	}
-
-	tsClient := &tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
-	return tsClient, cmd.Process, nil
-}
-
-// tailscaledArgs uses cfg to construct the argv for tailscaled.
-func tailscaledArgs(cfg *settings) []string {
-	args := []string{"--socket=" + cfg.Socket}
-	switch {
-	case cfg.InKubernetes && cfg.KubeSecret != "":
-		args = append(args, "--state=kube:"+cfg.KubeSecret)
-		if cfg.StateDir == "" {
-			cfg.StateDir = "/tmp"
-		}
-		fallthrough
-	case cfg.StateDir != "":
-		args = append(args, "--statedir="+cfg.StateDir)
-	default:
-		args = append(args, "--state=mem:", "--statedir=/tmp")
-	}
-
-	if cfg.UserspaceMode {
-		args = append(args, "--tun=userspace-networking")
-	} else if err := ensureTunFile(cfg.Root); err != nil {
-		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
-	}
-
-	if cfg.SOCKSProxyAddr != "" {
-		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
-	}
-	if cfg.HTTPProxyAddr != "" {
-		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
-	}
-	if cfg.TailscaledConfigFilePath != "" {
-		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
-	}
-	if cfg.DaemonExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
-	}
-	return args
-}
-
-// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
-// if TS_AUTH_ONCE is set, only the first time containerboot starts.
-func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	if cfg.AuthKey != "" {
-		args = append(args, "--authkey="+cfg.AuthKey)
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
-	}
-	return nil
-}
-
-// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
-// options that are passed in via environment variables. This is run after the
-// node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}
-
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
 func ensureTunFile(root string) error {
@@ -866,344 +662,6 @@ func ensureTunFile(root string) error {
 	return nil
 }
 
-// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
-	var (
-		v4Forwarding, v6Forwarding bool
-	)
-	if clusterProxyTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid cluster destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	if tailnetTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid tailnet destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	// Currently we only proxy traffic to the IPv4 address of the tailnet
-	// target.
-	if tailnetTargetFQDN != "" {
-		v4Forwarding = true
-	}
-	if routes != nil && *routes != "" {
-		for _, route := range strings.Split(*routes, ",") {
-			cidr, err := netip.ParsePrefix(route)
-			if err != nil {
-				return fmt.Errorf("invalid subnet route: %v", err)
-			}
-			if cidr.Addr().Is4() {
-				v4Forwarding = true
-			} else {
-				v6Forwarding = true
-			}
-		}
-	}
-	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
-}
-
-func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
-	var paths []string
-	if v4Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
-	}
-	if v6Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
-	}
-
-	// In some common configurations (e.g. default docker,
-	// kubernetes), the container environment denies write access to
-	// most sysctls, including IP forwarding controls. Check the
-	// sysctl values before trying to change them, so that we
-	// gracefully do nothing if the container's already been set up
-	// properly by e.g. a k8s initContainer.
-	for _, path := range paths {
-		bs, err := os.ReadFile(path)
-		if err != nil {
-			return fmt.Errorf("reading %q: %w", path, err)
-		}
-		if v := strings.TrimSpace(string(bs)); v != "1" {
-			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
-				return fmt.Errorf("enabling %q: %w", path, err)
-			}
-		}
-	}
-	return nil
-}
-
-func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	return nil
-}
-
-// installTSForwardingRuleForDestination accepts a destination address and a
-// list of node's tailnet addresses, sets up rules to forward traffic for
-// destination to the tailnet IP matching the destination IP family.
-// Destination can be Pod IP of this node.
-func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstFilter)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
-	}
-	if err := nfr.AddDNATRule(dst, local); err != nil {
-		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	proxyHasIPv4Address := false
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() {
-			proxyHasIPv4Address = true
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if proxyHasIPv4Address && dst.Is6() {
-		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.AddDNATRule(local, dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRuleForDNSTarget(ctx context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	var (
-		tsv4       netip.Addr
-		tsv6       netip.Addr
-		v4Backends []netip.Addr
-		v6Backends []netip.Addr
-	)
-	for _, pfx := range tsIPs {
-		if pfx.IsSingleIP() && pfx.Addr().Is4() {
-			tsv4 = pfx.Addr()
-			continue
-		}
-		if pfx.IsSingleIP() && pfx.Addr().Is6() {
-			tsv6 = pfx.Addr()
-			continue
-		}
-	}
-	// TODO: log if more than one backend address is found and firewall is
-	// in nftables mode that only the first IP will be used.
-	for _, ip := range backendAddrs {
-		if ip.To4() != nil {
-			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
-		}
-		if ip.To16() != nil {
-			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
-		}
-	}
-
-	// Enable IP forwarding here as opposed to at the start of containerboot
-	// as the IPv4/IPv6 requirements might have changed.
-	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
-	// enabled by an init container, so in practice enabling forwarding here
-	// is only needed if this proxy has been configured by manually setting
-	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
-	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
-		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
-	}
-
-	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
-		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
-			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
-		}
-		// The backend might advertize MSS higher than that of the
-		// tailscale interfaces. Clamp MSS of packets going out via
-		// tailscale0 interface to its MTU to prevent broken connections
-		// in environments where path MTU discovery is not working.
-		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
-		}
-		return nil
-	}
-
-	if len(v4Backends) != 0 {
-		if !tsv4.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
-		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
-			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
-		}
-	}
-	if len(v6Backends) != 0 && !tsv6.IsValid() {
-		if !tsv6.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
-		} else if !nfr.HasIPV6NAT() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
-		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
-			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
-		}
-	}
-	return nil
-}
-
-// settings is all the configuration for containerboot.
-type settings struct {
-	AuthKey  string
-	Hostname string
-	Routes   *string
-	// ProxyTargetIP is the destination IP to which all incoming
-	// Tailscale traffic should be proxied. If empty, no proxying
-	// is done. This is typically a locally reachable IP.
-	ProxyTargetIP string
-	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
-	// incoming Tailscale traffic should be proxied.
-	ProxyTargetDNSName string
-	// TailnetTargetIP is the destination IP to which all incoming
-	// non-Tailscale traffic should be proxied. This is typically a
-	// Tailscale IP.
-	TailnetTargetIP string
-	// TailnetTargetFQDN is an MagicDNS name to which all incoming
-	// non-Tailscale traffic should be proxied. This must be a full Tailnet
-	// node FQDN.
-	TailnetTargetFQDN             string
-	ServeConfigPath               string
-	DaemonExtraArgs               string
-	ExtraArgs                     string
-	InKubernetes                  bool
-	UserspaceMode                 bool
-	StateDir                      string
-	AcceptDNS                     *bool
-	KubeSecret                    string
-	SOCKSProxyAddr                string
-	HTTPProxyAddr                 string
-	Socket                        string
-	AuthOnce                      bool
-	Root                          string
-	KubernetesCanPatch            bool
-	TailscaledConfigFilePath      string
-	EnableForwardingOptimizations bool
-	// If set to true and, if this containerboot instance is a Kubernetes
-	// ingress proxy, set up rules to forward incoming cluster traffic to be
-	// forwarded to the ingress target in cluster.
-	AllowProxyingClusterTrafficViaIngress bool
-	// PodIP is the IP of the Pod if running in Kubernetes. This is used
-	// when setting up rules to proxy cluster traffic to cluster ingress
-	// target.
-	PodIP string
-}
-
-func (s *settings) validate() error {
-	if s.TailscaledConfigFilePath != "" {
-		dir, file := path.Split(s.TailscaledConfigFilePath)
-		if _, err := os.Stat(dir); err != nil {
-			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
-		}
-		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
-		}
-		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
-		}
-	}
-	if s.ProxyTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
-	}
-	if s.TailnetTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
-		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
-	}
-	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
-	}
-	if s.EnableForwardingOptimizations && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
-	}
-	return nil
-}
-
 func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	// TODO (irbekrm): look at using recursive.Resolver instead to resolve
 	// the DNS names as well as retrieve TTLs. It looks though that this
@@ -1226,57 +684,6 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	return append(ip4s, ip6s...), nil
 }
 
-// defaultEnv returns the value of the given envvar name, or defVal if
-// unset.
-func defaultEnv(name, defVal string) string {
-	if v, ok := os.LookupEnv(name); ok {
-		return v
-	}
-	return defVal
-}
-
-// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being set to empty string vs unset.
-func defaultEnvStringPointer(name string) *string {
-	if v, ok := os.LookupEnv(name); ok {
-		return &v
-	}
-	return nil
-}
-
-// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being explicitly set to false vs unset.
-func defaultEnvBoolPointer(name string) *bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return nil
-	}
-	return &ret
-}
-
-func defaultEnvs(names []string, defVal string) string {
-	for _, name := range names {
-		if v, ok := os.LookupEnv(name); ok {
-			return v
-		}
-	}
-	return defVal
-}
-
-// defaultBool returns the boolean value of the given envvar name, or
-// defVal if unset or not a bool.
-func defaultBool(name string, defVal bool) bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return defVal
-	}
-	return ret
-}
-
 // contextWithExitSignalWatch watches for SIGTERM/SIGINT signals. It returns a
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
@@ -1299,43 +706,6 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 	return ctx, f
 }
 
-// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
-// in two steps and login should only happen once.
-// Step 1: run 'tailscaled'
-// Step 2):
-// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
-// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
-func isTwoStepConfigAuthOnce(cfg *settings) bool {
-	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
-// in two steps and we should log in every time it starts.
-// Step 1: run 'tailscaled'
-// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
-func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
-	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isOneStepConfig returns true if the Tailscale node should always be ran and
-// configured in a single step by running 'tailscaled <config opts>'
-func isOneStepConfig(cfg *settings) bool {
-	return cfg.TailscaledConfigFilePath != ""
-}
-
-// isL3Proxy returns true if the Tailscale node needs to be configured to act
-// as an L3 proxy, proxying to an endpoint provided via one of the config env
-// vars.
-func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
-}
-
-// hasKubeStateStore returns true if the state must be stored in a Kubernetes
-// Secret.
-func hasKubeStateStore(cfg *settings) bool {
-	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
-}
-
 // tailscaledConfigFilePath returns the path to the tailscaled config file that
 // should be used for the current capability version. It is determined by the
 // TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a
@@ -1372,5 +742,5 @@ func tailscaledConfigFilePath() string {
 		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
 	}
 	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer))
+	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
 }

cmd/containerboot/serve.go

@@ -0,0 +1,96 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+)
+
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+	if certDomainAtomic == nil {
+		panic("cd must not be nil")
+	}
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(path)); err != nil {
+			log.Fatalf("failed to add fsnotify watch: %v", err)
+		}
+		eventChan = w.Events
+	}
+
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-eventChan:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		if certDomain == "" {
+			continue
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("failed to read serve config: %v", err)
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		log.Printf("Applying serve config")
+		if err := lc.SetServeConfig(ctx, sc); err != nil {
+			log.Fatalf("failed to set serve config: %v", err)
+		}
+		prevServeConfig = sc
+	}
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}

cmd/containerboot/services.go

@@ -0,0 +1,571 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"tailscale.com/ipn"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubeclient"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/linuxfw"
+	"tailscale.com/util/mak"
+)
+
+const tailscaleTunInterface = "tailscale0"
+
+// This file contains functionality to run containerboot as a proxy that can
+// route cluster traffic to one or more tailnet targets, based on portmapping
+// rules read from a configfile. Currently (9/2024) this is only used for the
+// Kubernetes operator egress proxies.
+
+// egressProxy knows how to configure firewall rules to route cluster traffic to
+// one or more tailnet services.
+type egressProxy struct {
+	cfgPath string // path to egress service config file
+
+	nfr linuxfw.NetfilterRunner // never nil
+
+	kc          kubeclient.Client // never nil
+	stateSecret string            // name of the kube state Secret
+
+	netmapChan chan ipn.Notify // chan to receive netmap updates on
+
+	podIPv4 string // never empty string, currently only IPv4 is supported
+
+	// tailnetFQDNs is the egress service FQDN to tailnet IP mappings that
+	// were last used to configure firewall rules for this proxy.
+	// TODO(irbekrm): target addresses are also stored in the state Secret.
+	// Evaluate whether we should retrieve them from there and not store in
+	// memory at all.
+	targetFQDNs map[string][]netip.Prefix
+
+	// used to configure firewall rules.
+	tailnetAddrs []netip.Prefix
+}
+
+// run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when:
+// - the mounted egress config has changed
+// - the proxy's tailnet IP addresses have changed
+// - tailnet IPs have changed for any backend targets specified by tailnet FQDN
+func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	// TODO (irbekrm): take a look if this can be pulled into a single func
+	// shared with serve config loader.
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil {
+			return fmt.Errorf("failed to add fsnotify watch: %w", err)
+		}
+		eventChan = w.Events
+	}
+
+	if err := ep.sync(ctx, n); err != nil {
+		return err
+	}
+	for {
+		var err error
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-tickChan:
+			err = ep.sync(ctx, n)
+		case <-eventChan:
+			log.Printf("config file change detected, ensuring firewall config is up to date...")
+			err = ep.sync(ctx, n)
+		case n = <-ep.netmapChan:
+			shouldResync := ep.shouldResync(n)
+			if shouldResync {
+				log.Printf("netmap change detected, ensuring firewall config is up to date...")
+				err = ep.sync(ctx, n)
+			}
+		}
+		if err != nil {
+			return fmt.Errorf("error syncing egress service config: %w", err)
+		}
+	}
+}
+
+// sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if
+// any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current
+// firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such
+// as failed firewall update
+func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error {
+	cfgs, err := ep.getConfigs()
+	if err != nil {
+		return fmt.Errorf("error retrieving egress service configs: %w", err)
+	}
+	status, err := ep.getStatus(ctx)
+	if err != nil {
+		return fmt.Errorf("error retrieving current egress proxy status: %w", err)
+	}
+	newStatus, err := ep.syncEgressConfigs(cfgs, status, n)
+	if err != nil {
+		return fmt.Errorf("error syncing egress service configs: %w", err)
+	}
+	if !servicesStatusIsEqual(newStatus, status) {
+		if err := ep.setStatus(ctx, newStatus, n); err != nil {
+			return fmt.Errorf("error setting egress proxy status: %w", err)
+		}
+	}
+	return nil
+}
+
+// addrsHaveChanged returns true if the provided netmap update contains tailnet address change for this proxy node.
+// Netmap must not be nil.
+func (ep *egressProxy) addrsHaveChanged(n ipn.Notify) bool {
+	return !reflect.DeepEqual(ep.tailnetAddrs, n.NetMap.SelfNode.Addresses())
+}
+
+// syncEgressConfigs adds and deletes firewall rules to match the desired
+// configuration. It uses the provided status to determine what is currently
+// applied and updates the status after a successful sync.
+func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *egressservices.Status, n ipn.Notify) (*egressservices.Status, error) {
+	if !(wantsServicesConfigured(cfgs) || hasServicesConfigured(status)) {
+		return nil, nil
+	}
+
+	// Delete unnecessary services.
+	if err := ep.deleteUnnecessaryServices(cfgs, status); err != nil {
+		return nil, fmt.Errorf("error deleting services: %w", err)
+
+	}
+	newStatus := &egressservices.Status{}
+	if !wantsServicesConfigured(cfgs) {
+		return newStatus, nil
+	}
+
+	// Add new services, update rules for any that have changed.
+	rulesPerSvcToAdd := make(map[string][]rule, 0)
+	rulesPerSvcToDelete := make(map[string][]rule, 0)
+	for svcName, cfg := range *cfgs {
+		tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, n)
+		if err != nil {
+			return nil, fmt.Errorf("error determining tailnet target IPs: %w", err)
+		}
+		rulesToAdd, rulesToDelete, err := updatesForCfg(svcName, cfg, status, tailnetTargetIPs)
+		if err != nil {
+			return nil, fmt.Errorf("error validating service changes: %v", err)
+		}
+		log.Printf("syncegressservices: looking at svc %s rulesToAdd %d rulesToDelete %d", svcName, len(rulesToAdd), len(rulesToDelete))
+		if len(rulesToAdd) != 0 {
+			mak.Set(&rulesPerSvcToAdd, svcName, rulesToAdd)
+		}
+		if len(rulesToDelete) != 0 {
+			mak.Set(&rulesPerSvcToDelete, svcName, rulesToDelete)
+		}
+		if len(rulesToAdd) != 0 || ep.addrsHaveChanged(n) {
+			// For each tailnet target, set up SNAT from the local tailnet device address of the matching
+			// family.
+			for _, t := range tailnetTargetIPs {
+				var local netip.Addr
+				for _, pfx := range n.NetMap.SelfNode.Addresses().All() {
+					if !pfx.IsSingleIP() {
+						continue
+					}
+					if pfx.Addr().Is4() != t.Is4() {
+						continue
+					}
+					local = pfx.Addr()
+					break
+				}
+				if !local.IsValid() {
+					return nil, fmt.Errorf("no valid local IP: %v", local)
+				}
+				if err := ep.nfr.EnsureSNATForDst(local, t); err != nil {
+					return nil, fmt.Errorf("error setting up SNAT rule: %w", err)
+				}
+			}
+		}
+		// Update the status. Status will be written back to the state Secret by the caller.
+		mak.Set(&newStatus.Services, svcName, &egressservices.ServiceStatus{TailnetTargetIPs: tailnetTargetIPs, TailnetTarget: cfg.TailnetTarget, Ports: cfg.Ports})
+	}
+
+	// Actually apply the firewall rules.
+	if err := ensureRulesAdded(rulesPerSvcToAdd, ep.nfr); err != nil {
+		return nil, fmt.Errorf("error adding rules: %w", err)
+	}
+	if err := ensureRulesDeleted(rulesPerSvcToDelete, ep.nfr); err != nil {
+		return nil, fmt.Errorf("error deleting rules: %w", err)
+	}
+
+	return newStatus, nil
+}
+
+// updatesForCfg calculates any rules that need to be added or deleted for an individucal egress service config.
+func updatesForCfg(svcName string, cfg egressservices.Config, status *egressservices.Status, tailnetTargetIPs []netip.Addr) ([]rule, []rule, error) {
+	rulesToAdd := make([]rule, 0)
+	rulesToDelete := make([]rule, 0)
+	currentConfig, ok := lookupCurrentConfig(svcName, status)
+
+	// If no rules for service are present yet, add them all.
+	if !ok {
+		for _, t := range tailnetTargetIPs {
+			for ports := range cfg.Ports {
+				log.Printf("syncegressservices: svc %s adding port %v", svcName, ports)
+				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: t})
+			}
+		}
+		return rulesToAdd, rulesToDelete, nil
+	}
+
+	// If there are no backend targets available, delete any currently configured rules.
+	if len(tailnetTargetIPs) == 0 {
+		log.Printf("tailnet target for egress service %s does not have any backend addresses, deleting all rules", svcName)
+		for _, ip := range currentConfig.TailnetTargetIPs {
+			for ports := range currentConfig.Ports {
+				rulesToDelete = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
+			}
+		}
+		return rulesToAdd, rulesToDelete, nil
+	}
+
+	// If there are rules present for backend targets that no longer match, delete them.
+	for _, ip := range currentConfig.TailnetTargetIPs {
+		var found bool
+		for _, wantsIP := range tailnetTargetIPs {
+			if reflect.DeepEqual(ip, wantsIP) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			for ports := range currentConfig.Ports {
+				rulesToDelete = append(rulesToDelete, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
+			}
+		}
+	}
+
+	// Sync rules for the currently wanted backend targets.
+	for _, ip := range tailnetTargetIPs {
+
+		// If the backend target is not yet present in status, add all rules.
+		var found bool
+		for _, gotIP := range currentConfig.TailnetTargetIPs {
+			if reflect.DeepEqual(ip, gotIP) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			for ports := range cfg.Ports {
+				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
+			}
+			continue
+		}
+
+		// If the backend target is present in status, check that the
+		// currently applied rules are up to date.
+
+		// Delete any current portmappings that are no longer present in config.
+		for port := range currentConfig.Ports {
+			if _, ok := cfg.Ports[port]; ok {
+				continue
+			}
+			rulesToDelete = append(rulesToDelete, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
+		}
+
+		// Add any new portmappings.
+		for port := range cfg.Ports {
+			if _, ok := currentConfig.Ports[port]; ok {
+				continue
+			}
+			rulesToAdd = append(rulesToAdd, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
+		}
+	}
+	return rulesToAdd, rulesToDelete, nil
+}
+
+// deleteUnneccessaryServices ensure that any services found on status, but not
+// present in config are deleted.
+func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, status *egressservices.Status) error {
+	if !hasServicesConfigured(status) {
+		return nil
+	}
+	if !wantsServicesConfigured(cfgs) {
+		for svcName, svc := range status.Services {
+			log.Printf("service %s is no longer required, deleting", svcName)
+			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
+				return fmt.Errorf("error deleting service %s: %w", svcName, err)
+			}
+		}
+		return nil
+	}
+
+	for svcName, svc := range status.Services {
+		if _, ok := (*cfgs)[svcName]; !ok {
+			log.Printf("service %s is no longer required, deleting", svcName)
+			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
+				return fmt.Errorf("error deleting service %s: %w", svcName, err)
+			}
+			// TODO (irbekrm): also delete the SNAT rule here
+		}
+	}
+	return nil
+}
+
+// getConfigs gets the mounted egress service configuration.
+func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) {
+	j, err := os.ReadFile(ep.cfgPath)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(j) == 0 || string(j) == "" {
+		return nil, nil
+	}
+	cfg := &egressservices.Configs{}
+	if err := json.Unmarshal(j, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// getStatus gets the current status of the configured firewall. The current
+// status is stored in state Secret. Returns nil status if no status that
+// applies to the current proxy Pod was found. Uses the Pod IP to determine if a
+// status found in the state Secret applies to this proxy Pod.
+func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, error) {
+	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving state secret: %w", err)
+	}
+	status := &egressservices.Status{}
+	raw, ok := secret.Data[egressservices.KeyEgressServices]
+	if !ok {
+		return nil, nil
+	}
+	if err := json.Unmarshal([]byte(raw), status); err != nil {
+		return nil, fmt.Errorf("error unmarshalling previous config: %w", err)
+	}
+	if reflect.DeepEqual(status.PodIPv4, ep.podIPv4) {
+		return status, nil
+	}
+	return nil, nil
+}
+
+// setStatus writes egress proxy's currently configured firewall to the state
+// Secret and updates proxy's tailnet addresses.
+func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, n ipn.Notify) error {
+	// Pod IP is used to determine if a stored status applies to THIS proxy Pod.
+	if status == nil {
+		status = &egressservices.Status{}
+	}
+	status.PodIPv4 = ep.podIPv4
+	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
+	if err != nil {
+		return fmt.Errorf("error retrieving state Secret: %w", err)
+	}
+	bs, err := json.Marshal(status)
+	if err != nil {
+		return fmt.Errorf("error marshalling service config: %w", err)
+	}
+	secret.Data[egressservices.KeyEgressServices] = bs
+	patch := kubeclient.JSONPatch{
+		Op:    "replace",
+		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
+		Value: bs,
+	}
+	if err := ep.kc.JSONPatchSecret(ctx, ep.stateSecret, []kubeclient.JSONPatch{patch}); err != nil {
+		return fmt.Errorf("error patching state Secret: %w", err)
+	}
+	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
+	return nil
+}
+
+// tailnetTargetIPsForSvc returns the tailnet IPs to which traffic for this
+// egress service should be proxied. The egress service can be configured by IP
+// or by FQDN. If it's configured by IP, just return that. If it's configured by
+// FQDN, resolve the FQDN and return the resolved IPs. It checks if the
+// netfilter runner supports IPv6 NAT and skips any IPv6 addresses if it
+// doesn't.
+func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.Notify) (addrs []netip.Addr, err error) {
+	if svc.TailnetTarget.IP != "" {
+		addr, err := netip.ParseAddr(svc.TailnetTarget.IP)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing tailnet target IP: %w", err)
+		}
+		if addr.Is6() && !ep.nfr.HasIPV6NAT() {
+			log.Printf("tailnet target is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode. This will probably not work.")
+			return addrs, nil
+		}
+		return []netip.Addr{addr}, nil
+	}
+
+	if svc.TailnetTarget.FQDN == "" {
+		return nil, errors.New("unexpected egress service config- neither tailnet target IP nor FQDN is set")
+	}
+	if n.NetMap == nil {
+		log.Printf("netmap is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN)
+		return addrs, nil
+	}
+	var (
+		node      tailcfg.NodeView
+		nodeFound bool
+	)
+	for _, nn := range n.NetMap.Peers {
+		if equalFQDNs(nn.Name(), svc.TailnetTarget.FQDN) {
+			node = nn
+			nodeFound = true
+			break
+		}
+	}
+	if nodeFound {
+		for _, addr := range node.Addresses().AsSlice() {
+			if addr.Addr().Is6() && !ep.nfr.HasIPV6NAT() {
+				log.Printf("tailnet target %v is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode, skipping.", addr.Addr().String())
+				continue
+			}
+			addrs = append(addrs, addr.Addr())
+		}
+		// Egress target endpoints configured via FQDN are stored, so
+		// that we can determine if a netmap update should trigger a
+		// resync.
+		mak.Set(&ep.targetFQDNs, svc.TailnetTarget.FQDN, node.Addresses().AsSlice())
+	}
+	return addrs, nil
+}
+
+// shouldResync parses netmap update and returns true if the update contains
+// changes for which the egress proxy's firewall should be reconfigured.
+func (ep *egressProxy) shouldResync(n ipn.Notify) bool {
+	if n.NetMap == nil {
+		return false
+	}
+
+	// If proxy's tailnet addresses have changed, resync.
+	if !reflect.DeepEqual(n.NetMap.SelfNode.Addresses().AsSlice(), ep.tailnetAddrs) {
+		log.Printf("node addresses have changed, trigger egress config resync")
+		ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
+		return true
+	}
+
+	// If the IPs for any of the egress services configured via FQDN have
+	// changed, resync.
+	for fqdn, ips := range ep.targetFQDNs {
+		for _, nn := range n.NetMap.Peers {
+			if equalFQDNs(nn.Name(), fqdn) {
+				if !reflect.DeepEqual(ips, nn.Addresses().AsSlice()) {
+					log.Printf("backend addresses for egress target %q have changed old IPs %v, new IPs %v trigger egress config resync", nn.Name(), ips, nn.Addresses().AsSlice())
+				}
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// ensureServiceDeleted ensures that any rules for an egress service are removed
+// from the firewall configuration.
+func ensureServiceDeleted(svcName string, svc *egressservices.ServiceStatus, nfr linuxfw.NetfilterRunner) error {
+
+	// Note that the portmap is needed for iptables based firewall only.
+	// Nftables group rules for a service in a chain, so there is no need to
+	// specify individual portmapping based rules.
+	pms := make([]linuxfw.PortMap, 0)
+	for pm := range svc.Ports {
+		pms = append(pms, linuxfw.PortMap{MatchPort: pm.MatchPort, TargetPort: pm.TargetPort, Protocol: pm.Protocol})
+	}
+
+	if err := nfr.DeleteSvc(svcName, tailscaleTunInterface, svc.TailnetTargetIPs, pms); err != nil {
+		return fmt.Errorf("error deleting service %s: %w", svcName, err)
+	}
+	return nil
+}
+
+// ensureRulesAdded ensures that all portmapping rules are added to the firewall
+// configuration. For any rules that already exist, calling this function is a
+// no-op. In case of nftables, a service consists of one or two (one per IP
+// family) chains that conain the portmapping rules for the service and the
+// chains as needed when this function is called.
+func ensureRulesAdded(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
+	for svc, rules := range rulesPerSvc {
+		for _, rule := range rules {
+			log.Printf("ensureRulesAdded svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
+			if err := nfr.EnsurePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
+				return fmt.Errorf("error ensuring rule: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+// ensureRulesDeleted ensures that the given rules are deleted from the firewall
+// configuration. For any rules that do not exist, calling this funcion is a
+// no-op.
+func ensureRulesDeleted(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
+	for svc, rules := range rulesPerSvc {
+		for _, rule := range rules {
+			log.Printf("ensureRulesDeleted svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
+			if err := nfr.DeletePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
+				return fmt.Errorf("error deleting rule: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func lookupCurrentConfig(svcName string, status *egressservices.Status) (*egressservices.ServiceStatus, bool) {
+	if status == nil || len(status.Services) == 0 {
+		return nil, false
+	}
+	c, ok := status.Services[svcName]
+	return c, ok
+}
+
+func equalFQDNs(s, s1 string) bool {
+	s, _ = strings.CutSuffix(s, ".")
+	s1, _ = strings.CutSuffix(s1, ".")
+	return strings.EqualFold(s, s1)
+}
+
+// rule contains configuration for an egress proxy firewall rule.
+type rule struct {
+	containerPort uint16     // port to match incoming traffic
+	tailnetPort   uint16     // tailnet service port
+	tailnetIP     netip.Addr // tailnet service IP
+	protocol      string
+}
+
+func wantsServicesConfigured(cfgs *egressservices.Configs) bool {
+	return cfgs != nil && len(*cfgs) != 0
+}
+
+func hasServicesConfigured(status *egressservices.Status) bool {
+	return status != nil && len(status.Services) != 0
+}
+
+func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
+	if st == nil && st1 == nil {
+		return true
+	}
+	if st == nil || st1 == nil {
+		return false
+	}
+	st.PodIPv4 = ""
+	st1.PodIPv4 = ""
+	return reflect.DeepEqual(*st, *st1)
+}

cmd/containerboot/services_test.go

@@ -0,0 +1,175 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"net/netip"
+	"reflect"
+	"testing"
+
+	"tailscale.com/kube/egressservices"
+)
+
+func Test_updatesForSvc(t *testing.T) {
+	tailnetIPv4, tailnetIPv6 := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
+	tailnetIPv4_1, tailnetIPv6_1 := netip.MustParseAddr("100.88.88.88"), netip.MustParseAddr("fd7a:115c:a1e0::4101:512f")
+	ports := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {}}
+	ports1 := map[egressservices.PortMap]struct{}{{Protocol: "udp", MatchPort: 4004, TargetPort: 53}: {}}
+	ports2 := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {},
+		{Protocol: "tcp", MatchPort: 4005, TargetPort: 443}: {}}
+	fqdnSpec := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
+		Ports:         ports,
+	}
+	fqdnSpec1 := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
+		Ports:         ports1,
+	}
+	fqdnSpec2 := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+		Ports:         ports,
+	}
+	fqdnSpec3 := egressservices.Config{
+		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+		Ports:         ports2,
+	}
+	r := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4}
+	r1 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6}
+	r2 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv4}
+	r3 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv6}
+	r4 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4_1}
+	r5 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6_1}
+	r6 := rule{containerPort: 4005, tailnetPort: 443, protocol: "tcp", tailnetIP: tailnetIPv4}
+
+	tests := []struct {
+		name              string
+		svcName           string
+		tailnetTargetIPs  []netip.Addr
+		podIP             string
+		spec              egressservices.Config
+		status            *egressservices.Status
+		wantRulesToAdd    []rule
+		wantRulesToDelete []rule
+	}{
+		{
+			name:              "add_fqdn_svc_that_does_not_yet_exist",
+			svcName:           "test",
+			tailnetTargetIPs:  []netip.Addr{tailnetIPv4, tailnetIPv6},
+			spec:              fqdnSpec,
+			status:            &egressservices.Status{},
+			wantRulesToAdd:    []rule{r, r1},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "fqdn_svc_already_exists",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+			spec:             fqdnSpec,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "fqdn_svc_already_exists_add_port_remove_port",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+			spec:             fqdnSpec1,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{r2, r3},
+			wantRulesToDelete: []rule{r, r1},
+		},
+		{
+			name:             "fqdn_svc_already_exists_change_fqdn_backend_ips",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4_1, tailnetIPv6_1},
+			spec:             fqdnSpec,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
+					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{r4, r5},
+			wantRulesToDelete: []rule{r, r1},
+		},
+		{
+			name:              "add_ip_service",
+			svcName:           "test",
+			tailnetTargetIPs:  []netip.Addr{tailnetIPv4},
+			spec:              fqdnSpec2,
+			status:            &egressservices.Status{},
+			wantRulesToAdd:    []rule{r},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "add_ip_service_already_exists",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
+			spec:             fqdnSpec2,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
+					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "ip_service_add_port",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
+			spec:             fqdnSpec3,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
+					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+					Ports:            ports,
+				}}},
+			wantRulesToAdd:    []rule{r6},
+			wantRulesToDelete: []rule{},
+		},
+		{
+			name:             "ip_service_delete_port",
+			svcName:          "test",
+			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
+			spec:             fqdnSpec,
+			status: &egressservices.Status{
+				Services: map[string]*egressservices.ServiceStatus{"test": {
+					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
+					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
+					Ports:            ports2,
+				}}},
+			wantRulesToAdd:    []rule{},
+			wantRulesToDelete: []rule{r6},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotRulesToAdd, gotRulesToDelete, err := updatesForCfg(tt.svcName, tt.spec, tt.status, tt.tailnetTargetIPs)
+			if err != nil {
+				t.Errorf("updatesForSvc() unexpected error %v", err)
+				return
+			}
+			if !reflect.DeepEqual(gotRulesToAdd, tt.wantRulesToAdd) {
+				t.Errorf("updatesForSvc() got rulesToAdd = \n%v\n want rulesToAdd \n%v", gotRulesToAdd, tt.wantRulesToAdd)
+			}
+			if !reflect.DeepEqual(gotRulesToDelete, tt.wantRulesToDelete) {
+				t.Errorf("updatesForSvc() got rulesToDelete = \n%v\n want rulesToDelete \n%v", gotRulesToDelete, tt.wantRulesToDelete)
+			}
+		})
+	}
+}

cmd/containerboot/settings.go

@@ -0,0 +1,324 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/netip"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"tailscale.com/ipn/conffile"
+	"tailscale.com/kube/kubeclient"
+)
+
+// settings is all the configuration for containerboot.
+type settings struct {
+	AuthKey  string
+	Hostname string
+	Routes   *string
+	// ProxyTargetIP is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. This is typically a
+	// Tailscale IP.
+	TailnetTargetIP string
+	// TailnetTargetFQDN is an MagicDNS name to which all incoming
+	// non-Tailscale traffic should be proxied. This must be a full Tailnet
+	// node FQDN.
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	// Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters
+	PodIP               string
+	PodIPv4             string
+	PodIPv6             string
+	HealthCheckAddrPort string
+	EgressSvcsCfgPath   string
+}
+
+func configFromEnv() (*settings, error) {
+	cfg := &settings{
+		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
+		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
+		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
+		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
+		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
+		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
+		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
+		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
+		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
+		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
+		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
+		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
+		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
+		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
+		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
+		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
+		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
+		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
+		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
+		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
+		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
+		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
+		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
+		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
+	}
+	podIPs, ok := os.LookupEnv("POD_IPS")
+	if ok {
+		ips := strings.Split(podIPs, ",")
+		if len(ips) > 2 {
+			return nil, fmt.Errorf("POD_IPs can contain at most 2 IPs, got %d (%v)", len(ips), ips)
+		}
+		for _, ip := range ips {
+			parsed, err := netip.ParseAddr(ip)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing IP address %s: %w", ip, err)
+			}
+			if parsed.Is4() {
+				cfg.PodIPv4 = parsed.String()
+				continue
+			}
+			cfg.PodIPv6 = parsed.String()
+		}
+	}
+	if err := cfg.validate(); err != nil {
+		return nil, fmt.Errorf("invalid configuration: %v", err)
+	}
+	return cfg, nil
+}
+
+func (s *settings) validate() error {
+	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
+		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
+		}
+	}
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
+	if s.TailnetTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
+		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
+	}
+	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
+	if s.HealthCheckAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+		}
+	}
+	return nil
+}
+
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil {
+		if !kubeclient.IsNotFoundErr(err) {
+			return fmt.Errorf("getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+		}
+
+		if !canCreate {
+			return fmt.Errorf("tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+				"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+				"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+				"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+		}
+	}
+
+	// Return early if we already have an auth key.
+	if cfg.AuthKey != "" || isOneStepConfig(cfg) {
+		return nil
+	}
+
+	if s == nil {
+		log.Print("TS_AUTHKEY not provided and state Secret does not exist, login will be interactive if needed.")
+		return nil
+	}
+
+	keyBytes, _ := s.Data["authkey"]
+	key := string(keyBytes)
+
+	if key != "" {
+		// Enforce that we must be able to patch out the authkey after
+		// authenticating if you want to use this feature. This avoids
+		// us having to deal with the case where we might leave behind
+		// an unnecessary reusable authkey in a secret, like a rake in
+		// the grass.
+		if !cfg.KubernetesCanPatch {
+			return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the Secret to manage the authkey.")
+		}
+		cfg.AuthKey = key
+	}
+
+	log.Print("No authkey found in state Secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+
+	return nil
+}
+
+// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
+// in two steps and login should only happen once.
+// Step 1: run 'tailscaled'
+// Step 2):
+// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
+// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
+func isTwoStepConfigAuthOnce(cfg *settings) bool {
+	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
+// in two steps and we should log in every time it starts.
+// Step 1: run 'tailscaled'
+// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
+func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
+	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isOneStepConfig returns true if the Tailscale node should always be ran and
+// configured in a single step by running 'tailscaled <config opts>'
+func isOneStepConfig(cfg *settings) bool {
+	return cfg.TailscaledConfigFilePath != ""
+}
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != ""
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
+// defaultEnv returns the value of the given envvar name, or defVal if
+// unset.
+func defaultEnv(name, defVal string) string {
+	if v, ok := os.LookupEnv(name); ok {
+		return v
+	}
+	return defVal
+}
+
+// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being set to empty string vs unset.
+func defaultEnvStringPointer(name string) *string {
+	if v, ok := os.LookupEnv(name); ok {
+		return &v
+	}
+	return nil
+}
+
+// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being explicitly set to false vs unset.
+func defaultEnvBoolPointer(name string) *bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return nil
+	}
+	return &ret
+}
+
+func defaultEnvs(names []string, defVal string) string {
+	for _, name := range names {
+		if v, ok := os.LookupEnv(name); ok {
+			return v
+		}
+	}
+	return defVal
+}
+
+// defaultBool returns the boolean value of the given envvar name, or
+// defVal if unset or not a bool.
+func defaultBool(name string, defVal bool) bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return defVal
+	}
+	return ret
+}

cmd/containerboot/tailscaled.go

@@ -0,0 +1,162 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+
+	"tailscale.com/client/tailscale"
+)
+
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+	args := tailscaledArgs(cfg)
+	// tailscaled runs without context, since it needs to persist
+	// beyond the startup timeout in ctx.
+	cmd := exec.Command("tailscaled", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	log.Printf("Starting tailscaled")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
+	}
+
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
+	log.Printf("Waiting for tailscaled socket")
+	for {
+		if ctx.Err() != nil {
+			log.Fatalf("Timed out waiting for tailscaled socket")
+		}
+		_, err := os.Stat(cfg.Socket)
+		if errors.Is(err, fs.ErrNotExist) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		} else if err != nil {
+			log.Fatalf("Waiting for tailscaled socket: %v", err)
+		}
+		break
+	}
+
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process, nil
+}
+
+// tailscaledArgs uses cfg to construct the argv for tailscaled.
+func tailscaledArgs(cfg *settings) []string {
+	args := []string{"--socket=" + cfg.Socket}
+	switch {
+	case cfg.InKubernetes && cfg.KubeSecret != "":
+		args = append(args, "--state=kube:"+cfg.KubeSecret)
+		if cfg.StateDir == "" {
+			cfg.StateDir = "/tmp"
+		}
+		fallthrough
+	case cfg.StateDir != "":
+		args = append(args, "--statedir="+cfg.StateDir)
+	default:
+		args = append(args, "--state=mem:", "--statedir=/tmp")
+	}
+
+	if cfg.UserspaceMode {
+		args = append(args, "--tun=userspace-networking")
+	} else if err := ensureTunFile(cfg.Root); err != nil {
+		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
+	}
+
+	if cfg.SOCKSProxyAddr != "" {
+		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
+	}
+	if cfg.HTTPProxyAddr != "" {
+		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
+	}
+	if cfg.TailscaledConfigFilePath != "" {
+		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
+	}
+	if cfg.DaemonExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
+	}
+	return args
+}
+
+// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
+// if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleUp(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "up"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale up'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale up failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state and only if TS_AUTH_ONCE is set.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	log.Printf("Running 'tailscale set'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale set failed: %v", err)
+	}
+	return nil
+}

cmd/derper/depaware.txt

@@ -7,10 +7,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/cmd/derper+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -48,7 +52,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
@@ -82,10 +86,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoiface                from google.golang.org/protobuf/internal/impl+
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
-        nhooyr.io/websocket                                          from tailscale.com/cmd/derper+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -99,6 +99,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
@@ -112,6 +113,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
@@ -127,7 +129,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -146,9 +148,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
+     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -159,6 +163,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
@@ -171,15 +179,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from tailscale.com/tka
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -251,6 +261,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/ipn+
@@ -266,7 +277,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -295,3 +306,4 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/derper/websocket.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"strings"
 
-	"nhooyr.io/websocket"
+	"github.com/coder/websocket"
 	"tailscale.com/derp"
 	"tailscale.com/net/wsconn"
 )

cmd/derpprobe/derpprobe.go

@@ -75,6 +75,11 @@ func main() {
 		prober.WithPageLink("Prober metrics", "/debug/varz"),
 		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
 	), tsweb.HandlerOptions{Logf: log.Printf}))
+	mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok\n"))
+	}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }

cmd/get-authkey/main.go

@@ -51,6 +51,7 @@ func main() {
 
 	ctx := context.Background()
 	tsClient := tailscale.NewClient("-", nil)
+	tsClient.UserAgent = "tailscale-get-authkey"
 	tsClient.HTTPClient = credentials.Client(ctx)
 	tsClient.BaseURL = baseURL
 

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/k8s-operator/connector.go

@@ -26,6 +26,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -61,11 +62,11 @@ type ConnectorReconciler struct {
 
 var (
 	// gaugeConnectorResources tracks the overall number of Connectors currently managed by this operator instance.
-	gaugeConnectorResources = clientmetric.NewGauge("k8s_connector_resources")
+	gaugeConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorResourceCount)
 	// gaugeConnectorSubnetRouterResources tracks the number of Connectors managed by this operator instance that are subnet routers.
-	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
+	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
-	gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
+	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {

cmd/k8s-operator/connector_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/util/mak"
 )
@@ -74,6 +75,7 @@ func TestConnector(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -169,6 +171,7 @@ func TestConnector(t *testing.T) {
 		parentType:   "connector",
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -254,6 +257,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
+		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -274,7 +278,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassready),
+				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/depaware.txt

@@ -80,6 +80,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -96,7 +100,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -139,7 +143,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
+   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
         github.com/imdario/mergo                                     from k8s.io/client-go/tools/clientcmd
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
@@ -167,7 +171,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink+
         github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
         github.com/modern-go/concurrent                              from github.com/json-iterator/go
@@ -212,6 +216,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tailscale/hujson                                  from tailscale.com/ipn/conffile
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/net/routetable+
+   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/peercred                                from tailscale.com/ipn/ipnauth
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
      💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
@@ -227,7 +232,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
         go.uber.org/multierr                                         from go.uber.org/zap+
@@ -306,11 +310,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
-        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack/gro
         gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
         gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
@@ -417,6 +421,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/util/naming                          from k8s.io/apimachinery/pkg/runtime+
         k8s.io/apimachinery/pkg/util/net                             from k8s.io/apimachinery/pkg/watch+
         k8s.io/apimachinery/pkg/util/rand                            from k8s.io/apiserver/pkg/storage/names
+        k8s.io/apimachinery/pkg/util/remotecommand                   from tailscale.com/k8s-operator/sessionrecording/ws
         k8s.io/apimachinery/pkg/util/runtime                         from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
         k8s.io/apimachinery/pkg/util/sets                            from k8s.io/apimachinery/pkg/api/meta+
         k8s.io/apimachinery/pkg/util/strategicpatch                  from k8s.io/client-go/tools/record+
@@ -594,10 +599,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
         k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
         k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
-        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
-        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
-        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
         sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
         sigs.k8s.io/controller-runtime/pkg/cache/internal            from sigs.k8s.io/controller-runtime/pkg/cache
@@ -653,11 +654,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
-        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+  LW    tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
+        tailscale.com/control/keyfallback                            from tailscale.com/control/controlclient
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/ipn/localapi+
         tailscale.com/disco                                          from tailscale.com/derp+
@@ -667,6 +669,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -686,10 +689,13 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/k8s-operator/apis                              from tailscale.com/k8s-operator/apis/v1alpha1
         tailscale.com/k8s-operator/apis/v1alpha1                     from tailscale.com/cmd/k8s-operator+
         tailscale.com/k8s-operator/sessionrecording                  from tailscale.com/cmd/k8s-operator
-        tailscale.com/k8s-operator/sessionrecording/conn             from tailscale.com/k8s-operator/sessionrecording/spdy
         tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
         tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
-        tailscale.com/kube                                           from tailscale.com/cmd/k8s-operator+
+        tailscale.com/k8s-operator/sessionrecording/ws               from tailscale.com/k8s-operator/sessionrecording
+        tailscale.com/kube/egressservices                            from tailscale.com/cmd/k8s-operator
+        tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
+        tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
+        tailscale.com/kube/kubetypes                                 from tailscale.com/cmd/k8s-operator+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -730,6 +736,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
@@ -741,18 +748,19 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
-        tailscale.com/sessionrecording                               from tailscale.com/cmd/k8s-operator+
+        tailscale.com/sessionrecording                               from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
         tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
@@ -794,7 +802,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/osdiag                                    from tailscale.com/ipn/localapi
    W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
         tailscale.com/util/osshare                                   from tailscale.com/ipn/ipnlocal
-        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal
         tailscale.com/util/progresstracking                          from tailscale.com/ipn/localapi
         tailscale.com/util/race                                      from tailscale.com/net/dns/resolver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -804,11 +812,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
         tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/usermetric                                from tailscale.com/health+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
@@ -825,6 +837,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/tsnet
+        tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -843,8 +856,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
@@ -861,6 +875,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
+        golang.org/x/net/websocket                                   from tailscale.com/k8s-operator/sessionrecording/ws
         golang.org/x/oauth2                                          from golang.org/x/oauth2/clientcredentials+
         golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
         golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+
@@ -945,6 +960,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
+        iter                                                         from go/ast+
         log                                                          from expvar+
         log/internal                                                 from log+
         log/slog                                                     from github.com/go-logr/logr+
@@ -982,7 +998,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from encoding/base32+
-        sort                                                         from archive/tar+
+        sort                                                         from compress/flate+
         strconv                                                      from archive/tar+
         strings                                                      from archive/tar+
         sync                                                         from archive/tar+
@@ -995,3 +1011,4 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -77,6 +77,10 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            {{- if .Values.proxyConfig.defaultProxyClass }}
+            - name: PROXY_DEFAULT_CLASS
+              value: {{ .Values.proxyConfig.defaultProxyClass }}
+            {{- end }}
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -14,19 +14,22 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["events", "services", "services/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingresses", "ingresses/status"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["tailscale.com"]
-  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
+  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status", "proxygroups", "proxygroups/status"]
   verbs: ["get", "list", "watch", "update"]
 - apiGroups: ["tailscale.com"]
   resources: ["dnsconfigs", "dnsconfigs/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["recorders", "recorders/status"]
+  verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -49,13 +52,19 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get","list","watch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings"]
+  verbs: ["get", "create", "patch", "update", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -15,7 +15,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["*"]
+  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -57,12 +57,12 @@ operatorConfig:
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.
-# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
-# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
+# https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
+# https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
 # Note that this section contains only a few global configuration options and
 # will not be updated with more configuration options in the future.
 # If you need more configuration options, take a look at ProxyClass:
-# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
+# https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
@@ -78,10 +78,14 @@ proxyConfig:
   # Note that if you pass multiple tags to this field via `--set` flag to helm upgrade/install commands you must escape the comma (for example, "tag:k8s-proxies\,tag:prod"). See https://github.com/helm/helm/issues/1556
   defaultTags: "tag:k8s"
   firewallMode: auto
+  # If defined, this proxy class will be used as the default proxy class for
+  # service and ingress resources that do not have a proxy class defined. It
+  # does not apply to Connector resources.
+  defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.
-# https://tailscale.com/kb/1236/kubernetes-operator/#accessing-the-kubernetes-control-plane-using-an-api-server-proxy
+# https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
 apiServerProxyConfig:
   mode: "false" # "true", "false", "noauth"
 

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -37,7 +37,7 @@ spec:
             exit node.
             Connector is a cluster-scoped resource.
             More info:
-            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
+            https://tailscale.com/kb/1441/kubernetes-operator-connector
           type: object
           required:
             - spec
@@ -115,7 +115,7 @@ spec:
                     To autoapprove the subnet routes or exit node defined by a Connector,
                     you can configure Tailscale ACLs to give these tags the necessary
                     permissions.
-                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
+                    See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
                     If you specify custom tags here, you must also make the operator an owner of these tags.
                     See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
                     Tags cannot be changed once a Connector node has been created.

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -89,14 +89,14 @@ spec:
                   type: object
                   properties:
                     image:
-                      description: Nameserver image.
+                      description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
                       type: object
                       properties:
                         repo:
                           description: Repo defaults to tailscale/k8s-nameserver.
                           type: string
                         tag:
-                          description: Tag defaults to operator's own tag.
+                          description: Tag defaults to unstable.
                           type: string
             status:
               description: |-

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -30,7 +30,7 @@ spec:
             connector.spec.proxyClass field.
             ProxyClass is a cluster scoped resource.
             More info:
-            https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource.
+            https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
           type: object
           required:
             - spec
@@ -1908,7 +1908,7 @@ spec:
                         routes advertized by other nodes on the tailnet, such as subnet
                         routes.
                         This is equivalent of passing --accept-routes flag to a tailscale Linux client.
-                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
+                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
             status:

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -0,0 +1,187 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+  name: proxygroups.tailscale.com
+spec:
+  group: tailscale.com
+  names:
+    kind: ProxyGroup
+    listKind: ProxyGroupList
+    plural: proxygroups
+    shortNames:
+      - pg
+    singular: proxygroup
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Status of the deployed ProxyGroup resources.
+          jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
+          name: Status
+          type: string
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: Spec describes the desired ProxyGroup instances.
+              type: object
+              required:
+                - type
+              properties:
+                hostnamePrefix:
+                  description: |-
+                    HostnamePrefix is the hostname prefix to use for tailnet devices created
+                    by the ProxyGroup. Each device will have the integer number from its
+                    StatefulSet pod appended to this prefix to form the full hostname.
+                    HostnamePrefix can contain lower case letters, numbers and dashes, it
+                    must not start with a dash and must be between 1 and 62 characters long.
+                  type: string
+                  pattern: ^[a-z0-9][a-z0-9-]{0,61}$
+                proxyClass:
+                  description: |-
+                    ProxyClass is the name of the ProxyClass custom resource that contains
+                    configuration options that should be applied to the resources created
+                    for this ProxyGroup. If unset, and there is no default ProxyClass
+                    configured, the operator will create resources with the default
+                    configuration.
+                  type: string
+                replicas:
+                  description: |-
+                    Replicas specifies how many replicas to create the StatefulSet with.
+                    Defaults to 2.
+                  type: integer
+                  format: int32
+                tags:
+                  description: |-
+                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
+                    If you specify custom tags here, make sure you also make the operator
+                    an owner of these tags.
+                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+                    Tags cannot be changed once a ProxyGroup device has been created.
+                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                  type: array
+                  items:
+                    type: string
+                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+                type:
+                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                  type: string
+                  enum:
+                    - egress
+            status:
+              description: |-
+                ProxyGroupStatus describes the status of the ProxyGroup resources. This is
+                set and managed by the Tailscale operator.
+              type: object
+              properties:
+                conditions:
+                  description: |-
+                    List of status conditions to indicate the status of the ProxyGroup
+                    resources. Known condition types are `ProxyGroupReady`.
+                  type: array
+                  items:
+                    description: Condition contains details for one aspect of the current state of this API Resource.
+                    type: object
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        type: string
+                        format: date-time
+                      message:
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
+                        type: string
+                        maxLength: 32768
+                      observedGeneration:
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
+                        type: integer
+                        format: int64
+                        minimum: 0
+                      reason:
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
+                        type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                  x-kubernetes-list-map-keys:
+                    - type
+                  x-kubernetes-list-type: map
+                devices:
+                  description: List of tailnet devices associated with the ProxyGroup StatefulSet.
+                  type: array
+                  items:
+                    type: object
+                    required:
+                      - hostname
+                    properties:
+                      hostname:
+                        description: |-
+                          Hostname is the fully qualified domain name of the device.
+                          If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+                          node.
+                        type: string
+                      tailnetIPs:
+                        description: |-
+                          TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+                          assigned to the device.
+                        type: array
+                        items:
+                          type: string
+                  x-kubernetes-list-map-keys:
+                    - hostname
+                  x-kubernetes-list-type: map
+      served: true
+      storage: true
+      subresources:
+        status: {}

cmd/k8s-operator/deploy/examples/proxygroup.yaml

@@ -0,0 +1,7 @@
+apiVersion: tailscale.com/v1alpha1
+kind: ProxyGroup
+metadata:
+  name: egress-proxies
+spec:
+  type: egress
+  replicas: 3

cmd/k8s-operator/deploy/examples/recorder.yaml

@@ -0,0 +1,6 @@
+apiVersion: tailscale.com/v1alpha1
+kind: Recorder
+metadata:
+  name: recorder
+spec:
+  enableUI: true

cmd/k8s-operator/dnsrecords.go

@@ -24,6 +24,7 @@ import (
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
 )
 
 const (
@@ -167,36 +168,49 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 		}
 	}
 
-	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
-	// headless Service.
+	// Get the Pod IP addresses for the proxy from the EndpointSlices for
+	// the headless Service. The Service can have multiple EndpointSlices
+	// associated with it, for example in dual-stack clusters.
 	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
-	if err != nil {
-		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
+	var eps = new(discoveryv1.EndpointSliceList)
+	if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error listing EndpointSlices for the proxy's headless Service: %w", err)
 	}
-	if eps == nil {
+	if len(eps.Items) == 0 {
 		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
 		return nil
 	}
-	// An EndpointSlice for a Service can have a list of endpoints that each
+	// Each EndpointSlice for a Service can have a list of endpoints that each
 	// can have multiple addresses - these are the IP addresses of any Pods
 	// selected by that Service. Pick all the IPv4 addresses.
-	ips := make([]string, 0)
-	for _, ep := range eps.Endpoints {
-		for _, ip := range ep.Addresses {
-			if !net.IsIPv4String(ip) {
-				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
-			} else {
-				ips = append(ips, ip)
+	// It is also possible that multiple EndpointSlices have overlapping addresses.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints
+	ips := make(set.Set[string], 0)
+	for _, slice := range eps.Items {
+		if slice.AddressType != discoveryv1.AddressTypeIPv4 {
+			logger.Infof("EndpointSlice is for AddressType %s, currently only IPv4 address type is supported", slice.AddressType)
+			continue
+		}
+		for _, ep := range slice.Endpoints {
+			if !epIsReady(&ep) {
+				logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String())
+				continue
+			}
+			for _, ip := range ep.Addresses {
+				if !net.IsIPv4String(ip) {
+					logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+				} else {
+					ips.Add(ip)
+				}
 			}
 		}
 	}
-	if len(ips) == 0 {
+	if ips.Len() == 0 {
 		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
 		return nil
 	}
 	updateFunc := func(rec *operatorutils.Records) {
-		mak.Set(&rec.IP4, fqdn, ips)
+		mak.Set(&rec.IP4, fqdn, ips.Slice())
 	}
 	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
 		return fmt.Errorf("error updating DNS records: %w", err)
@@ -204,6 +218,17 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 	return nil
 }
 
+// epIsReady reports whether the endpoint is currently in a state to receive new
+// traffic. As per kube docs, only explicitly set 'false' for 'Ready' or
+// 'Serving' conditions or explicitly set 'true' for 'Terminating' condition
+// means that the Endpoint is NOT ready.
+// https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/apis/discovery/types.go#L109-L131
+func epIsReady(ep *discoveryv1.Endpoint) bool {
+	return (ep.Conditions.Ready == nil || *ep.Conditions.Ready) &&
+		(ep.Conditions.Serving == nil || *ep.Conditions.Serving) &&
+		(ep.Conditions.Terminating == nil || !*ep.Conditions.Terminating)
+}
+
 // maybeCleanup ensures that the DNS record for the proxy has been removed from
 // dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
 // has been removed from the Service. If the record is not found in the

cmd/k8s-operator/dnsrecords_test.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -87,13 +88,16 @@ func TestDNSRecordsReconciler(t *testing.T) {
 		},
 	}
 	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
-	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4)
+	epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6)
+
 	mustCreate(t, fc, egressSvcFQDN)
 	mustCreate(t, fc, headlessForEgressSvcFQDN)
 	mustCreate(t, fc, ep)
+	mustCreate(t, fc, epv6)
 	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
 	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
-	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} // IPv6 endpoint is currently ignored
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
@@ -106,7 +110,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 3. DNS record is updated if the IP address of the proxy Pod changes.
-	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4)
 	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
 		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
 	})
@@ -116,7 +120,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 
 	// 4. DNS record is created for an ingress proxy configured via Ingress
 	headlessForIngress := headlessSvcForParent(ing, "ingress")
-	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4)
 	mustCreate(t, fc, headlessForIngress)
 	mustCreate(t, fc, ep)
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
@@ -140,6 +144,17 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
 	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
 	expectHostsRecords(t, fc, wantHosts)
+
+	// 7. A not-ready Endpoint is removed from DNS config.
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Conditions.Ready = ptr.To(false)
+		ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{
+			Addresses: []string{"1.2.3.4"},
+		})
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
+	wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"}
+	expectHostsRecords(t, fc, wantHosts)
 }
 
 func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
@@ -162,15 +177,21 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 	}
 }
 
-func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
+func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.AddressType) *discoveryv1.EndpointSlice {
 	return &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      svc.Name,
+			Name:      fmt.Sprintf("%s-%s", svc.Name, string(fam)),
 			Namespace: svc.Namespace,
 			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
 		},
+		AddressType: fam,
 		Endpoints: []discoveryv1.Endpoint{{
 			Addresses: []string{ip},
+			Conditions: discoveryv1.EndpointConditions{
+				Ready:       ptr.To(true),
+				Serving:     ptr.To(true),
+				Terminating: ptr.To(false),
+			},
 		}},
 	}
 }

cmd/k8s-operator/egress-eps.go

@@ -0,0 +1,213 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/netip"
+	"reflect"
+	"strings"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsoperator "tailscale.com/k8s-operator"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/types/ptr"
+)
+
+// egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies.
+type egressEpsReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	tsNamespace string
+}
+
+// Reconcile reconciles an EndpointSlice for a tailnet service. It updates the EndpointSlice with the endpoints of
+// those ProxyGroup Pods that are ready to route traffic to the tailnet service.
+// It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired
+// configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready.
+func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := er.logger.With("Service", req.NamespacedName)
+	l.Debugf("starting reconcile")
+	defer l.Debugf("reconcile finished")
+
+	eps := new(discoveryv1.EndpointSlice)
+	err = er.Get(ctx, req.NamespacedName, eps)
+	if apierrors.IsNotFound(err) {
+		l.Debugf("EndpointSlice not found")
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get EndpointSlice: %w", err)
+	}
+	if !eps.DeletionTimestamp.IsZero() {
+		l.Debugf("EnpointSlice is being deleted")
+		return res, nil
+	}
+
+	// Get the user-created ExternalName Service and use its status conditions to determine whether cluster
+	// resources are set up for this tailnet service.
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      eps.Labels[LabelParentName],
+			Namespace: eps.Labels[LabelParentNamespace],
+		},
+	}
+	err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
+	if apierrors.IsNotFound(err) {
+		l.Infof("ExternalName Service %s/%s not found, perhaps it was deleted", svc.Namespace, svc.Name)
+		return res, nil
+	}
+	if err != nil {
+		return res, fmt.Errorf("error retrieving ExternalName Service: %w", err)
+	}
+	if !tsoperator.EgressServiceIsValidAndConfigured(svc) {
+		l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name)
+		return res, nil
+	}
+
+	// TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is
+	// wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to
+	// determine if a reconcile is needed.
+
+	oldEps := eps.DeepCopy()
+	proxyGroupName := eps.Labels[labelProxyGroup]
+	tailnetSvc := tailnetSvcName(svc)
+	l = l.With("tailnet-service-name", tailnetSvc)
+
+	// Retrieve the desired tailnet service configuration from the ConfigMap.
+	_, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace)
+	if err != nil {
+		return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err)
+	}
+	cfg, ok := (*cfgs)[tailnetSvc]
+	if !ok {
+		l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc)
+		return res, nil
+	}
+
+	// Check which Pods in ProxyGroup are ready to route traffic to this
+	// egress service.
+	podList := &corev1.PodList{}
+	if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil {
+		return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
+	}
+	newEndpoints := make([]discoveryv1.Endpoint, 0)
+	for _, pod := range podList.Items {
+		ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, l)
+		if err != nil {
+			return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err)
+		}
+		if !ready {
+			continue // maybe next time
+		}
+		podIP, err := podIPv4(&pod) // we currently only support IPv4
+		if err != nil {
+			return res, fmt.Errorf("error determining IPv4 address for Pod: %w", err)
+		}
+		newEndpoints = append(newEndpoints, discoveryv1.Endpoint{
+			Hostname:  (*string)(&pod.UID),
+			Addresses: []string{podIP},
+			Conditions: discoveryv1.EndpointConditions{
+				Ready:       ptr.To(true),
+				Serving:     ptr.To(true),
+				Terminating: ptr.To(false),
+			},
+		})
+	}
+	// Note that Endpoints are being overwritten with the currently valid endpoints so we don't need to explicitly
+	// run a cleanup for deleted Pods etc.
+	eps.Endpoints = newEndpoints
+	if !reflect.DeepEqual(eps, oldEps) {
+		l.Infof("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods")
+		if err := er.Update(ctx, eps); err != nil {
+			return res, fmt.Errorf("error updating EndpointSlice: %w", err)
+		}
+	}
+	return res, nil
+}
+
+func podIPv4(pod *corev1.Pod) (string, error) {
+	for _, ip := range pod.Status.PodIPs {
+		parsed, err := netip.ParseAddr(ip.IP)
+		if err != nil {
+			return "", fmt.Errorf("error parsing IP address %s: %w", ip, err)
+		}
+		if parsed.Is4() {
+			return parsed.String(), nil
+		}
+	}
+	return "", nil
+}
+
+// podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to
+// route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service
+// status written there to the desired service configuration.
+func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, l *zap.SugaredLogger) (bool, error) {
+	l = l.With("proxy_pod", pod.Name)
+	l.Debugf("checking whether proxy is ready to route to egress service")
+	if !pod.DeletionTimestamp.IsZero() {
+		l.Debugf("proxy Pod is being deleted, ignore")
+		return false, nil
+	}
+	podIP, err := podIPv4(&pod)
+	if err != nil {
+		return false, fmt.Errorf("error determining Pod IP address: %v", err)
+	}
+	if podIP == "" {
+		l.Infof("[unexpected] Pod does not have an IPv4 address, and IPv6 is not currently supported")
+		return false, nil
+	}
+	stateS := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pod.Name,
+			Namespace: pod.Namespace,
+		},
+	}
+	err = er.Get(ctx, client.ObjectKeyFromObject(stateS), stateS)
+	if apierrors.IsNotFound(err) {
+		l.Debugf("proxy does not have a state Secret, waiting...")
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("error getting state Secret: %w", err)
+	}
+	svcStatusBS := stateS.Data[egressservices.KeyEgressServices]
+	if len(svcStatusBS) == 0 {
+		l.Debugf("proxy's state Secret does not contain egress services status, waiting...")
+		return false, nil
+	}
+	svcStatus := &egressservices.Status{}
+	if err := json.Unmarshal(svcStatusBS, svcStatus); err != nil {
+		return false, fmt.Errorf("error unmarshalling egress service status: %w", err)
+	}
+	if !strings.EqualFold(podIP, svcStatus.PodIPv4) {
+		l.Infof("proxy's egress service status is for Pod IP %s, current proxy's Pod IP %s, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP)
+		return false, nil
+	}
+	st, ok := (*svcStatus).Services[tailnetSvcName]
+	if !ok {
+		l.Infof("proxy's state Secret does not have egress service status, waiting...")
+		return false, nil
+	}
+	if !reflect.DeepEqual(cfg.TailnetTarget, st.TailnetTarget) {
+		l.Infof("proxy has configured egress service for tailnet target %v, current target is %v, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget)
+		return false, nil
+	}
+	if !reflect.DeepEqual(cfg.Ports, st.Ports) {
+		l.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports)
+		return false, nil
+	}
+	l.Debugf("proxy is ready to route traffic to egress service")
+	return true, nil
+}

cmd/k8s-operator/egress-eps_test.go

@@ -0,0 +1,211 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand/v2"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/tstest"
+	"tailscale.com/util/mak"
+)
+
+func TestTailscaleEgressEndpointSlices(t *testing.T) {
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: "foo.bar.ts.net",
+				AnnotationProxyGroup:        "foo",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+			},
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{
+				condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, "", "", clock),
+				condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "", "", clock),
+			},
+		},
+	}
+	port := randomPort()
+	cm := configMapForSvc(t, svc, port)
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(svc, cm).
+		WithStatusSubresource(svc).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	er := &egressEpsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		tsNamespace: "operator-ns",
+	}
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: "operator-ns",
+			Labels: map[string]string{
+				LabelParentName:      "test",
+				LabelParentNamespace: "default",
+				labelSvcType:         typeEgress,
+				labelProxyGroup:      "foo"},
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+	mustCreate(t, fc, eps)
+
+	t.Run("no_proxy_group_resources", func(t *testing.T) {
+		expectReconciled(t, er, "operator-ns", "foo") // should not error
+	})
+
+	t.Run("no_pods_ready_to_route_traffic", func(t *testing.T) {
+		pod, stateS := podAndSecretForProxyGroup("foo")
+		mustCreate(t, fc, pod)
+		mustCreate(t, fc, stateS)
+		expectReconciled(t, er, "operator-ns", "foo") // should not error
+	})
+
+	t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) {
+		pod, stateS := podAndSecretForProxyGroup("foo")
+		stBs := serviceStatusForPodIP(t, svc, pod.Status.PodIPs[0].IP, port)
+		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
+			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
+		})
+		expectReconciled(t, er, "operator-ns", "foo")
+		eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
+			Addresses: []string{"10.0.0.1"},
+			Hostname:  pointer.To("foo"),
+			Conditions: discoveryv1.EndpointConditions{
+				Serving:     pointer.ToBool(true),
+				Ready:       pointer.ToBool(true),
+				Terminating: pointer.ToBool(false),
+			},
+		})
+		expectEqual(t, fc, eps, nil)
+	})
+	t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
+		_, stateS := podAndSecretForProxyGroup("foo")           // replica Pod has IP 10.0.0.1
+		stBs := serviceStatusForPodIP(t, svc, "10.0.0.2", port) // status is for a Pod with IP 10.0.0.2
+		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
+			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
+		})
+		expectReconciled(t, er, "operator-ns", "foo")
+		eps.Endpoints = []discoveryv1.Endpoint{}
+		expectEqual(t, fc, eps, nil)
+	})
+}
+
+func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.ConfigMap {
+	t.Helper()
+	ports := make(map[egressservices.PortMap]struct{})
+	for _, port := range svc.Spec.Ports {
+		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
+	}
+	cfg := egressservices.Config{
+		Ports: ports,
+	}
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		cfg.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
+	}
+	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
+		cfg.TailnetTarget = egressservices.TailnetTarget{IP: ip}
+	}
+	name := tailnetSvcName(svc)
+	cfgs := egressservices.Configs{name: cfg}
+	bs, err := json.Marshal(&cfgs)
+	if err != nil {
+		t.Fatalf("error marshalling config: %v", err)
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgEgressCMName(svc.Annotations[AnnotationProxyGroup]),
+			Namespace: "operator-ns",
+		},
+		BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs},
+	}
+	return cm
+}
+
+func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint16) []byte {
+	t.Helper()
+	ports := make(map[egressservices.PortMap]struct{})
+	for _, port := range svc.Spec.Ports {
+		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
+	}
+	svcSt := egressservices.ServiceStatus{Ports: ports}
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		svcSt.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
+	}
+	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
+		svcSt.TailnetTarget = egressservices.TailnetTarget{IP: ip}
+	}
+	svcName := tailnetSvcName(svc)
+	st := egressservices.Status{
+		PodIPv4:  ip,
+		Services: map[string]*egressservices.ServiceStatus{svcName: &svcSt},
+	}
+	bs, err := json.Marshal(st)
+	if err != nil {
+		t.Fatalf("error marshalling service status: %v", err)
+	}
+	return bs
+}
+
+func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
+	p := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-0", pg),
+			Namespace: "operator-ns",
+			Labels:    pgLabels(pg, nil),
+			UID:       "foo",
+		},
+		Status: corev1.PodStatus{
+			PodIPs: []corev1.PodIP{
+				{IP: "10.0.0.1"},
+			},
+		},
+	}
+	s := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-0", pg),
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels(pg, "state"),
+		},
+	}
+	return p, s
+}
+
+func randomPort() uint16 {
+	return uint16(rand.Int32N(1000) + 1000)
+}

cmd/k8s-operator/egress-services-readiness.go

@@ -0,0 +1,179 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstime"
+)
+
+const (
+	reasonReadinessCheckFailed     = "ReadinessCheckFailed"
+	reasonClusterResourcesNotReady = "ClusterResourcesNotReady"
+	reasonNoProxies                = "NoProxiesConfigured"
+	reasonNotReady                 = "NotReadyToRouteTraffic"
+	reasonReady                    = "ReadyToRouteTraffic"
+	reasonPartiallyReady           = "PartiallyReadyToRouteTraffic"
+	msgReadyToRouteTemplate        = "%d out of %d replicas are ready to route traffic"
+)
+
+type egressSvcsReadinessReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	clock       tstime.Clock
+	tsNamespace string
+}
+
+// Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the
+// EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to
+// route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress
+// service to determine how many replicas are currently able to route traffic.
+func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := esrr.logger.With("Service", req.NamespacedName)
+	defer l.Info("reconcile finished")
+
+	svc := new(corev1.Service)
+	if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
+		l.Info("Service not found")
+		return res, nil
+	} else if err != nil {
+		return res, fmt.Errorf("failed to get Service: %w", err)
+	}
+	var (
+		reason, msg string
+		st          metav1.ConditionStatus = metav1.ConditionUnknown
+	)
+	oldStatus := svc.Status.DeepCopy()
+	defer func() {
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
+		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
+			err = errors.Join(err, esrr.Status().Update(ctx, svc))
+		}
+	}()
+
+	crl := egressSvcChildResourceLabels(svc)
+	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl)
+	if err != nil {
+		err = fmt.Errorf("error getting EndpointSlice: %w", err)
+		reason = reasonReadinessCheckFailed
+		msg = err.Error()
+		return res, err
+	}
+	if eps == nil {
+		l.Infof("EndpointSlice for Service does not yet exist, waiting...")
+		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: svc.Annotations[AnnotationProxyGroup],
+		},
+	}
+	err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg)
+	if apierrors.IsNotFound(err) {
+		l.Infof("ProxyGroup for Service does not exist, waiting...")
+		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	if err != nil {
+		err = fmt.Errorf("error retrieving ProxyGroup: %w", err)
+		reason = reasonReadinessCheckFailed
+		msg = err.Error()
+		return res, err
+	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		l.Infof("ProxyGroup for Service is not ready, waiting...")
+		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+
+	replicas := pgReplicas(pg)
+	if replicas == 0 {
+		l.Infof("ProxyGroup replicas set to 0")
+		reason, msg = reasonNoProxies, reasonNoProxies
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	podLabels := pgLabels(pg.Name, nil)
+	var readyReplicas int32
+	for i := range replicas {
+		podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
+		pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
+		if err != nil {
+			err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err)
+			reason = reasonReadinessCheckFailed
+			msg = err.Error()
+			return res, err
+		}
+		if pod == nil {
+			l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
+			reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
+			return res, nil
+		}
+		l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
+		ready := false
+		for _, ep := range eps.Endpoints {
+			l.Infof("looking at endpoint with addresses %v", ep.Addresses)
+			if endpointReadyForPod(&ep, pod, l) {
+				l.Infof("endpoint is ready for Pod")
+				ready = true
+				break
+			}
+		}
+		if ready {
+			readyReplicas++
+		}
+	}
+	msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
+	if readyReplicas == 0 {
+		reason = reasonNotReady
+		st = metav1.ConditionFalse
+		return res, nil
+	}
+	st = metav1.ConditionTrue
+	if readyReplicas < replicas {
+		reason = reasonPartiallyReady
+	} else {
+		reason = reasonReady
+	}
+	return res, nil
+}
+
+// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic.
+// Endpoint must not be nil.
+func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
+	podIP, err := podIPv4(pod)
+	if err != nil {
+		l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
+		return false
+	}
+	// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.
+	if len(ep.Addresses) != 1 {
+		return false
+	}
+	return strings.EqualFold(ep.Addresses[0], podIP) &&
+		*ep.Conditions.Ready &&
+		*ep.Conditions.Serving &&
+		!*ep.Conditions.Terminating
+}

cmd/k8s-operator/egress-services-readiness_test.go

@@ -0,0 +1,169 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+)
+
+func TestEgressServiceReadiness(t *testing.T) {
+	// We need to pass a ProxyGroup object to WithStatusSubresource because of some quirks in how the fake client
+	// works. Without this code further down would not be able to update ProxyGroup status.
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithStatusSubresource(&tsapi.ProxyGroup{}).
+		Build()
+	zl, _ := zap.NewDevelopment()
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	rec := &egressSvcsReadinessReconciler{
+		tsNamespace: "operator-ns",
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       cl,
+	}
+	tailnetFQDN := "my-app.tailnetxyz.ts.net"
+	egressSvc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-app",
+			Namespace: "dev",
+			Annotations: map[string]string{
+				AnnotationProxyGroup:        "dev",
+				AnnotationTailnetTargetFQDN: tailnetFQDN,
+			},
+		},
+	}
+	fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}}
+	l := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-app",
+			Namespace: "operator-ns",
+			Labels:    l,
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "dev",
+		},
+	}
+	mustCreate(t, fc, egressSvc)
+	setClusterNotReady(egressSvc, cl, zl.Sugar())
+	t.Run("endpointslice_does_not_exist", func(t *testing.T) {
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc, nil) // not ready
+	})
+	t.Run("proxy_group_does_not_exist", func(t *testing.T) {
+		mustCreate(t, fc, eps)
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc, nil) // still not ready
+	})
+	t.Run("proxy_group_not_ready", func(t *testing.T) {
+		mustCreate(t, fc, pg)
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc, nil) // still not ready
+	})
+	t.Run("no_ready_replicas", func(t *testing.T) {
+		setPGReady(pg, cl, zl.Sugar())
+		mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
+			p.Status = pg.Status
+		})
+		expectEqual(t, fc, pg, nil)
+		for i := range pgReplicas(pg) {
+			p := pod(pg, i)
+			mustCreate(t, fc, p)
+			mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
+				existing.Status.PodIPs = p.Status.PodIPs
+			})
+		}
+		expectReconciled(t, rec, "dev", "my-app")
+		setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
+		expectEqual(t, fc, egressSvc, nil) // still not ready
+	})
+	t.Run("one_ready_replica", func(t *testing.T) {
+		setEndpointForReplica(pg, 0, eps)
+		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
+			e.Endpoints = eps.Endpoints
+		})
+		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc, nil) // partially ready
+	})
+	t.Run("all_replicas_ready", func(t *testing.T) {
+		for i := range pgReplicas(pg) {
+			setEndpointForReplica(pg, i, eps)
+		}
+		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
+			e.Endpoints = eps.Endpoints
+		})
+		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
+		expectReconciled(t, rec, "dev", "my-app")
+		expectEqual(t, fc, egressSvc, nil) // ready
+	})
+}
+
+func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger) {
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, l)
+}
+
+func setNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas int32) {
+	msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas)
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, l)
+}
+
+func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas, readyReplicas int32) {
+	reason := reasonPartiallyReady
+	if readyReplicas == replicas {
+		reason = reasonReady
+	}
+	msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, l)
+}
+
+func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) {
+	tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l)
+}
+
+func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {
+	p := pod(pg, ordinal)
+	eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
+		Addresses: []string{p.Status.PodIPs[0].IP},
+		Conditions: discoveryv1.EndpointConditions{
+			Ready:       pointer.ToBool(true),
+			Serving:     pointer.ToBool(true),
+			Terminating: pointer.ToBool(false),
+		},
+	})
+}
+
+func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
+	l := pgLabels(pg.Name, nil)
+	l[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
+	ip := fmt.Sprintf("10.0.0.%d", ordinal)
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%d", pg.Name, ordinal),
+			Namespace: "operator-ns",
+			Labels:    l,
+		},
+		Status: corev1.PodStatus{
+			PodIPs: []corev1.PodIP{{IP: ip}},
+		},
+	}
+}

cmd/k8s-operator/egress-services.go

@@ -0,0 +1,716 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"reflect"
+	"slices"
+	"strings"
+	"sync"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/storage/names"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonEgressSvcInvalid        = "EgressSvcInvalid"
+	reasonEgressSvcValid          = "EgressSvcValid"
+	reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
+	reasonProxyGroupNotReady      = "ProxyGroupNotReady"
+
+	labelProxyGroup = "tailscale.com/proxy-group"
+
+	labelSvcType = "tailscale.com/svc-type" // ingress or egress
+	typeEgress   = "egress"
+	// maxPorts is the maximum number of ports that can be exposed on a
+	// container. In practice this will be ports in range [3000 - 4000). The
+	// high range should make it easier to distinguish container ports from
+	// the tailnet target ports for debugging purposes (i.e when reading
+	// netfilter rules). The limit of 10000 is somewhat arbitrary, the
+	// assumption is that this would not be hit in practice.
+	maxPorts = 10000
+
+	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
+)
+
+var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
+
+// egressSvcsReconciler reconciles user created ExternalName Services that specify a tailnet
+// endpoint that should be exposed to cluster workloads and an egress ProxyGroup
+// on whose proxies it should be exposed.
+type egressSvcsReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+
+	mu   sync.Mutex           // protects following
+	svcs set.Slice[types.UID] // UIDs of all currently managed egress Services for ProxyGroup
+}
+
+// Reconcile reconciles an ExternalName Service that specifies a tailnet target and a ProxyGroup on whose proxies should
+// forward cluster traffic to the target.
+// For an ExternalName Service the reconciler:
+//
+// - for each port N defined on the ExternalName Service, allocates a port X in range [3000- 4000), unique for the
+// ProxyGroup proxies. Proxies will forward cluster traffic received on port N to port M on the tailnet target
+//
+// - creates a ClusterIP Service in the operator's namespace with portmappings for all M->N port pairs. This will allow
+// cluster workloads to send traffic on the user-defined tailnet target port and get it transparently mapped to the
+// randomly selected port on proxy Pods.
+//
+// - creates an EndpointSlice in the operator's namespace with kubernetes.io/service-name label pointing to the
+// ClusterIP Service. The endpoints will get dynamically updates to proxy Pod IPs as the Pods become ready to route
+// traffic to the tailnet target. kubernetes.io/service-name label ensures that kube-proxy sets up routing rules to
+// forward cluster traffic received on ClusterIP Service's IP address to the endpoints (Pod IPs).
+//
+// - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the
+// portmappings.
+func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	l := esr.logger.With("Service", req.NamespacedName)
+	defer l.Info("reconcile finished")
+
+	svc := new(corev1.Service)
+	if err = esr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
+		l.Info("Service not found")
+		return res, nil
+	} else if err != nil {
+		return res, fmt.Errorf("failed to get Service: %w", err)
+	}
+
+	// Name of the 'egress service', meaning the tailnet target.
+	tailnetSvc := tailnetSvcName(svc)
+	l = l.With("tailnet-service", tailnetSvc)
+
+	// Note that resources for egress Services are only cleaned up when the
+	// Service is actually deleted (and not if, for example, user decides to
+	// remove the Tailscale annotation from it). This should be fine- we
+	// assume that the egress ExternalName Services are always created for
+	// Tailscale operator specifically.
+	if !svc.DeletionTimestamp.IsZero() {
+		l.Info("Service is being deleted, ensuring resource cleanup")
+		return res, esr.maybeCleanup(ctx, svc, l)
+	}
+
+	oldStatus := svc.Status.DeepCopy()
+	defer func() {
+		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
+			err = errors.Join(err, esr.Status().Update(ctx, svc))
+		}
+	}()
+
+	// Validate the user-created ExternalName Service and the associated ProxyGroup.
+	if ok, err := esr.validateClusterResources(ctx, svc, l); err != nil {
+		return res, fmt.Errorf("error validating cluster resources: %w", err)
+	} else if !ok {
+		return res, nil
+	}
+
+	if !slices.Contains(svc.Finalizers, FinalizerName) {
+		l.Infof("configuring tailnet service") // logged exactly once
+		svc.Finalizers = append(svc.Finalizers, FinalizerName)
+		if err := esr.Update(ctx, svc); err != nil {
+			err := fmt.Errorf("failed to add finalizer: %w", err)
+			r := svcConfiguredReason(svc, false, l)
+			tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
+			return res, err
+		}
+		esr.mu.Lock()
+		esr.svcs.Add(svc.UID)
+		gaugeEgressServices.Set(int64(esr.svcs.Len()))
+		esr.mu.Unlock()
+	}
+
+	if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, l); err != nil {
+		err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err)
+		r := svcConfiguredReason(svc, false, l)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
+		return res, err
+	}
+
+	return res, esr.maybeProvision(ctx, svc, l)
+}
+
+func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) {
+	r := svcConfiguredReason(svc, false, l)
+	st := metav1.ConditionFalse
+	defer func() {
+		msg := r
+		if st != metav1.ConditionTrue && err != nil {
+			msg = err.Error()
+		}
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, l)
+	}()
+
+	crl := egressSvcChildResourceLabels(svc)
+	clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl)
+	if err != nil {
+		err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
+		return err
+	}
+	if clusterIPSvc == nil {
+		clusterIPSvc = esr.clusterIPSvcForEgress(crl)
+	}
+	upToDate := svcConfigurationUpToDate(svc, l)
+	provisioned := true
+	if !upToDate {
+		if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, l); err != nil {
+			return err
+		}
+	}
+	if !provisioned {
+		l.Infof("unable to provision cluster resources")
+		return nil
+	}
+
+	// Update ExternalName Service to point at the ClusterIP Service.
+	clusterDomain := retrieveClusterDomain(esr.tsNamespace, l)
+	clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain)
+	if svc.Spec.ExternalName != clusterIPSvcFQDN {
+		l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
+		svc.Spec.ExternalName = clusterIPSvcFQDN
+		if err = esr.Update(ctx, svc); err != nil {
+			err = fmt.Errorf("error updating ExternalName Service: %w", err)
+			return err
+		}
+	}
+	r = svcConfiguredReason(svc, true, l)
+	st = metav1.ConditionTrue
+	return nil
+}
+
+func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, l *zap.SugaredLogger) (*corev1.Service, bool, error) {
+	l.Infof("updating configuration...")
+	usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName)
+	if err != nil {
+		return nil, false, fmt.Errorf("error calculating used ports for ProxyGroup %s: %w", proxyGroupName, err)
+	}
+
+	oldClusterIPSvc := clusterIPSvc.DeepCopy()
+	// loop over ClusterIP Service ports, remove any that are not needed.
+	for i := len(clusterIPSvc.Spec.Ports) - 1; i >= 0; i-- {
+		pm := clusterIPSvc.Spec.Ports[i]
+		found := false
+		for _, wantsPM := range svc.Spec.Ports {
+			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			l.Debugf("portmapping %s:%d -> %s:%d is no longer required, removing", pm.Protocol, pm.TargetPort.IntVal, pm.Protocol, pm.Port)
+			clusterIPSvc.Spec.Ports = slices.Delete(clusterIPSvc.Spec.Ports, i, i+1)
+		}
+	}
+
+	// loop over ExternalName Service ports, for each one not found on
+	// ClusterIP Service produce new target port and add a portmapping to
+	// the ClusterIP Service.
+	for _, wantsPM := range svc.Spec.Ports {
+		found := false
+		for _, gotPM := range clusterIPSvc.Spec.Ports {
+			if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			// Calculate a free port to expose on container and add
+			// a new PortMap to the ClusterIP Service.
+			if usedPorts.Len() == maxPorts {
+				// TODO(irbekrm): refactor to avoid extra reconciles here. Low priority as in practice,
+				// the limit should not be hit.
+				return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts)
+			}
+			p := unusedPort(usedPorts)
+			l.Debugf("mapping tailnet target port %d to container port %d", wantsPM.Port, p)
+			usedPorts.Insert(p)
+			clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
+				Name:       wantsPM.Name,
+				Protocol:   wantsPM.Protocol,
+				Port:       wantsPM.Port,
+				TargetPort: intstr.FromInt32(p),
+			})
+		}
+	}
+	if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
+		if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
+			svc.Labels = clusterIPSvc.Labels
+			svc.Spec = clusterIPSvc.Spec
+		}); err != nil {
+			return nil, false, fmt.Errorf("error ensuring ClusterIP Service: %v", err)
+		}
+	}
+
+	crl := egressSvcEpsLabels(svc, clusterIPSvc)
+	// TODO(irbekrm): support IPv6, but need to investigate how kube proxy
+	// sets up Service -> Pod routing when IPv6 is involved.
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
+			Namespace: esr.tsNamespace,
+			Labels:    crl,
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Ports:       epsPortsFromSvc(clusterIPSvc),
+	}
+	if eps, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
+		e.Labels = eps.Labels
+		e.AddressType = eps.AddressType
+		e.Ports = eps.Ports
+		for _, p := range e.Endpoints {
+			p.Conditions.Ready = nil
+		}
+	}); err != nil {
+		return nil, false, fmt.Errorf("error ensuring EndpointSlice: %w", err)
+	}
+
+	cm, cfgs, err := egressSvcsConfigs(ctx, esr.Client, proxyGroupName, esr.tsNamespace)
+	if err != nil {
+		return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err)
+	}
+	if cm == nil {
+		l.Info("ConfigMap not yet created, waiting..")
+		return nil, false, nil
+	}
+	tailnetSvc := tailnetSvcName(svc)
+	gotCfg := (*cfgs)[tailnetSvc]
+	wantsCfg := egressSvcCfg(svc, clusterIPSvc)
+	if !reflect.DeepEqual(gotCfg, wantsCfg) {
+		l.Debugf("updating egress services ConfigMap %s", cm.Name)
+		mak.Set(cfgs, tailnetSvc, wantsCfg)
+		bs, err := json.Marshal(cfgs)
+		if err != nil {
+			return nil, false, fmt.Errorf("error marshalling egress services configs: %w", err)
+		}
+		mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
+		if err := esr.Update(ctx, cm); err != nil {
+			return nil, false, fmt.Errorf("error updating egress services ConfigMap: %w", err)
+		}
+	}
+	l.Infof("egress service configuration has been updated")
+	return clusterIPSvc, true, nil
+}
+
+func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
+	logger.Info("ensuring that resources created for egress service are deleted")
+
+	// Delete egress service config from the ConfigMap mounted by the proxies.
+	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, logger); err != nil {
+		return fmt.Errorf("error deleting egress service config: %w", err)
+	}
+
+	// Delete the ClusterIP Service and EndpointSlice for the egress
+	// service.
+	types := []client.Object{
+		&corev1.Service{},
+		&discoveryv1.EndpointSlice{},
+	}
+	crl := egressSvcChildResourceLabels(svc)
+	for _, typ := range types {
+		if err := esr.DeleteAllOf(ctx, typ, client.InNamespace(esr.tsNamespace), client.MatchingLabels(crl)); err != nil {
+			return fmt.Errorf("error deleting %s: %w", typ, err)
+		}
+	}
+
+	ix := slices.Index(svc.Finalizers, FinalizerName)
+	if ix != -1 {
+		logger.Debug("Removing Tailscale finalizer from Service")
+		svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
+		if err := esr.Update(ctx, svc); err != nil {
+			return fmt.Errorf("failed to remove finalizer: %w", err)
+		}
+	}
+	esr.mu.Lock()
+	esr.svcs.Remove(svc.UID)
+	gaugeEgressServices.Set(int64(esr.svcs.Len()))
+	esr.mu.Unlock()
+	logger.Info("successfully cleaned up resources for egress Service")
+	return nil
+}
+
+func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) error {
+	wantsProxyGroup := svc.Annotations[AnnotationProxyGroup]
+	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
+	if cond == nil {
+		return nil
+	}
+	ss := strings.Split(cond.Reason, ":")
+	if len(ss) < 3 {
+		return nil
+	}
+	if strings.EqualFold(wantsProxyGroup, ss[2]) {
+		return nil
+	}
+	esr.logger.Infof("egress Service configured on ProxyGroup %s, wants ProxyGroup %s, cleaning up...", ss[2], wantsProxyGroup)
+	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, l); err != nil {
+		return fmt.Errorf("error deleting egress service config: %w", err)
+	}
+	return nil
+}
+
+// usedPortsForPG calculates the currently used match ports for ProxyGroup
+// containers. It does that by looking by retrieving all target ports of all
+// ClusterIP Services created for egress services exposed on this ProxyGroup's
+// proxies.
+// TODO(irbekrm): this is currently good enough because we only have a single worker and
+// because these Services are created by us, so we can always expect to get the
+// latest ClusterIP Services via the controller cache. It will not work as well
+// once we split into multiple workers- at that point we probably want to set
+// used ports on ProxyGroup's status.
+func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) {
+	svcList := &corev1.ServiceList{}
+	if err := esr.List(ctx, svcList, client.InNamespace(esr.tsNamespace), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
+		return nil, fmt.Errorf("error listing Services: %w", err)
+	}
+	usedPorts := sets.New[int32]()
+	for _, s := range svcList.Items {
+		for _, p := range s.Spec.Ports {
+			usedPorts.Insert(p.TargetPort.IntVal)
+		}
+	}
+	return usedPorts, nil
+}
+
+// clusterIPSvcForEgress returns a template for the ClusterIP Service created
+// for an egress service exposed on ProxyGroup proxies. The ClusterIP Service
+// has no selector. Traffic sent to it will be routed to the endpoints defined
+// by an EndpointSlice created for this egress service.
+func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: svcNameBase(crl[LabelParentName]),
+			Namespace:    esr.tsNamespace,
+			Labels:       crl,
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeClusterIP,
+		},
+	}
+}
+
+func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
+	crl := egressSvcChildResourceLabels(svc)
+	cmName := pgEgressCMName(crl[labelProxyGroup])
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cmName,
+			Namespace: esr.tsNamespace,
+		},
+	}
+	l := logger.With("ConfigMap", client.ObjectKeyFromObject(cm))
+	l.Debug("ensuring that egress service configuration is removed from proxy config")
+	if err := esr.Get(ctx, client.ObjectKeyFromObject(cm), cm); apierrors.IsNotFound(err) {
+		l.Debugf("ConfigMap not found")
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("error retrieving ConfigMap: %w", err)
+	}
+	bs := cm.BinaryData[egressservices.KeyEgressServices]
+	if len(bs) == 0 {
+		l.Debugf("ConfigMap does not contain egress service configs")
+		return nil
+	}
+	cfgs := &egressservices.Configs{}
+	if err := json.Unmarshal(bs, cfgs); err != nil {
+		return fmt.Errorf("error unmarshalling egress services configs")
+	}
+	tailnetSvc := tailnetSvcName(svc)
+	_, ok := (*cfgs)[tailnetSvc]
+	if !ok {
+		l.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted")
+		return nil
+	}
+	l.Infof("before deleting config %+#v", *cfgs)
+	delete(*cfgs, tailnetSvc)
+	l.Infof("after deleting config %+#v", *cfgs)
+	bs, err := json.Marshal(cfgs)
+	if err != nil {
+		return fmt.Errorf("error marshalling egress services configs: %w", err)
+	}
+	mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
+	return esr.Update(ctx, cm)
+}
+
+func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (bool, error) {
+	proxyGroupName := svc.Annotations[AnnotationProxyGroup]
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: proxyGroupName,
+		},
+	}
+	if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
+		l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	} else if err != nil {
+		err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, err
+	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	}
+
+	if violations := validateEgressService(svc, pg); len(violations) > 0 {
+		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
+		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
+		l.Info(msg)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	}
+	l.Debugf("egress service is valid")
+	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
+	return true, nil
+}
+
+func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
+	violations := validateService(svc)
+
+	// We check that only one of these two is set in the earlier validateService function.
+	if svc.Annotations[AnnotationTailnetTargetFQDN] == "" && svc.Annotations[AnnotationTailnetTargetIP] == "" {
+		violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", AnnotationTailnetTargetFQDN, AnnotationTailnetTargetIP))
+	}
+	if len(svc.Spec.Ports) == 0 {
+		violations = append(violations, "egress Service for ProxyGroup must have at least one target Port specified")
+	}
+	if svc.Spec.Type != corev1.ServiceTypeExternalName {
+		violations = append(violations, fmt.Sprintf("unexpected egress Service type %s. The only supported type is ExternalName.", svc.Spec.Type))
+	}
+	if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
+		violations = append(violations, fmt.Sprintf("egress Service references ProxyGroup of type %s, must be type %s", pg.Spec.Type, tsapi.ProxyGroupTypeEgress))
+	}
+	return violations
+}
+
+// egressSvcNameBase returns a name base that can be passed to
+// ObjectMeta.GenerateName to generate a name for the ClusterIP Service.
+// The generated name needs to be short enough so that it can later be used to
+// generate a valid Kubernetes resource name for the EndpointSlice in form
+// 'ipv4-|ipv6-<ClusterIP Service name>.
+// A valid Kubernetes resource name must not be longer than 253 chars.
+func svcNameBase(s string) string {
+	// -ipv4 - ipv6
+	const maxClusterIPSvcNameLength = 253 - 5
+	base := fmt.Sprintf("ts-%s-", s)
+	generator := names.SimpleNameGenerator
+	for {
+		generatedName := generator.GenerateName(base)
+		excess := len(generatedName) - maxClusterIPSvcNameLength
+		if excess <= 0 {
+			return base
+		}
+		base = base[:len(base)-1-excess] // cut off the excess chars
+		base = base + "-"                // re-instate the dash
+	}
+}
+
+// unusedPort returns a port in range [3000 - 4000). The caller must ensure that
+// usedPorts does not contain all ports in range [3000 - 4000).
+func unusedPort(usedPorts sets.Set[int32]) int32 {
+	foundFreePort := false
+	var suggestPort int32
+	for !foundFreePort {
+		suggestPort = rand.Int32N(maxPorts) + 3000
+		if !usedPorts.Has(suggestPort) {
+			foundFreePort = true
+		}
+	}
+	return suggestPort
+}
+
+// tailnetTargetFromSvc returns a tailnet target for the given egress Service.
+// Service must contain exactly one of tailscale.com/tailnet-ip,
+// tailscale.com/tailnet-fqdn annotations.
+func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		return egressservices.TailnetTarget{
+			FQDN: fqdn,
+		}
+	}
+	return egressservices.TailnetTarget{
+		IP: svc.Annotations[AnnotationTailnetTargetIP],
+	}
+}
+
+func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config {
+	tt := tailnetTargetFromSvc(externalNameSvc)
+	cfg := egressservices.Config{TailnetTarget: tt}
+	for _, svcPort := range clusterIPSvc.Spec.Ports {
+		pm := portMap(svcPort)
+		mak.Set(&cfg.Ports, pm, struct{}{})
+	}
+	return cfg
+}
+
+func portMap(p corev1.ServicePort) egressservices.PortMap {
+	// TODO (irbekrm): out of bounds check?
+	return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)}
+}
+
+func isEgressSvcForProxyGroup(obj client.Object) bool {
+	s, ok := obj.(*corev1.Service)
+	if !ok {
+		return false
+	}
+	annots := s.ObjectMeta.Annotations
+	return annots[AnnotationProxyGroup] != "" && (annots[AnnotationTailnetTargetFQDN] != "" || annots[AnnotationTailnetTargetIP] != "")
+}
+
+// egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well
+// as unmarshalled configuration from the ConfigMap.
+func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) {
+	name := pgEgressCMName(proxyGroupName)
+	cm = &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: tsNamespace,
+		},
+	}
+	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
+		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
+	}
+	cfgs = &egressservices.Configs{}
+	if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 {
+		if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], cfgs); err != nil {
+			return nil, nil, fmt.Errorf("error unmarshaling egress services config %v: %w", cm.BinaryData[egressservices.KeyEgressServices], err)
+		}
+	}
+	return cm, cfgs, nil
+}
+
+// egressSvcChildResourceLabels returns labels that should be applied to the
+// ClusterIP Service and the EndpointSlice created for the egress service.
+// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes
+// resource names (ProxyGroup, Service). Maximum allowed label length is 63
+// chars whilst the maximum allowed resource name length is 253 chars, so we
+// should probably validate and truncate (?) the names is they are too long.
+func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
+	return map[string]string{
+		LabelManaged:         "true",
+		LabelParentType:      "svc",
+		LabelParentName:      svc.Name,
+		LabelParentNamespace: svc.Namespace,
+		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:         typeEgress,
+	}
+}
+
+// egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service.
+func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string {
+	l := egressSvcChildResourceLabels(extNSvc)
+	// Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the
+	// endpoints defined on this EndpointSlice.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
+	l[discoveryv1.LabelServiceName] = clusterIPSvc.Name
+	// Kubernetes recommends setting this label.
+	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management
+	l[discoveryv1.LabelManagedBy] = "tailscale.com"
+	return l
+}
+
+func svcConfigurationUpToDate(svc *corev1.Service, l *zap.SugaredLogger) bool {
+	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
+	if cond == nil {
+		return false
+	}
+	if cond.Status != metav1.ConditionTrue {
+		return false
+	}
+	wantsReadyReason := svcConfiguredReason(svc, true, l)
+	return strings.EqualFold(wantsReadyReason, cond.Reason)
+}
+
+func cfgHash(c cfg, l *zap.SugaredLogger) string {
+	bs, err := json.Marshal(c)
+	if err != nil {
+		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
+		l.Infof("error marhsalling Config: %v", err)
+		return ""
+	}
+	h := sha256.New()
+	if _, err := h.Write(bs); err != nil {
+		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
+		l.Infof("error producing Config hash: %v", err)
+		return ""
+	}
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+type cfg struct {
+	Ports         []corev1.ServicePort         `json:"ports"`
+	TailnetTarget egressservices.TailnetTarget `json:"tailnetTarget"`
+	ProxyGroup    string                       `json:"proxyGroup"`
+}
+
+func svcConfiguredReason(svc *corev1.Service, configured bool, l *zap.SugaredLogger) string {
+	var r string
+	if configured {
+		r = "ConfiguredFor:"
+	} else {
+		r = fmt.Sprintf("ConfigurationFailed:%s", r)
+	}
+	r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[AnnotationProxyGroup])
+	tt := tailnetTargetFromSvc(svc)
+	s := cfg{
+		Ports:         svc.Spec.Ports,
+		TailnetTarget: tt,
+		ProxyGroup:    svc.Annotations[AnnotationProxyGroup],
+	}
+	r += fmt.Sprintf(":Config:%s", cfgHash(s, l))
+	return r
+}
+
+// tailnetSvc accepts and ExternalName Service name and returns a name that will be used to distinguish this tailnet
+// service from other tailnet services exposed to cluster workloads.
+func tailnetSvcName(extNSvc *corev1.Service) string {
+	return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name)
+}
+
+// epsPortsFromSvc takes the ClusterIP Service created for an egress service and
+// returns its Port array in a form that can be used for an EndpointSlice.
+func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
+	for _, p := range svc.Spec.Ports {
+		ep = append(ep, discoveryv1.EndpointPort{
+			Protocol: &p.Protocol,
+			Port:     &p.TargetPort.IntVal,
+			Name:     &p.Name,
+		})
+	}
+	return ep
+}

cmd/k8s-operator/egress-services_test.go

@@ -0,0 +1,268 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+)
+
+func TestTailscaleEgressServices(t *testing.T) {
+	pg := &tsapi.ProxyGroup{
+		TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foo",
+			UID:  types.UID("1234-UID"),
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Replicas: pointer.To[int32](3),
+			Type:     tsapi.ProxyGroupTypeEgress,
+		},
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pgEgressCMName("foo"),
+			Namespace: "operator-ns",
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pg, cm).
+		WithStatusSubresource(pg).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+
+	esr := &egressSvcsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       clock,
+		tsNamespace: "operator-ns",
+	}
+	tailnetTargetFQDN := "foo.bar.ts.net."
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
+				AnnotationProxyGroup:        "foo",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+				{
+					Name:     "https",
+					Protocol: "TCP",
+					Port:     443,
+				},
+			},
+		},
+	}
+
+	t.Run("proxy_group_not_ready", func(t *testing.T) {
+		mustCreate(t, fc, svc)
+		expectReconciled(t, esr, "default", "test")
+		// Service should have EgressSvcValid condition set to Unknown.
+		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
+		expectEqual(t, fc, svc, nil)
+	})
+
+	t.Run("proxy_group_ready", func(t *testing.T) {
+		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
+			pg.Status.Conditions = []metav1.Condition{
+				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
+			}
+		})
+		// Quirks of the fake client.
+		mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
+			svc.Status.Conditions = []metav1.Condition{}
+		})
+		expectReconciled(t, esr, "default", "test")
+		// Verify that a ClusterIP Service has been created.
+		name := findGenNameForEgressSvcResources(t, fc, svc)
+		expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
+		clusterSvc := mustGetClusterIPSvc(t, fc, name)
+		// Verify that an EndpointSlice has been created.
+		expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
+		// Verify that ConfigMap contains configuration for the new egress service.
+		mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
+		r := svcConfiguredReason(svc, true, zl.Sugar())
+		// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
+		// CluterIP Service.
+		svc.Status.Conditions = []metav1.Condition{
+			condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
+		}
+		svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
+		svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
+		expectEqual(t, fc, svc, nil)
+	})
+
+	t.Run("delete_external_name_service", func(t *testing.T) {
+		name := findGenNameForEgressSvcResources(t, fc, svc)
+		if err := fc.Delete(context.Background(), svc); err != nil {
+			t.Fatalf("error deleting ExternalName Service: %v", err)
+		}
+		expectReconciled(t, esr, "default", "test")
+		// Verify that ClusterIP Service and EndpointSlice have been deleted.
+		expectMissing[corev1.Service](t, fc, "operator-ns", name)
+		expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
+		// Verify that service config has been deleted from the ConfigMap.
+		mustNotHaveConfigForSvc(t, fc, svc, cm)
+	})
+}
+
+func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
+	return metav1.Condition{
+		Type:               string(typ),
+		Status:             st,
+		LastTransitionTime: conditionTime(clock),
+		Reason:             r,
+		Message:            msg,
+	}
+}
+
+func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
+	t.Helper()
+	labels := egressSvcChildResourceLabels(svc)
+	s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
+	if err != nil {
+		t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
+	}
+	if s == nil {
+		t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name)
+	}
+	return s.GetName()
+}
+
+func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
+	labels := egressSvcChildResourceLabels(extNSvc)
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:         name,
+			Namespace:    "operator-ns",
+			GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name),
+			Labels:       labels,
+		},
+		Spec: corev1.ServiceSpec{
+			Type:  corev1.ServiceTypeClusterIP,
+			Ports: extNSvc.Spec.Ports,
+		},
+	}
+}
+
+func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service {
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: "operator-ns",
+		},
+	}
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil {
+		t.Fatalf("error retrieving Service")
+	}
+	return svc
+}
+
+func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
+	labels := egressSvcChildResourceLabels(extNSvc)
+	labels[discoveryv1.LabelManagedBy] = "tailscale.com"
+	labels[discoveryv1.LabelServiceName] = name
+	return &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-ipv4", name),
+			Namespace: "operator-ns",
+			Labels:    labels,
+		},
+		Ports:       portsForEndpointSlice(clusterIPSvc),
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+}
+
+func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
+	ports := make([]discoveryv1.EndpointPort, 0)
+	for _, p := range svc.Spec.Ports {
+		ports = append(ports, discoveryv1.EndpointPort{
+			Name:     &p.Name,
+			Protocol: &p.Protocol,
+			Port:     pointer.ToInt32(p.TargetPort.IntVal),
+		})
+	}
+	return ports
+}
+
+func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
+	t.Helper()
+	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
+		t.Fatalf("Error retrieving ConfigMap: %v", err)
+	}
+	name := tailnetSvcName(extNSvc)
+	gotCfg := configFromCM(t, cm, name)
+	if gotCfg == nil {
+		t.Fatalf("No config found for service %q", name)
+	}
+	if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" {
+		t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff)
+	}
+}
+
+func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) {
+	t.Helper()
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
+		t.Fatalf("Error retrieving ConfigMap: %v", err)
+	}
+	name := tailnetSvcName(extNSvc)
+	gotCfg := configFromCM(t, cm, name)
+	if gotCfg != nil {
+		t.Fatalf("Config  %#+v for service %q found when it should not be present", gotCfg, name)
+	}
+}
+
+func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config {
+	t.Helper()
+	cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices]
+	if !ok {
+		return nil
+	}
+	cfgs := &egressservices.Configs{}
+	if err := json.Unmarshal(cfgBs, cfgs); err != nil {
+		t.Fatalf("error unmarshalling config: %v", err)
+	}
+	cfg, ok := (*cfgs)[svcName]
+	if ok {
+		return &cfg
+	}
+	return nil
+}

cmd/k8s-operator/generate/main.go

@@ -24,10 +24,14 @@ const (
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
+	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
+	proxyGroupCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
+	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
+	proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -111,7 +115,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass, DNSConfig, Recorder) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -137,28 +141,34 @@ func generate(baseDir string) error {
 		}
 		return nil
 	}
-	if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
-	}
-	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
-		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
+	for _, crd := range []struct {
+		crdPath, templatePath string
+	}{
+		{connectorCRDPath, connectorCRDHelmTemplatePath},
+		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
+		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
+		{recorderCRDPath, recorderCRDHelmTemplatePath},
+		{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
+	} {
+		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
+			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
+		}
 	}
 	return nil
 }
 
 func cleanup(baseDir string) error {
 	log.Print("Cleaning up CRD from Helm templates")
-	if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up Connector CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
-	}
-	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
+	for _, path := range []string{
+		connectorCRDHelmTemplatePath,
+		proxyClassCRDHelmTemplatePath,
+		dnsConfigCRDHelmTemplatePath,
+		recorderCRDHelmTemplatePath,
+		proxyGroupCRDHelmTemplatePath,
+	} {
+		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("error cleaning up %s: %w", path, err)
+		}
 	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -59,6 +59,12 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD not found in default chart install")
+	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: proxygroups.tailscale.com") {
+		t.Errorf("ProxyGroup CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -77,4 +83,10 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
+		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
+	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: proxygroups.tailscale.com") {
+		t.Errorf("ProxyGroup CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/ingress.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -46,12 +47,14 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
+
+	defaultProxyClass string
 }
 
 var (
 	// gaugeIngressResources tracks the number of ingress resources that we're
 	// currently managing.
-	gaugeIngressResources = clientmetric.NewGauge("k8s_ingress_resources")
+	gaugeIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressResourceCount)
 )
 
 func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
@@ -133,7 +136,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing)
+	proxyClass := proxyClassForObject(ing, a.defaultProxyClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/ingress_test.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
@@ -93,6 +94,7 @@ func TestTailscaleIngress(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -224,6 +226,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -250,7 +253,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassready),
+				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/nameserver.go

@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/yaml"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -62,9 +63,7 @@ type NameserverReconciler struct {
 	managedNameservers set.Slice[types.UID] // one or none
 }
 
-var (
-	gaugeNameserverResources = clientmetric.NewGauge("k8s_nameserver_resources")
-)
+var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
 
 func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := a.logger.With("dnsConfig", req.Name)

cmd/k8s-operator/nameserver_test.go

@@ -33,7 +33,7 @@ func TestNameserverReconciler(t *testing.T) {
 		},
 		Spec: tsapi.DNSConfigSpec{
 			Nameserver: &tsapi.Nameserver{
-				Image: &tsapi.Image{
+				Image: &tsapi.NameserverImage{
 					Repo: "test",
 					Tag:  "v0.0.1",
 				},

cmd/k8s-operator/operator.go

@@ -22,6 +22,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -39,6 +40,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/kubestore"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
 	"tailscale.com/types/logger"
@@ -66,6 +68,7 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
@@ -86,9 +89,9 @@ func main() {
 	// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
 	mode := parseAPIProxyMode()
 	if mode == apiserverProxyModeDisabled {
-		hostinfo.SetApp("k8s-operator")
+		hostinfo.SetApp(kubetypes.AppOperator)
 	} else {
-		hostinfo.SetApp("k8s-operator-proxy")
+		hostinfo.SetApp(kubetypes.AppAPIServerProxy)
 	}
 
 	s, tsClient := initTSNet(zlog)
@@ -106,6 +109,7 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
+		defaultProxyClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -139,6 +143,7 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
 	}
 	tsClient := tailscale.NewClient("-", nil)
+	tsClient.UserAgent = "tailscale-k8s-operator"
 	tsClient.HTTPClient = credentials.Client(context.Background())
 
 	s := &tsnet.Server{
@@ -234,10 +239,13 @@ func runReconcilers(opts reconcilerOpts) {
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.Secret{}:             nsFilter,
 				&corev1.ServiceAccount{}:     nsFilter,
+				&corev1.Pod{}:                nsFilter,
 				&corev1.ConfigMap{}:          nsFilter,
 				&appsv1.StatefulSet{}:        nsFilter,
 				&appsv1.Deployment{}:         nsFilter,
 				&discoveryv1.EndpointSlice{}: nsFilter,
+				&rbacv1.Role{}:               nsFilter,
+				&rbacv1.RoleBinding{}:        nsFilter,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
@@ -279,6 +287,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
+			defaultProxyClass:     opts.defaultProxyClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -297,10 +306,11 @@ func runReconcilers(opts reconcilerOpts) {
 		Watches(&corev1.Service{}, svcHandlerForIngress).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress).
 		Complete(&IngressReconciler{
-			ssr:      ssr,
-			recorder: eventRecorder,
-			Client:   mgr.GetClient(),
-			logger:   opts.log.Named("ingress-reconciler"),
+			ssr:               ssr,
+			recorder:          eventRecorder,
+			Client:            mgr.GetClient(),
+			logger:            opts.log.Named("ingress-reconciler"),
+			defaultProxyClass: opts.defaultProxyClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -345,6 +355,65 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create nameserver reconciler: %v", err)
 	}
+
+	egressSvcFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsHandler)
+	egressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsFromEgressProxyGroup(mgr.GetClient(), opts.log))
+	err = builder.
+		ControllerManagedBy(mgr).
+		Named("egress-svcs-reconciler").
+		Watches(&corev1.Service{}, egressSvcFilter).
+		Watches(&tsapi.ProxyGroup{}, egressProxyGroupFilter).
+		Complete(&egressSvcsReconciler{
+			Client:      mgr.GetClient(),
+			tsNamespace: opts.tailscaleNamespace,
+			recorder:    eventRecorder,
+			clock:       tstime.DefaultClock{},
+			logger:      opts.log.Named("egress-svcs-reconciler"),
+		})
+	if err != nil {
+		startlog.Fatalf("could not create egress Services reconciler: %v", err)
+	}
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexEgressProxyGroup, indexEgressServices); err != nil {
+		startlog.Fatalf("failed setting up indexer for egress Services: %v", err)
+	}
+
+	egressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(egressSvcFromEps)
+	err = builder.
+		ControllerManagedBy(mgr).
+		Named("egress-svcs-readiness-reconciler").
+		Watches(&corev1.Service{}, egressSvcFilter).
+		Watches(&discoveryv1.EndpointSlice{}, egressSvcFromEpsFilter).
+		Complete(&egressSvcsReadinessReconciler{
+			Client:      mgr.GetClient(),
+			tsNamespace: opts.tailscaleNamespace,
+			clock:       tstime.DefaultClock{},
+			logger:      opts.log.Named("egress-svcs-readiness-reconciler"),
+		})
+	if err != nil {
+		startlog.Fatalf("could not create egress Services readiness reconciler: %v", err)
+	}
+
+	epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
+	podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace))
+	secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace))
+	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
+
+	err = builder.
+		ControllerManagedBy(mgr).
+		Named("egress-eps-reconciler").
+		Watches(&discoveryv1.EndpointSlice{}, epsFilter).
+		Watches(&corev1.Pod{}, podsFilter).
+		Watches(&corev1.Secret{}, secretsFilter).
+		Watches(&corev1.Service{}, epsFromExtNSvcFilter).
+		Complete(&egressEpsReconciler{
+			Client:      mgr.GetClient(),
+			tsNamespace: opts.tailscaleNamespace,
+			logger:      opts.log.Named("egress-eps-reconciler"),
+		})
+	if err != nil {
+		startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
+	}
+
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
 		Complete(&ProxyClassReconciler{
@@ -384,6 +453,56 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create DNS records reconciler: %v", err)
 	}
+
+	// Recorder reconciler.
+	recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
+	err = builder.ControllerManagedBy(mgr).
+		For(&tsapi.Recorder{}).
+		Watches(&appsv1.StatefulSet{}, recorderFilter).
+		Watches(&corev1.ServiceAccount{}, recorderFilter).
+		Watches(&corev1.Secret{}, recorderFilter).
+		Watches(&rbacv1.Role{}, recorderFilter).
+		Watches(&rbacv1.RoleBinding{}, recorderFilter).
+		Complete(&RecorderReconciler{
+			recorder:    eventRecorder,
+			tsNamespace: opts.tailscaleNamespace,
+			Client:      mgr.GetClient(),
+			l:           opts.log.Named("recorder-reconciler"),
+			clock:       tstime.DefaultClock{},
+			tsClient:    opts.tsClient,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create Recorder reconciler: %v", err)
+	}
+
+	// Recorder reconciler.
+	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
+	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
+	err = builder.ControllerManagedBy(mgr).
+		For(&tsapi.ProxyGroup{}).
+		Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
+		Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
+		Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
+		Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
+		Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
+		Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
+		Complete(&ProxyGroupReconciler{
+			recorder: eventRecorder,
+			Client:   mgr.GetClient(),
+			l:        opts.log.Named("proxygroup-reconciler"),
+			clock:    tstime.DefaultClock{},
+			tsClient: opts.tsClient,
+
+			tsNamespace:       opts.tailscaleNamespace,
+			proxyImage:        opts.proxyImage,
+			defaultTags:       strings.Split(opts.proxyTags, ","),
+			tsFirewallMode:    opts.proxyFirewallMode,
+			defaultProxyClass: opts.defaultProxyClass,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)
+	}
+
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
@@ -424,6 +543,10 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
+	// defaultProxyClass is the name of the ProxyClass to use as the default
+	// class for proxies that do not have a ProxyClass set.
+	// this is defined by an operator env variable.
+	defaultProxyClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
@@ -516,6 +639,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
+	Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error
 }
 
@@ -611,6 +735,27 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
+// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass,
+// returns a list of reconcile requests for all Connectors that have
+// .spec.proxyClass set.
+func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		pgList := new(tsapi.ProxyGroupList)
+		if err := cl.List(ctx, pgList); err != nil {
+			logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		proxyClassName := o.GetName()
+		for _, pg := range pgList.Items {
+			if pg.Spec.ProxyClass == proxyClassName {
+				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
+			}
+		}
+		return reqs
+	}
+}
+
 // serviceHandlerForIngress returns a handler for Service events for ingress
 // reconciler that ensures that if the Service associated with an event is of
 // interest to the reconciler, the associated Ingress(es) gets be reconciled.
@@ -652,6 +797,10 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl
 }
 
 func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
+	if _, ok := o.GetAnnotations()[AnnotationProxyGroup]; ok {
+		// Do not reconcile Services for ProxyGroup.
+		return nil
+	}
 	if isManagedByType(o, "svc") {
 		// If this is a Service managed by a Service we want to enqueue its parent
 		return []reconcile.Request{{NamespacedName: parentFromObjectLabels(o)}}
@@ -677,3 +826,195 @@ func isMagicDNSName(name string) bool {
 	validMagicDNSName := regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`)
 	return validMagicDNSName.MatchString(name)
 }
+
+// egressSvcsHandler returns accepts a Kubernetes object and returns a reconcile
+// request for it , if the object is a Tailscale egress Service meant to be
+// exposed on a ProxyGroup.
+func egressSvcsHandler(_ context.Context, o client.Object) []reconcile.Request {
+	if !isEgressSvcForProxyGroup(o) {
+		return nil
+	}
+	return []reconcile.Request{
+		{
+			NamespacedName: types.NamespacedName{
+				Namespace: o.GetNamespace(),
+				Name:      o.GetName(),
+			},
+		},
+	}
+}
+
+// egressEpsHandler returns accepts an EndpointSlice and, if the EndpointSlice
+// is for an egress service, returns a reconcile request for it.
+func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
+	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
+		return nil
+	}
+	return []reconcile.Request{
+		{
+			NamespacedName: types.NamespacedName{
+				Namespace: o.GetNamespace(),
+				Name:      o.GetName(),
+			},
+		},
+	}
+}
+
+// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
+// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
+func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
+	return func(_ context.Context, o client.Object) []reconcile.Request {
+		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+			return nil
+		}
+		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
+		// have ingress ProxyGroups.
+		if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
+			return nil
+		}
+		pg, ok := o.GetLabels()[LabelParentName]
+		if !ok {
+			return nil
+		}
+		return reconcileRequestsForPG(pg, cl, ns)
+	}
+}
+
+// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is,
+// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
+func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
+	return func(_ context.Context, o client.Object) []reconcile.Request {
+		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+			return nil
+		}
+		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
+		// have ingress ProxyGroups.
+		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
+			return nil
+		}
+		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
+			return nil
+		}
+		pg, ok := o.GetLabels()[LabelParentName]
+		if !ok {
+			return nil
+		}
+		return reconcileRequestsForPG(pg, cl, ns)
+	}
+}
+
+// egressSvcFromEps is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service
+// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service.
+func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
+	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
+		return nil
+	}
+	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		return nil
+	}
+	svcName, ok := o.GetLabels()[LabelParentName]
+	if !ok {
+		return nil
+	}
+	svcNs, ok := o.GetLabels()[LabelParentNamespace]
+	if !ok {
+		return nil
+	}
+	return []reconcile.Request{
+		{
+			NamespacedName: types.NamespacedName{
+				Namespace: svcNs,
+				Name:      svcName,
+			},
+		},
+	}
+}
+
+func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request {
+	epsList := discoveryv1.EndpointSliceList{}
+	if err := cl.List(context.Background(), &epsList,
+		client.InNamespace(ns),
+		client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
+		return nil
+	}
+	reqs := make([]reconcile.Request, 0)
+	for _, ep := range epsList.Items {
+		reqs = append(reqs, reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: ep.Namespace,
+				Name:      ep.Name,
+			},
+		})
+	}
+	return reqs
+}
+
+// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
+// user-created ExternalName Services that should be exposed on this ProxyGroup.
+func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(_ context.Context, o client.Object) []reconcile.Request {
+		pg, ok := o.(*tsapi.ProxyGroup)
+		if !ok {
+			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
+			return nil
+		}
+		if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
+			return nil
+		}
+		svcList := &corev1.ServiceList{}
+		if err := cl.List(context.Background(), svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
+			logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, svc := range svcList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: svc.Namespace,
+					Name:      svc.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
+// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
+// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
+func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
+	return func(_ context.Context, o client.Object) []reconcile.Request {
+		svc, ok := o.(*corev1.Service)
+		if !ok {
+			logger.Infof("[unexpected] Service handler triggered for an object that is not a Service")
+			return nil
+		}
+		if !isEgressSvcForProxyGroup(svc) {
+			return nil
+		}
+		epsList := &discoveryv1.EndpointSliceList{}
+		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
+			client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
+			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, eps := range epsList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: eps.Namespace,
+					Name:      eps.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
+// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
+// index is used a list filter.
+func indexEgressServices(o client.Object) []string {
+	if !isEgressSvcForProxyGroup(o) {
+		return nil
+	}
+	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
+}

cmd/k8s-operator/operator_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime"
@@ -123,6 +124,7 @@ func TestLoadBalancerClass(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
@@ -260,6 +262,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		parentType:        "svc",
 		tailnetTargetFQDN: tailnetTargetFQDN,
 		hostname:          "default-test",
+		app:               kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -371,6 +374,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		parentType:      "svc",
 		tailnetTargetIP: tailnetTargetIP,
 		hostname:        "default-test",
+		app:             kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -479,6 +483,7 @@ func TestAnnotations(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -584,6 +589,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -713,6 +719,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -852,6 +859,7 @@ func TestCustomHostname(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "reindeer-flotilla",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -964,6 +972,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 		hostname:          "tailscale-critical",
 		priorityClassName: "custom-priority-class-name",
 		clusterTargetIP:   "10.20.30.40",
+		app:               kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@@ -1032,6 +1041,7 @@ func TestProxyClassForService(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
@@ -1054,7 +1064,7 @@ func TestProxyClassForService(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassready),
+				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
@@ -1125,6 +1135,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -1181,6 +1192,7 @@ func TestProxyFirewallMode(t *testing.T) {
 		hostname:        "default-test",
 		firewallMode:    "nftables",
 		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 }
@@ -1235,6 +1247,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
 		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
+		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 
@@ -1474,6 +1487,72 @@ func Test_clusterDomainFromResolverConf(t *testing.T) {
 		})
 	}
 }
+func Test_authKeyRemoval(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 1. A new Service that should be exposed via Tailscale gets created, a Secret with a config that contains auth
+	// key is generated.
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+		clock:  clock,
+	}
+
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
+	opts := configOpts{
+		stsName:         shortName,
+		secretName:      fullName,
+		namespace:       "default",
+		parentType:      "svc",
+		hostname:        "default-test",
+		clusterTargetIP: "10.20.30.40",
+		app:             kubetypes.AppIngressProxy,
+	}
+
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+
+	// 2. Apply update to the Secret that imitates the proxy setting device_id.
+	s := expectedSecret(t, fc, opts)
+	mustUpdate(t, fc, s.Namespace, s.Name, func(s *corev1.Secret) {
+		mak.Set(&s.Data, "device_id", []byte("dkkdi4CNTRL"))
+	})
+
+	// 3. Config should no longer contain auth key
+	expectReconciled(t, sr, "default", "test")
+	opts.shouldRemoveAuthKey = true
+	opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")}
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+}
 
 func Test_externalNameService(t *testing.T) {
 	fc := fake.NewFakeClient()
@@ -1529,6 +1608,7 @@ func Test_externalNameService(t *testing.T) {
 		parentType:       "svc",
 		hostname:         "default-test",
 		clusterTargetDNS: "foo.com",
+		app:              kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)

cmd/k8s-operator/proxy.go

@@ -22,9 +22,8 @@ import (
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
-	kubesessionrecording "tailscale.com/k8s-operator/sessionrecording"
-	tskube "tailscale.com/kube"
-	"tailscale.com/sessionrecording"
+	ksr "tailscale.com/k8s-operator/sessionrecording"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -32,11 +31,10 @@ import (
 	"tailscale.com/util/set"
 )
 
-var whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
-
 var (
 	// counterNumRequestsproxies counts the number of API server requests proxied via this proxy.
 	counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
+	whoIsKey                  = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
 )
 
 type apiServerProxyMode int
@@ -168,7 +166,8 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", ap.serveDefault)
-	mux.HandleFunc("/api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExec)
+	mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
+	mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
 
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
@@ -209,9 +208,25 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
 	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
-// serveExec serves 'kubectl exec' requests, optionally configuring the kubectl
-// exec sessions to be recorded.
-func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
+// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.SPDYProtocol)
+}
+
+// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
+// optionally configuring the kubectl exec sessions to be recorded.
+func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
+	ap.execForProto(w, r, ksr.WSProtocol)
+}
+
+func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
+	const (
+		podNameKey       = "pod"
+		namespaceNameKey = "namespace"
+		upgradeHeaderKey = "Upgrade"
+	)
+
 	who, err := ap.whoIs(r)
 	if err != nil {
 		ap.authError(w, err)
@@ -227,15 +242,17 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 		return
 	}
-	kubesessionrecording.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
+	ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
 	if !failOpen && len(addrs) == 0 {
 		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
 		ap.log.Error(msg)
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-	if r.Method != "POST" || r.Header.Get("Upgrade") != "SPDY/3.1" {
-		msg := "'kubectl exec' session recording is configured, but the request is not over SPDY. Session recording is currently only supported for SPDY based clients"
+
+	wantsHeader := upgradeHeaderForProto[proto]
+	if h := r.Header.Get(upgradeHeaderKey); h != wantsHeader {
+		msg := fmt.Sprintf("[unexpected] unable to verify that streaming protocol is %s, wants Upgrade header %q, got: %q", proto, wantsHeader, h)
 		if failOpen {
 			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
 			ap.log.Warn(msg)
@@ -247,9 +264,22 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-	spdyH := kubesessionrecording.New(ap.ts, r, who, w, r.PathValue("pod"), r.PathValue("namespace"), kubesessionrecording.SPDYProtocol, addrs, failOpen, sessionrecording.ConnectToRecorder, ap.log)
 
-	ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	opts := ksr.HijackerOpts{
+		Req:       r,
+		W:         w,
+		Proto:     proto,
+		TS:        ap.ts,
+		Who:       who,
+		Addrs:     addrs,
+		FailOpen:  failOpen,
+		Pod:       r.PathValue(podNameKey),
+		Namespace: r.PathValue(namespaceNameKey),
+		Log:       ap.log,
+	}
+	h := ksr.New(opts)
+
+	ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
 func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
@@ -284,9 +314,11 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
 		log.Printf("failed to add impersonation headers: " + err.Error())
 	}
 }
+
 func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
 	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
 }
+
 func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
 	ap.log.Errorf("failed to authenticate caller: %v", err)
 	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
@@ -307,10 +339,10 @@ const (
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)
@@ -360,7 +392,7 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 		return false, nil, errors.New("[unexpected] cannot determine caller")
 	}
 	failOpen = true
-	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if err != nil {
 		return failOpen, nil, fmt.Errorf("failed to unmarshal Kubernetes capability: %w", err)
 	}
@@ -382,3 +414,8 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 	}
 	return failOpen, recorderAddresses, nil
 }
+
+var upgradeHeaderForProto = map[ksr.Protocol]string{
+	ksr.SPDYProtocol: "SPDY/3.1",
+	ksr.WSProtocol:   "websocket",
+}

cmd/k8s-operator/proxyclass.go

@@ -98,9 +98,9 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	if errs := pcr.validate(pc); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
 		pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
-		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
+		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
 	} else {
-		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
+		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
 	}
 	if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) {
 		if err := pcr.Client.Status().Update(ctx, pc); err != nil {

cmd/k8s-operator/proxyclass_test.go

@@ -69,7 +69,7 @@ func TestProxyClass(t *testing.T) {
 	// 1. A valid ProxyClass resource gets its status updated to Ready.
 	expectReconciled(t, pcr, "", "test")
 	pc.Status.Conditions = append(pc.Status.Conditions, metav1.Condition{
-		Type:               string(tsapi.ProxyClassready),
+		Type:               string(tsapi.ProxyClassReady),
 		Status:             metav1.ConditionTrue,
 		Reason:             reasonProxyClassValid,
 		Message:            reasonProxyClassValid,
@@ -85,7 +85,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
 	expectEvents(t, fr, []string{expectedEvent})
@@ -99,7 +99,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
 	expectEvents(t, fr, []string{expectedEvent})
@@ -118,7 +118,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
 	expectEvents(t, fr, []string{expectedEvent})

cmd/k8s-operator/proxygroup.go

@@ -0,0 +1,533 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"sync"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstime"
+	"tailscale.com/types/ptr"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
+	reasonProxyGroupReady          = "ProxyGroupReady"
+	reasonProxyGroupCreating       = "ProxyGroupCreating"
+	reasonProxyGroupInvalid        = "ProxyGroupInvalid"
+)
+
+var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupCount)
+
+// ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
+type ProxyGroupReconciler struct {
+	client.Client
+	l        *zap.SugaredLogger
+	recorder record.EventRecorder
+	clock    tstime.Clock
+	tsClient tsClient
+
+	// User-specified defaults from the helm installation.
+	tsNamespace       string
+	proxyImage        string
+	defaultTags       []string
+	tsFirewallMode    string
+	defaultProxyClass string
+
+	mu          sync.Mutex           // protects following
+	proxyGroups set.Slice[types.UID] // for proxygroups gauge
+}
+
+func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
+	return r.l.With("ProxyGroup", name)
+}
+
+func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
+	logger := r.logger(req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	pg := new(tsapi.ProxyGroup)
+	err = r.Get(ctx, req.NamespacedName, pg)
+	if apierrors.IsNotFound(err) {
+		logger.Debugf("ProxyGroup not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyGroup: %w", err)
+	}
+	if markedForDeletion(pg) {
+		logger.Debugf("ProxyGroup is being deleted, cleaning up resources")
+		ix := xslices.Index(pg.Finalizers, FinalizerName)
+		if ix < 0 {
+			logger.Debugf("no finalizer, nothing to do")
+			return reconcile.Result{}, nil
+		}
+
+		if done, err := r.maybeCleanup(ctx, pg); err != nil {
+			return reconcile.Result{}, err
+		} else if !done {
+			logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...")
+			return reconcile.Result{RequeueAfter: shortRequeue}, nil
+		}
+
+		pg.Finalizers = slices.Delete(pg.Finalizers, ix, ix+1)
+		if err := r.Update(ctx, pg); err != nil {
+			return reconcile.Result{}, err
+		}
+		return reconcile.Result{}, nil
+	}
+
+	oldPGStatus := pg.Status.DeepCopy()
+	setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
+		if !apiequality.Semantic.DeepEqual(oldPGStatus, pg.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
+		}
+		return reconcile.Result{}, err
+	}
+
+	if !slices.Contains(pg.Finalizers, FinalizerName) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to log that the high level, multi-reconcile
+		// operation is underway.
+		logger.Infof("ensuring ProxyGroup is set up")
+		pg.Finalizers = append(pg.Finalizers, FinalizerName)
+		if err = r.Update(ctx, pg); err != nil {
+			err = fmt.Errorf("error adding finalizer: %w", err)
+			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, reasonProxyGroupCreationFailed)
+		}
+	}
+
+	if err = r.validate(pg); err != nil {
+		message := fmt.Sprintf("ProxyGroup is invalid: %s", err)
+		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupInvalid, message)
+		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupInvalid, message)
+	}
+
+	proxyClassName := r.defaultProxyClass
+	if pg.Spec.ProxyClass != "" {
+		proxyClassName = pg.Spec.ProxyClass
+	}
+
+	var proxyClass *tsapi.ProxyClass
+	if proxyClassName != "" {
+		proxyClass = new(tsapi.ProxyClass)
+		err := r.Get(ctx, types.NamespacedName{Name: proxyClassName}, proxyClass)
+		if apierrors.IsNotFound(err) {
+			err = nil
+			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s does not (yet) exist", proxyClassName)
+			logger.Info(message)
+			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
+		}
+		if err != nil {
+			err = fmt.Errorf("error getting ProxyGroup's ProxyClass %s: %s", proxyClassName, err)
+			r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
+			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
+		}
+		if !tsoperator.ProxyClassIsReady(proxyClass) {
+			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName)
+			logger.Info(message)
+			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
+		}
+	}
+
+	if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
+		err = fmt.Errorf("error provisioning ProxyGroup resources: %w", err)
+		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
+		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
+	}
+
+	desiredReplicas := int(pgReplicas(pg))
+	if len(pg.Status.Devices) < desiredReplicas {
+		message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
+		logger.Debug(message)
+		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
+	}
+
+	if len(pg.Status.Devices) > desiredReplicas {
+		message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas)
+		logger.Debug(message)
+		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
+	}
+
+	logger.Info("ProxyGroup resources synced")
+	return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady)
+}
+
+func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
+	logger := r.logger(pg.Name)
+	r.mu.Lock()
+	r.proxyGroups.Add(pg.UID)
+	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
+	r.mu.Unlock()
+
+	cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass)
+	if err != nil {
+		return fmt.Errorf("error provisioning config Secrets: %w", err)
+	}
+	// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
+	stateSecrets := pgStateSecrets(pg, r.tsNamespace)
+	for _, sec := range stateSecrets {
+		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
+			s.ObjectMeta.Labels = sec.ObjectMeta.Labels
+			s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
+			s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
+		}); err != nil {
+			return fmt.Errorf("error provisioning state Secrets: %w", err)
+		}
+	}
+	sa := pgServiceAccount(pg, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
+		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error provisioning ServiceAccount: %w", err)
+	}
+	role := pgRole(pg, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
+		r.ObjectMeta.Labels = role.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
+		r.Rules = role.Rules
+	}); err != nil {
+		return fmt.Errorf("error provisioning Role: %w", err)
+	}
+	roleBinding := pgRoleBinding(pg, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
+		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
+		r.RoleRef = roleBinding.RoleRef
+		r.Subjects = roleBinding.Subjects
+	}); err != nil {
+		return fmt.Errorf("error provisioning RoleBinding: %w", err)
+	}
+	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+		cm := pgEgressCM(pg, r.tsNamespace)
+		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
+			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
+			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
+		}); err != nil {
+			return fmt.Errorf("error provisioning ConfigMap: %w", err)
+		}
+	}
+	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
+	if err != nil {
+		return fmt.Errorf("error generating StatefulSet spec: %w", err)
+	}
+	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
+		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
+		s.Spec = ss.Spec
+	}); err != nil {
+		return fmt.Errorf("error provisioning StatefulSet: %w", err)
+	}
+
+	if err := r.cleanupDanglingResources(ctx, pg); err != nil {
+		return fmt.Errorf("error cleaning up dangling resources: %w", err)
+	}
+
+	devices, err := r.getDeviceInfo(ctx, pg)
+	if err != nil {
+		return fmt.Errorf("failed to get device info: %w", err)
+	}
+
+	pg.Status.Devices = devices
+
+	return nil
+}
+
+// cleanupDanglingResources ensures we don't leak config secrets, state secrets, and
+// tailnet devices when the number of replicas specified is reduced.
+func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error {
+	logger := r.logger(pg.Name)
+	metadata, err := r.getNodeMetadata(ctx, pg)
+	if err != nil {
+		return err
+	}
+
+	for _, m := range metadata {
+		if m.ordinal+1 <= int(pgReplicas(pg)) {
+			continue
+		}
+
+		// Dangling resource, delete the config + state Secrets, as well as
+		// deleting the device from the tailnet.
+		if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
+			return err
+		}
+		if err := r.Delete(ctx, m.stateSecret); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err)
+			}
+		}
+		configSecret := m.stateSecret.DeepCopy()
+		configSecret.Name += "-config"
+		if err := r.Delete(ctx, configSecret); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// maybeCleanup just deletes the device from the tailnet. All the kubernetes
+// resources linked to a ProxyGroup will get cleaned up via owner references
+// (which we can use because they are all in the same namespace).
+func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.ProxyGroup) (bool, error) {
+	logger := r.logger(pg.Name)
+
+	metadata, err := r.getNodeMetadata(ctx, pg)
+	if err != nil {
+		return false, err
+	}
+
+	for _, m := range metadata {
+		if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
+			return false, err
+		}
+	}
+
+	logger.Infof("cleaned up ProxyGroup resources")
+	r.mu.Lock()
+	r.proxyGroups.Remove(pg.UID)
+	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
+	r.mu.Unlock()
+	return true, nil
+}
+
+func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error {
+	logger.Debugf("deleting device %s from control", string(id))
+	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
+			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
+		} else {
+			return fmt.Errorf("error deleting device: %w", err)
+		}
+	} else {
+		logger.Debugf("device %s deleted from control", string(id))
+	}
+
+	return nil
+}
+
+func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) {
+	logger := r.logger(pg.Name)
+	var allConfigs []tailscaledConfigs
+	for i := range pgReplicas(pg) {
+		cfgSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            fmt.Sprintf("%s-%d-config", pg.Name, i),
+				Namespace:       r.tsNamespace,
+				Labels:          pgSecretLabels(pg.Name, "config"),
+				OwnerReferences: pgOwnerReference(pg),
+			},
+		}
+
+		var existingCfgSecret *corev1.Secret // unmodified copy of secret
+		if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
+			logger.Debugf("secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
+			existingCfgSecret = cfgSecret.DeepCopy()
+		} else if !apierrors.IsNotFound(err) {
+			return "", err
+		}
+
+		var authKey string
+		if existingCfgSecret == nil {
+			logger.Debugf("creating authkey for new ProxyGroup proxy")
+			tags := pg.Spec.Tags.Stringify()
+			if len(tags) == 0 {
+				tags = r.defaultTags
+			}
+			authKey, err = newAuthKey(ctx, r.tsClient, tags)
+			if err != nil {
+				return "", err
+			}
+		}
+
+		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret)
+		if err != nil {
+			return "", fmt.Errorf("error creating tailscaled config: %w", err)
+		}
+		allConfigs = append(allConfigs, configs)
+
+		for cap, cfg := range configs {
+			cfgJSON, err := json.Marshal(cfg)
+			if err != nil {
+				return "", fmt.Errorf("error marshalling tailscaled config: %w", err)
+			}
+			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
+		}
+
+		if existingCfgSecret != nil {
+			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
+			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
+				return "", err
+			}
+		} else {
+			logger.Debugf("creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
+			if err := r.Create(ctx, cfgSecret); err != nil {
+				return "", err
+			}
+		}
+	}
+
+	sum := sha256.New()
+	b, err := json.Marshal(allConfigs)
+	if err != nil {
+		return "", err
+	}
+	if _, err := sum.Write(b); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", sum.Sum(nil)), nil
+}
+
+func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
+	conf := &ipn.ConfigVAlpha{
+		Version:      "alpha0",
+		AcceptDNS:    "false",
+		AcceptRoutes: "false", // AcceptRoutes defaults to true
+		Locked:       "false",
+		Hostname:     ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)),
+	}
+
+	if pg.Spec.HostnamePrefix != "" {
+		conf.Hostname = ptr.To(fmt.Sprintf("%s%d", pg.Spec.HostnamePrefix, idx))
+	}
+
+	if shouldAcceptRoutes(class) {
+		conf.AcceptRoutes = "true"
+	}
+
+	deviceAuthed := false
+	for _, d := range pg.Status.Devices {
+		if d.Hostname == *conf.Hostname {
+			deviceAuthed = true
+			break
+		}
+	}
+
+	if authKey != "" {
+		conf.AuthKey = &authKey
+	} else if !deviceAuthed {
+		key, err := authKeyFromSecret(oldSecret)
+		if err != nil {
+			return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
+		}
+		conf.AuthKey = key
+	}
+	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[106] = *conf
+	return capVerConfigs, nil
+}
+
+func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error {
+	return nil
+}
+
+// getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by
+// querying their state Secrets. It may not return the same number of items as
+// specified in the ProxyGroup spec if e.g. it is getting scaled up or down, or
+// some pods have failed to write state.
+func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) {
+	// List all state secrets owned by this ProxyGroup.
+	secrets := &corev1.SecretList{}
+	if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil {
+		return nil, fmt.Errorf("failed to list state Secrets: %w", err)
+	}
+	for _, secret := range secrets.Items {
+		var ordinal int
+		if _, err := fmt.Sscanf(secret.Name, pg.Name+"-%d", &ordinal); err != nil {
+			return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err)
+		}
+
+		id, dnsName, ok, err := getNodeMetadata(ctx, &secret)
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+
+		metadata = append(metadata, nodeMetadata{
+			ordinal:     ordinal,
+			stateSecret: &secret,
+			tsID:        id,
+			dnsName:     dnsName,
+		})
+	}
+
+	return metadata, nil
+}
+
+func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
+	metadata, err := r.getNodeMetadata(ctx, pg)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range metadata {
+		device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret)
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+		devices = append(devices, tsapi.TailnetDevice{
+			Hostname:   device.Hostname,
+			TailnetIPs: device.TailnetIPs,
+		})
+	}
+
+	return devices, nil
+}
+
+type nodeMetadata struct {
+	ordinal     int
+	stateSecret *corev1.Secret
+	tsID        tailcfg.StableNodeID
+	dnsName     string
+}

cmd/k8s-operator/proxygroup_specs.go

@@ -0,0 +1,294 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/types/ptr"
+)
+
+// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
+// applied over the top after.
+func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) {
+	ss := new(appsv1.StatefulSet)
+	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
+	}
+	// Validate some base assumptions.
+	if len(ss.Spec.Template.Spec.InitContainers) != 1 {
+		return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers))
+	}
+	if len(ss.Spec.Template.Spec.Containers) != 1 {
+		return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers))
+	}
+
+	// StatefulSet config.
+	ss.ObjectMeta = metav1.ObjectMeta{
+		Name:            pg.Name,
+		Namespace:       namespace,
+		Labels:          pgLabels(pg.Name, nil),
+		OwnerReferences: pgOwnerReference(pg),
+	}
+	ss.Spec.Replicas = ptr.To(pgReplicas(pg))
+	ss.Spec.Selector = &metav1.LabelSelector{
+		MatchLabels: pgLabels(pg.Name, nil),
+	}
+
+	// Template config.
+	tmpl := &ss.Spec.Template
+	tmpl.ObjectMeta = metav1.ObjectMeta{
+		Name:                       pg.Name,
+		Namespace:                  namespace,
+		Labels:                     pgLabels(pg.Name, nil),
+		DeletionGracePeriodSeconds: ptr.To[int64](10),
+		Annotations: map[string]string{
+			podAnnotationLastSetConfigFileHash: cfgHash,
+		},
+	}
+	tmpl.Spec.ServiceAccountName = pg.Name
+	tmpl.Spec.InitContainers[0].Image = image
+	tmpl.Spec.Volumes = func() []corev1.Volume {
+		var volumes []corev1.Volume
+		for i := range pgReplicas(pg) {
+			volumes = append(volumes, corev1.Volume{
+				Name: fmt.Sprintf("tailscaledconfig-%d", i),
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i),
+					},
+				},
+			})
+		}
+
+		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+			volumes = append(volumes, corev1.Volume{
+				Name: pgEgressCMName(pg.Name),
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: pgEgressCMName(pg.Name),
+						},
+					},
+				},
+			})
+		}
+
+		return volumes
+	}()
+
+	// Main container config.
+	c := &ss.Spec.Template.Spec.Containers[0]
+	c.Image = image
+	c.VolumeMounts = func() []corev1.VolumeMount {
+		var mounts []corev1.VolumeMount
+		for i := range pgReplicas(pg) {
+			mounts = append(mounts, corev1.VolumeMount{
+				Name:      fmt.Sprintf("tailscaledconfig-%d", i),
+				ReadOnly:  true,
+				MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
+			})
+		}
+
+		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+			mounts = append(mounts, corev1.VolumeMount{
+				Name:      pgEgressCMName(pg.Name),
+				MountPath: "/etc/proxies",
+				ReadOnly:  true,
+			})
+		}
+
+		return mounts
+	}()
+	c.Env = func() []corev1.EnvVar {
+		envs := []corev1.EnvVar{
+			{
+				// TODO(irbekrm): verify that .status.podIPs are always set, else read in .status.podIP as well.
+				Name: "POD_IPS", // this will be a comma separate list i.e 10.136.0.6,2600:1900:4011:161:0:e:0:6
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "status.podIPs",
+					},
+				},
+			},
+			{
+				Name: "POD_NAME",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						// Secret is named after the pod.
+						FieldPath: "metadata.name",
+					},
+				},
+			},
+			{
+				Name:  "TS_KUBE_SECRET",
+				Value: "$(POD_NAME)",
+			},
+			{
+				Name:  "TS_STATE",
+				Value: "kube:$(POD_NAME)",
+			},
+			{
+				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
+				Value: "/etc/tsconfig/$(POD_NAME)",
+			},
+			{
+				Name:  "TS_USERSPACE",
+				Value: "false",
+			},
+		}
+
+		if tsFirewallMode != "" {
+			envs = append(envs, corev1.EnvVar{
+				Name:  "TS_DEBUG_FIREWALL_MODE",
+				Value: tsFirewallMode,
+			})
+		}
+
+		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+			envs = append(envs, corev1.EnvVar{
+				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
+				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
+			})
+		}
+
+		return envs
+	}()
+
+	return ss, nil
+}
+
+func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount {
+	return &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pg.Name,
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+	}
+}
+
+func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pg.Name,
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"get",
+					"patch",
+					"update",
+				},
+				ResourceNames: func() (secrets []string) {
+					for i := range pgReplicas(pg) {
+						secrets = append(secrets,
+							fmt.Sprintf("%s-%d-config", pg.Name, i), // Config with auth key.
+							fmt.Sprintf("%s-%d", pg.Name, i),        // State.
+						)
+					}
+					return secrets
+				}(),
+			},
+		},
+	}
+}
+
+func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pg.Name,
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      pg.Name,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: pg.Name,
+		},
+	}
+}
+
+func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) {
+	for i := range pgReplicas(pg) {
+		secrets = append(secrets, &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            fmt.Sprintf("%s-%d", pg.Name, i),
+				Namespace:       namespace,
+				Labels:          pgSecretLabels(pg.Name, "state"),
+				OwnerReferences: pgOwnerReference(pg),
+			},
+		})
+	}
+
+	return secrets
+}
+
+func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pgEgressCMName(pg.Name),
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+	}
+}
+
+func pgSecretLabels(pgName, typ string) map[string]string {
+	return pgLabels(pgName, map[string]string{
+		labelSecretType: typ, // "config" or "state".
+	})
+}
+
+func pgLabels(pgName string, customLabels map[string]string) map[string]string {
+	l := make(map[string]string, len(customLabels)+3)
+	for k, v := range customLabels {
+		l[k] = v
+	}
+
+	l[LabelManaged] = "true"
+	l[LabelParentType] = "proxygroup"
+	l[LabelParentName] = pgName
+
+	return l
+}
+
+func pgOwnerReference(owner *tsapi.ProxyGroup) []metav1.OwnerReference {
+	return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("ProxyGroup"))}
+}
+
+func pgReplicas(pg *tsapi.ProxyGroup) int32 {
+	if pg.Spec.Replicas != nil {
+		return *pg.Spec.Replicas
+	}
+
+	return 2
+}
+
+func pgEgressCMName(pg string) string {
+	return fmt.Sprintf("%s-egress-config", pg)
+}

cmd/k8s-operator/proxygroup_test.go

@@ -0,0 +1,267 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"tailscale.com/client/tailscale"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
+)
+
+const testProxyImage = "tailscale/tailscale:test"
+
+var defaultProxyClassAnnotations = map[string]string{
+	"some-annotation": "from-the-proxy-class",
+}
+
+func TestProxyGroup(t *testing.T) {
+	pc := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "default-pc",
+		},
+		Spec: tsapi.ProxyClassSpec{
+			StatefulSet: &tsapi.StatefulSet{
+				Annotations: defaultProxyClassAnnotations,
+			},
+		},
+	}
+	pg := &tsapi.ProxyGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Finalizers: []string{"tailscale.com/finalizer"},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pg, pc).
+		WithStatusSubresource(pg, pc).
+		Build()
+	tsClient := &fakeTSClient{}
+	zl, _ := zap.NewDevelopment()
+	fr := record.NewFakeRecorder(1)
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	reconciler := &ProxyGroupReconciler{
+		tsNamespace:       tsNamespace,
+		proxyImage:        testProxyImage,
+		defaultTags:       []string{"tag:test-tag"},
+		tsFirewallMode:    "auto",
+		defaultProxyClass: "default-pc",
+
+		Client:   fc,
+		tsClient: tsClient,
+		recorder: fr,
+		l:        zl.Sugar(),
+		clock:    cl,
+	}
+
+	t.Run("proxyclass_not_ready", func(t *testing.T) {
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
+		expectEqual(t, fc, pg, nil)
+	})
+
+	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
+		pc.Status = tsapi.ProxyClassStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyClassReady),
+				Status:             metav1.ConditionTrue,
+				Reason:             reasonProxyClassValid,
+				Message:            reasonProxyClassValid,
+				LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+			}},
+		}
+		if err := fc.Status().Update(context.Background(), pc); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
+		expectEqual(t, fc, pg, nil)
+		if expected := 1; reconciler.proxyGroups.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
+		}
+		expectProxyGroupResources(t, fc, pg, true)
+		keyReq := tailscale.KeyCapabilities{
+			Devices: tailscale.KeyDeviceCapabilities{
+				Create: tailscale.KeyDeviceCreateCapabilities{
+					Reusable:      false,
+					Ephemeral:     false,
+					Preauthorized: true,
+					Tags:          []string{"tag:test-tag"},
+				},
+			},
+		}
+		if diff := cmp.Diff(tsClient.KeyRequests(), []tailscale.KeyCapabilities{keyReq, keyReq}); diff != "" {
+			t.Fatalf("unexpected secrets (-got +want):\n%s", diff)
+		}
+	})
+
+	t.Run("simulate_successful_device_auth", func(t *testing.T) {
+		addNodeIDToStateSecrets(t, fc, pg)
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		pg.Status.Devices = []tsapi.TailnetDevice{
+			{
+				Hostname:   "hostname-nodeid-0",
+				TailnetIPs: []string{"1.2.3.4", "::1"},
+			},
+			{
+				Hostname:   "hostname-nodeid-1",
+				TailnetIPs: []string{"1.2.3.4", "::1"},
+			},
+		}
+		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
+		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true)
+	})
+
+	t.Run("scale_up_to_3", func(t *testing.T) {
+		pg.Spec.Replicas = ptr.To[int32](3)
+		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
+			p.Spec = pg.Spec
+		})
+		expectReconciled(t, reconciler, "", pg.Name)
+		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
+		expectEqual(t, fc, pg, nil)
+
+		addNodeIDToStateSecrets(t, fc, pg)
+		expectReconciled(t, reconciler, "", pg.Name)
+		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
+		pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{
+			Hostname:   "hostname-nodeid-2",
+			TailnetIPs: []string{"1.2.3.4", "::1"},
+		})
+		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true)
+	})
+
+	t.Run("scale_down_to_1", func(t *testing.T) {
+		pg.Spec.Replicas = ptr.To[int32](1)
+		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
+			p.Spec = pg.Spec
+		})
+		expectReconciled(t, reconciler, "", pg.Name)
+		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
+		expectEqual(t, fc, pg, nil)
+
+		expectProxyGroupResources(t, fc, pg, true)
+	})
+
+	t.Run("delete_and_cleanup", func(t *testing.T) {
+		if err := fc.Delete(context.Background(), pg); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		expectMissing[tsapi.Recorder](t, fc, "", pg.Name)
+		if expected := 0; reconciler.proxyGroups.Len() != expected {
+			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
+		}
+		// 2 nodes should get deleted as part of the scale down, and then finally
+		// the first node gets deleted with the ProxyGroup cleanup.
+		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" {
+			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
+		}
+		// The fake client does not clean up objects whose owner has been
+		// deleted, so we can't test for the owned resources getting deleted.
+	})
+}
+
+func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool) {
+	t.Helper()
+
+	role := pgRole(pg, tsNamespace)
+	roleBinding := pgRoleBinding(pg, tsNamespace)
+	serviceAccount := pgServiceAccount(pg, tsNamespace)
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	statefulSet.Annotations = defaultProxyClassAnnotations
+
+	if shouldExist {
+		expectEqual(t, fc, role, nil)
+		expectEqual(t, fc, roleBinding, nil)
+		expectEqual(t, fc, serviceAccount, nil)
+		expectEqual(t, fc, statefulSet, func(ss *appsv1.StatefulSet) {
+			ss.Spec.Template.Annotations[podAnnotationLastSetConfigFileHash] = ""
+		})
+	} else {
+		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
+		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
+		expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
+		expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
+	}
+
+	var expectedSecrets []string
+	for i := range pgReplicas(pg) {
+		expectedSecrets = append(expectedSecrets,
+			fmt.Sprintf("%s-%d", pg.Name, i),
+			fmt.Sprintf("%s-%d-config", pg.Name, i),
+		)
+	}
+	expectSecrets(t, fc, expectedSecrets)
+}
+
+func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) {
+	t.Helper()
+
+	secrets := &corev1.SecretList{}
+	if err := fc.List(context.Background(), secrets); err != nil {
+		t.Fatal(err)
+	}
+
+	var actual []string
+	for _, secret := range secrets.Items {
+		actual = append(actual, secret.Name)
+	}
+
+	if diff := cmp.Diff(actual, expected); diff != "" {
+		t.Fatalf("unexpected secrets (-got +want):\n%s", diff)
+	}
+}
+
+func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) {
+	const key = "profile-abc"
+	for i := range pgReplicas(pg) {
+		bytes, err := json.Marshal(map[string]any{
+			"Config": map[string]any{
+				"NodeID": fmt.Sprintf("nodeid-%d", i),
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) {
+			s.Data = map[string][]byte{
+				currentProfileKey: []byte(key),
+				key:               bytes,
+			}
+		})
+	}
+}

cmd/k8s-operator/sts.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
@@ -46,11 +47,11 @@ const (
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
+	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
-	// LabelProxyClass can be set by users on Connectors, tailscale
-	// Ingresses and Services that define cluster ingress or cluster egress,
-	// to specify that configuration in this ProxyClass should be applied to
-	// resources created for the Connector, Ingress or Service.
+	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
+	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
+	// the Ingress or Service.
 	LabelProxyClass = "tailscale.com/proxy-class"
 
 	FinalizerName = "tailscale.com/finalizer"
@@ -64,6 +65,8 @@ const (
 	//MagicDNS name of tailnet node.
 	AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
 
+	AnnotationProxyGroup = "tailscale.com/proxy-group"
+
 	// Annotations settable by users on ingresses.
 	AnnotationFunnel = "tailscale.com/funnel"
 
@@ -301,7 +304,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaleConfigs, _ error) {
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaledConfigs, _ error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			// Hardcode a -0 suffix so that in future, if we support
@@ -342,7 +345,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		if len(tags) == 0 {
 			tags = a.defaultTags
 		}
-		authKey, err = a.newAuthKey(ctx, tags)
+		authKey, err = newAuthKey(ctx, a.tsClient, tags)
 		if err != nil {
 			return "", "", nil, err
 		}
@@ -359,7 +362,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 	latest := tailcfg.CapabilityVersion(-1)
 	var latestConfig ipn.ConfigVAlpha
 	for key, val := range configs {
-		fn := tsoperator.TailscaledConfigFileNameForCap(key)
+		fn := tsoperator.TailscaledConfigFileName(key)
 		b, err := json.Marshal(val)
 		if err != nil {
 			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
@@ -418,6 +421,11 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	if sec == nil {
 		return "", "", nil, nil
 	}
+
+	return deviceInfo(sec)
+}
+
+func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	id = tailcfg.StableNodeID(sec.Data["device_id"])
 	if id == "" {
 		return "", "", nil, nil
@@ -441,7 +449,7 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	return id, hostname, ips, nil
 }
 
-func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {
+func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) {
 	caps := tailscale.KeyCapabilities{
 		Devices: tailscale.KeyDeviceCapabilities{
 			Create: tailscale.KeyDeviceCreateCapabilities{
@@ -452,7 +460,7 @@ func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string)
 		},
 	}
 
-	key, _, err := a.tsClient.CreateKey(ctx, caps)
+	key, _, err := tsClient.CreateKey(ctx, caps)
 	if err != nil {
 		return "", err
 	}
@@ -598,6 +606,18 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	}
+	app, err := appInfoForProxy(sts)
+	if err != nil {
+		// No need to error out if now or in future we end up in a
+		// situation where app info cannot be determined for one of the
+		// many proxy configurations that the operator can produce.
+		logger.Error("[unexpected] unable to determine proxy type")
+	} else {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_INTERNAL_APP",
+			Value: app,
+		})
+	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	if sts.ProxyClassName != "" {
 		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
@@ -611,6 +631,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS)
 }
 
+func appInfoForProxy(cfg *tailscaleSTSConfig) (string, error) {
+	if cfg.ClusterTargetDNSName != "" || cfg.ClusterTargetIP != "" {
+		return kubetypes.AppIngressProxy, nil
+	}
+	if cfg.TailnetTargetFQDN != "" || cfg.TailnetTargetIP != "" {
+		return kubetypes.AppEgressProxy, nil
+	}
+	if cfg.ServeConfig != nil {
+		return kubetypes.AppIngressResource, nil
+	}
+	if cfg.Connector != nil {
+		return kubetypes.AppConnector, nil
+	}
+	return "", errors.New("unable to determine proxy type")
+}
+
 // mergeStatefulSetLabelsOrAnnots returns a map that contains all keys/values
 // present in 'custom' map as well as those keys/values from the current map
 // whose keys are present in the 'managed' map. The reason why this merge is
@@ -636,7 +672,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	if pc == nil || ss == nil {
 		return ss
 	}
-	if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
+	if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
 		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
 			enableMetrics(ss, pc)
 		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
@@ -758,7 +794,7 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 // TODO (irbekrm): remove the legacy config once we no longer need to support
 // versions older than cap94,
 // https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
-func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaleConfigs, error) {
+func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
 		AcceptDNS:           "false",
@@ -787,33 +823,12 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 
 	if newAuthkey != "" {
 		conf.AuthKey = &newAuthkey
-	} else if oldSecret != nil {
-		var err error
-		latest := tailcfg.CapabilityVersion(-1)
-		latestStr := ""
-		for k, data := range oldSecret.Data {
-			// write to StringData, read from Data as StringData is write-only
-			if len(data) == 0 {
-				continue
-			}
-			v, err := tsoperator.CapVerFromFileName(k)
-			if err != nil {
-				continue
-			}
-			if v > latest {
-				latestStr = k
-				latest = v
-			}
-		}
-		// Allow for configs that don't contain an auth key. Perhaps
-		// users have some mechanisms to delete them. Auth key is
-		// normally not needed after the initial login.
-		if latestStr != "" {
-			conf.AuthKey, err = readAuthKey(oldSecret, latestStr)
-			if err != nil {
-				return nil, err
-			}
+	} else if shouldRetainAuthKey(oldSecret) {
+		key, err := authKeyFromSecret(oldSecret)
+		if err != nil {
+			return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
 		}
+		conf.AuthKey = key
 	}
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
 	capVerConfigs[95] = *conf
@@ -823,6 +838,41 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 	return capVerConfigs, nil
 }
 
+func authKeyFromSecret(s *corev1.Secret) (key *string, err error) {
+	latest := tailcfg.CapabilityVersion(-1)
+	latestStr := ""
+	for k, data := range s.Data {
+		// write to StringData, read from Data as StringData is write-only
+		if len(data) == 0 {
+			continue
+		}
+		v, err := tsoperator.CapVerFromFileName(k)
+		if err != nil {
+			continue
+		}
+		if v > latest {
+			latestStr = k
+			latest = v
+		}
+	}
+	// Allow for configs that don't contain an auth key. Perhaps
+	// users have some mechanisms to delete them. Auth key is
+	// normally not needed after the initial login.
+	if latestStr != "" {
+		return readAuthKey(s, latestStr)
+	}
+	return key, nil
+}
+
+// shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be
+// retained (because the proxy has not yet successfully authenticated).
+func shouldRetainAuthKey(s *corev1.Secret) bool {
+	if s == nil {
+		return false // nothing to retain here
+	}
+	return len(s.Data["device_id"]) == 0 // proxy has not authed yet
+}
+
 func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
 	return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
 }
@@ -834,7 +884,7 @@ type ptrObject[T any] interface {
 	*T
 }
 
-type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
+type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
 
 // hashBytes produces a hash for the provided tailscaled config that is the same across
 // different invocations of this code. We do not use the
@@ -845,7 +895,7 @@ type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
 // thing that changed is operator version (the hash is also exposed to users via
 // an annotation and might be confusing if it changes without the config having
 // changed).
-func tailscaledConfigHash(c tailscaleConfigs) (string, error) {
+func tailscaledConfigHash(c tailscaledConfigs) (string, error) {
 	b, err := json.Marshal(c)
 	if err != nil {
 		return "", fmt.Errorf("error marshalling tailscaled configs: %w", err)

cmd/k8s-operator/svc.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
@@ -62,15 +63,17 @@ type ServiceReconciler struct {
 	tsNamespace string
 
 	clock tstime.Clock
+
+	defaultProxyClass string
 }
 
 var (
 	// gaugeEgressProxies tracks the number of egress proxies that we're
 	// currently managing.
-	gaugeEgressProxies = clientmetric.NewGauge("k8s_egress_proxies")
+	gaugeEgressProxies = clientmetric.NewGauge(kubetypes.MetricEgressProxyCount)
 	// gaugeIngressProxies tracks the number of ingress proxies that we're
 	// currently managing.
-	gaugeIngressProxies = clientmetric.NewGauge("k8s_ingress_proxies")
+	gaugeIngressProxies = clientmetric.NewGauge(kubetypes.MetricIngressProxyCount)
 )
 
 func childResourceLabels(name, ns, typ string) map[string]string {
@@ -109,6 +112,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
 
+	if _, ok := svc.Annotations[AnnotationProxyGroup]; ok {
+		return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup
+	}
+
 	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
@@ -208,7 +215,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc)
+	proxyClass := proxyClassForObject(svc, a.defaultProxyClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -325,7 +332,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	if err != nil {
 		msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
 		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
-		return fmt.Errorf(msg)
+		return errors.New(msg)
 	}
 	for _, ip := range tsIPs {
 		addr, err := netip.ParseAddr(ip)
@@ -351,6 +358,10 @@ func validateService(svc *corev1.Service) []string {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
+
+	// TODO(irbekrm): validate that tailscale.com/tailnet-ip annotation is a
+	// valid IP address (tailscale/tailscale#13671).
+
 	svcName := nameForService(svc)
 	if err := dnsname.ValidLabel(svcName); err != nil {
 		if _, ok := svc.Annotations[AnnotationHostname]; ok {
@@ -404,8 +415,14 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-func proxyClassForObject(o client.Object) string {
-	return o.GetLabels()[LabelProxyClass]
+// proxyClassForObject returns the proxy class for the given object. If the
+// object does not have a proxy class label, it returns the default proxy class
+func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+	proxyClass, exists := o.GetLabels()[LabelProxyClass]
+	if !exists {
+		proxyClass = proxyDefaultClass
+	}
+	return proxyClass
 }
 
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {

cmd/k8s-operator/testutils_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/netip"
+	"reflect"
 	"strings"
 	"sync"
 	"testing"
@@ -51,6 +52,9 @@ type configOpts struct {
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
 	proxyClass                                     string // configuration from the named ProxyClass should be applied to proxy resources
+	app                                            string
+	shouldRemoveAuthKey                            bool
+	secretExtraData                                map[string][]byte
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -142,6 +146,10 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
 		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
 	}
+	tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
+		Name:  "TS_INTERNAL_APP",
+		Value: opts.app,
+	})
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -224,6 +232,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
+			{Name: "TS_INTERNAL_APP", Value: opts.app},
 		},
 		ImagePullPolicy: "Always",
 		VolumeMounts: []corev1.VolumeMount{
@@ -358,6 +367,9 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 			conf.AcceptRoutes = "true"
 		}
 	}
+	if opts.shouldRemoveAuthKey {
+		conf.AuthKey = nil
+	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -398,6 +410,9 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		labels["tailscale.com/parent-resource-ns"] = "" // Connector is cluster scoped
 	}
 	s.Labels = labels
+	for key, val := range opts.secretExtraData {
+		mak.Set(&s.Data, key, val)
+	}
 	return s
 }
 
@@ -481,7 +496,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
-		t.Fatalf("unexpected object (-got +want):\n%s", diff)
+		t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
 	}
 }
 
@@ -492,7 +507,7 @@ func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns
 		Name:      name,
 		Namespace: ns,
 	}, obj); !apierrors.IsNotFound(err) {
-		t.Fatalf("object %s/%s unexpectedly present, wanted missing", ns, name)
+		t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
 	}
 }
 
@@ -586,6 +601,17 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili
 	return "secret-authkey", k, nil
 }
 
+func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) {
+	return &tailscale.Device{
+		DeviceID: deviceID,
+		Hostname: "hostname-" + deviceID,
+		Addresses: []string{
+			"1.2.3.4",
+			"::1",
+		},
+	}, nil
+}
+
 func (c *fakeTSClient) DeleteDevice(ctx context.Context, deviceID string) error {
 	c.Lock()
 	defer c.Unlock()
@@ -613,6 +639,14 @@ func removeHashAnnotation(sts *appsv1.StatefulSet) {
 	delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
 }
 
+func removeTargetPortsFromSvc(svc *corev1.Service) {
+	newPorts := make([]corev1.ServicePort, 0)
+	for _, p := range svc.Spec.Ports {
+		newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port})
+	}
+	svc.Spec.Ports = newPorts
+}
+
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()

cmd/k8s-operator/tsrecorder.go

@@ -0,0 +1,397 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"sync"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/tailscale"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonRecorderCreationFailed = "RecorderCreationFailed"
+	reasonRecorderCreated        = "RecorderCreated"
+	reasonRecorderInvalid        = "RecorderInvalid"
+
+	currentProfileKey = "_current-profile"
+)
+
+var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount)
+
+// RecorderReconciler syncs Recorder statefulsets with their definition in
+// Recorder CRs.
+type RecorderReconciler struct {
+	client.Client
+	l           *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+	tsClient    tsClient
+
+	mu        sync.Mutex           // protects following
+	recorders set.Slice[types.UID] // for recorders gauge
+}
+
+func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger {
+	return r.l.With("Recorder", name)
+}
+
+func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
+	logger := r.logger(req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	tsr := new(tsapi.Recorder)
+	err = r.Get(ctx, req.NamespacedName, tsr)
+	if apierrors.IsNotFound(err) {
+		logger.Debugf("Recorder not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err)
+	}
+	if markedForDeletion(tsr) {
+		logger.Debugf("Recorder is being deleted, cleaning up resources")
+		ix := xslices.Index(tsr.Finalizers, FinalizerName)
+		if ix < 0 {
+			logger.Debugf("no finalizer, nothing to do")
+			return reconcile.Result{}, nil
+		}
+
+		if done, err := r.maybeCleanup(ctx, tsr); err != nil {
+			return reconcile.Result{}, err
+		} else if !done {
+			logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
+			return reconcile.Result{RequeueAfter: shortRequeue}, nil
+		}
+
+		tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
+		if err := r.Update(ctx, tsr); err != nil {
+			return reconcile.Result{}, err
+		}
+		return reconcile.Result{}, nil
+	}
+
+	oldTSRStatus := tsr.Status.DeepCopy()
+	setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
+		if !apiequality.Semantic.DeepEqual(oldTSRStatus, tsr.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
+		}
+		return reconcile.Result{}, err
+	}
+
+	if !slices.Contains(tsr.Finalizers, FinalizerName) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to log that the high level, multi-reconcile
+		// operation is underway.
+		logger.Infof("ensuring Recorder is set up")
+		tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
+		if err := r.Update(ctx, tsr); err != nil {
+			logger.Errorf("error adding finalizer: %w", err)
+			return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
+		}
+	}
+
+	if err := r.validate(tsr); err != nil {
+		logger.Errorf("error validating Recorder spec: %w", err)
+		message := fmt.Sprintf("Recorder is invalid: %s", err)
+		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message)
+		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
+	}
+
+	if err = r.maybeProvision(ctx, tsr); err != nil {
+		logger.Errorf("error creating Recorder resources: %w", err)
+		message := fmt.Sprintf("failed creating Recorder: %s", err)
+		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
+		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, message)
+	}
+
+	logger.Info("Recorder resources synced")
+	return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated)
+}
+
+func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error {
+	logger := r.logger(tsr.Name)
+
+	r.mu.Lock()
+	r.recorders.Add(tsr.UID)
+	gaugeRecorderResources.Set(int64(r.recorders.Len()))
+	r.mu.Unlock()
+
+	if err := r.ensureAuthSecretCreated(ctx, tsr); err != nil {
+		return fmt.Errorf("error creating secrets: %w", err)
+	}
+	// State secret is precreated so we can use the Recorder CR as its owner ref.
+	sec := tsrStateSecret(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
+		s.ObjectMeta.Labels = sec.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error creating state Secret: %w", err)
+	}
+	sa := tsrServiceAccount(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
+		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
+	}); err != nil {
+		return fmt.Errorf("error creating ServiceAccount: %w", err)
+	}
+	role := tsrRole(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
+		r.ObjectMeta.Labels = role.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
+		r.Rules = role.Rules
+	}); err != nil {
+		return fmt.Errorf("error creating Role: %w", err)
+	}
+	roleBinding := tsrRoleBinding(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
+		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
+		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
+		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
+		r.RoleRef = roleBinding.RoleRef
+		r.Subjects = roleBinding.Subjects
+	}); err != nil {
+		return fmt.Errorf("error creating RoleBinding: %w", err)
+	}
+	ss := tsrStatefulSet(tsr, r.tsNamespace)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
+		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
+		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
+		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
+		s.Spec = ss.Spec
+	}); err != nil {
+		return fmt.Errorf("error creating StatefulSet: %w", err)
+	}
+
+	var devices []tsapi.RecorderTailnetDevice
+
+	device, ok, err := r.getDeviceInfo(ctx, tsr.Name)
+	if err != nil {
+		return fmt.Errorf("failed to get device info: %w", err)
+	}
+	if !ok {
+		logger.Debugf("no Tailscale hostname known yet, waiting for Recorder pod to finish auth")
+		return nil
+	}
+
+	devices = append(devices, device)
+
+	tsr.Status.Devices = devices
+
+	return nil
+}
+
+// maybeCleanup just deletes the device from the tailnet. All the kubernetes
+// resources linked to a Recorder will get cleaned up via owner references
+// (which we can use because they are all in the same namespace).
+func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
+	logger := r.logger(tsr.Name)
+
+	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
+	if err != nil {
+		return false, err
+	}
+	if !ok {
+		logger.Debugf("state Secret %s-0 not found or does not contain node ID, continuing cleanup", tsr.Name)
+		r.mu.Lock()
+		r.recorders.Remove(tsr.UID)
+		gaugeRecorderResources.Set(int64(r.recorders.Len()))
+		r.mu.Unlock()
+		return true, nil
+	}
+
+	logger.Debugf("deleting device %s from control", string(id))
+	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
+		errResp := &tailscale.ErrResponse{}
+		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
+			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
+		} else {
+			return false, fmt.Errorf("error deleting device: %w", err)
+		}
+	} else {
+		logger.Debugf("device %s deleted from control", string(id))
+	}
+
+	// Unlike most log entries in the reconcile loop, this will get printed
+	// exactly once at the very end of cleanup, because the final step of
+	// cleanup removes the tailscale finalizer, which will make all future
+	// reconciles exit early.
+	logger.Infof("cleaned up Recorder resources")
+	r.mu.Lock()
+	r.recorders.Remove(tsr.UID)
+	gaugeRecorderResources.Set(int64(r.recorders.Len()))
+	r.mu.Unlock()
+	return true, nil
+}
+
+func (r *RecorderReconciler) ensureAuthSecretCreated(ctx context.Context, tsr *tsapi.Recorder) error {
+	logger := r.logger(tsr.Name)
+	key := types.NamespacedName{
+		Namespace: r.tsNamespace,
+		Name:      tsr.Name,
+	}
+	if err := r.Get(ctx, key, &corev1.Secret{}); err == nil {
+		// No updates, already created the auth key.
+		logger.Debugf("auth Secret %s already exists", key.Name)
+		return nil
+	} else if !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// Create the auth key Secret which is going to be used by the StatefulSet
+	// to authenticate with Tailscale.
+	logger.Debugf("creating authkey for new Recorder")
+	tags := tsr.Spec.Tags
+	if len(tags) == 0 {
+		tags = tsapi.Tags{"tag:k8s"}
+	}
+	authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify())
+	if err != nil {
+		return err
+	}
+
+	logger.Debug("creating a new Secret for the Recorder")
+	if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *RecorderReconciler) validate(tsr *tsapi.Recorder) error {
+	if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil {
+		return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible")
+	}
+
+	return nil
+}
+
+func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string) (*corev1.Secret, error) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.tsNamespace,
+			Name:      fmt.Sprintf("%s-0", tsrName),
+		},
+	}
+	if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, nil
+		}
+
+		return nil, fmt.Errorf("error getting state Secret: %w", err)
+	}
+
+	return secret, nil
+}
+
+func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+	secret, err := r.getStateSecret(ctx, tsrName)
+	if err != nil || secret == nil {
+		return "", "", false, err
+	}
+
+	return getNodeMetadata(ctx, secret)
+}
+
+// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// is expected to always be non-empty if the node ID is, but not required.
+func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
+	currentProfile, ok := secret.Data[currentProfileKey]
+	if !ok {
+		return "", "", false, nil
+	}
+	profileBytes, ok := secret.Data[string(currentProfile)]
+	if !ok {
+		return "", "", false, nil
+	}
+	var profile profile
+	if err := json.Unmarshal(profileBytes, &profile); err != nil {
+		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
+	}
+
+	ok = profile.Config.NodeID != ""
+	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
+}
+
+func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
+	secret, err := r.getStateSecret(ctx, tsrName)
+	if err != nil || secret == nil {
+		return tsapi.RecorderTailnetDevice{}, false, err
+	}
+
+	return getDeviceInfo(ctx, r.tsClient, secret)
+}
+
+func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
+	nodeID, dnsName, ok, err := getNodeMetadata(ctx, secret)
+	if !ok || err != nil {
+		return tsapi.RecorderTailnetDevice{}, false, err
+	}
+
+	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
+	// need the API. Should we instead update the profile to include addresses?
+	device, err := tsClient.Device(ctx, string(nodeID), nil)
+	if err != nil {
+		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
+	}
+
+	d = tsapi.RecorderTailnetDevice{
+		Hostname:   device.Hostname,
+		TailnetIPs: device.Addresses,
+	}
+	if dnsName != "" {
+		d.URL = fmt.Sprintf("https://%s", dnsName)
+	}
+
+	return d, true, nil
+}
+
+type profile struct {
+	Config struct {
+		NodeID      string `json:"NodeID"`
+		UserProfile struct {
+			LoginName string `json:"LoginName"`
+		} `json:"UserProfile"`
+	} `json:"Config"`
+}
+
+func markedForDeletion(obj metav1.Object) bool {
+	return !obj.GetDeletionTimestamp().IsZero()
+}

cmd/k8s-operator/tsrecorder_specs.go

@@ -0,0 +1,278 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+	"tailscale.com/version"
+)
+
+func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet {
+	return &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Labels),
+			OwnerReferences: tsrOwnerReference(tsr),
+			Annotations:     tsr.Spec.StatefulSet.Annotations,
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: ptr.To[int32](1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels),
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        tsr.Name,
+					Namespace:   namespace,
+					Labels:      labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels),
+					Annotations: tsr.Spec.StatefulSet.Pod.Annotations,
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: tsr.Name,
+					Affinity:           tsr.Spec.StatefulSet.Pod.Affinity,
+					SecurityContext:    tsr.Spec.StatefulSet.Pod.SecurityContext,
+					ImagePullSecrets:   tsr.Spec.StatefulSet.Pod.ImagePullSecrets,
+					NodeSelector:       tsr.Spec.StatefulSet.Pod.NodeSelector,
+					Tolerations:        tsr.Spec.StatefulSet.Pod.Tolerations,
+					Containers: []corev1.Container{
+						{
+							Name: "recorder",
+							Image: func() string {
+								image := tsr.Spec.StatefulSet.Pod.Container.Image
+								if image == "" {
+									image = fmt.Sprintf("tailscale/tsrecorder:%s", selfVersionImageTag())
+								}
+
+								return image
+							}(),
+							ImagePullPolicy: tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy,
+							Resources:       tsr.Spec.StatefulSet.Pod.Container.Resources,
+							SecurityContext: tsr.Spec.StatefulSet.Pod.Container.SecurityContext,
+							Env:             env(tsr),
+							EnvFrom: func() []corev1.EnvFromSource {
+								if tsr.Spec.Storage.S3 == nil || tsr.Spec.Storage.S3.Credentials.Secret.Name == "" {
+									return nil
+								}
+
+								return []corev1.EnvFromSource{{
+									SecretRef: &corev1.SecretEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: tsr.Spec.Storage.S3.Credentials.Secret.Name,
+										},
+									},
+								}}
+							}(),
+							Command: []string{"/tsrecorder"},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "data",
+									MountPath: "/data",
+									ReadOnly:  false,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: "data",
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func tsrServiceAccount(tsr *tsapi.Recorder, namespace string) *corev1.ServiceAccount {
+	return &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+	}
+}
+
+func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"get",
+					"patch",
+					"update",
+				},
+				ResourceNames: []string{
+					tsr.Name,                      // Contains the auth key.
+					fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
+				},
+			},
+		},
+	}
+}
+
+func tsrRoleBinding(tsr *tsapi.Recorder, namespace string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            tsr.Name,
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      tsr.Name,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: tsr.Name,
+		},
+	}
+}
+
+func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:       namespace,
+			Name:            tsr.Name,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+		StringData: map[string]string{
+			"authkey": authKey,
+		},
+	}
+}
+
+func tsrStateSecret(tsr *tsapi.Recorder, namespace string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            fmt.Sprintf("%s-0", tsr.Name),
+			Namespace:       namespace,
+			Labels:          labels("recorder", tsr.Name, nil),
+			OwnerReferences: tsrOwnerReference(tsr),
+		},
+	}
+}
+
+func env(tsr *tsapi.Recorder) []corev1.EnvVar {
+	envs := []corev1.EnvVar{
+		{
+			Name: "TS_AUTHKEY",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: tsr.Name,
+					},
+					Key: "authkey",
+				},
+			},
+		},
+		{
+			Name: "POD_NAME",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					// Secret is named after the pod.
+					FieldPath: "metadata.name",
+				},
+			},
+		},
+		{
+			Name:  "TS_STATE",
+			Value: "kube:$(POD_NAME)",
+		},
+		{
+			Name:  "TSRECORDER_HOSTNAME",
+			Value: "$(POD_NAME)",
+		},
+	}
+
+	for _, env := range tsr.Spec.StatefulSet.Pod.Container.Env {
+		envs = append(envs, corev1.EnvVar{
+			Name:  string(env.Name),
+			Value: env.Value,
+		})
+	}
+
+	if tsr.Spec.Storage.S3 != nil {
+		envs = append(envs,
+			corev1.EnvVar{
+				Name:  "TSRECORDER_DST",
+				Value: fmt.Sprintf("s3://%s", tsr.Spec.Storage.S3.Endpoint),
+			},
+			corev1.EnvVar{
+				Name:  "TSRECORDER_BUCKET",
+				Value: tsr.Spec.Storage.S3.Bucket,
+			},
+		)
+	} else {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TSRECORDER_DST",
+			Value: "/data/recordings",
+		})
+	}
+
+	if tsr.Spec.EnableUI {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "TSRECORDER_UI",
+			Value: "true",
+		})
+	}
+
+	return envs
+}
+
+func labels(app, instance string, customLabels map[string]string) map[string]string {
+	l := make(map[string]string, len(customLabels)+3)
+	for k, v := range customLabels {
+		l[k] = v
+	}
+
+	// ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
+	l["app.kubernetes.io/name"] = app
+	l["app.kubernetes.io/instance"] = instance
+	l["app.kubernetes.io/managed-by"] = "tailscale-operator"
+
+	return l
+}
+
+func tsrOwnerReference(owner metav1.Object) []metav1.OwnerReference {
+	return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("Recorder"))}
+}
+
+// selfVersionImageTag returns the container image tag of the running operator
+// build.
+func selfVersionImageTag() string {
+	meta := version.GetMeta()
+	var versionPrefix string
+	if meta.UnstableBranch {
+		versionPrefix = "unstable-"
+	}
+	return fmt.Sprintf("%sv%s", versionPrefix, meta.MajorMinorPatch)
+}

cmd/k8s-operator/tsrecorder_specs_test.go

@@ -0,0 +1,143 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+)
+
+func TestRecorderSpecs(t *testing.T) {
+	t.Run("ensure spec fields are passed through correctly", func(t *testing.T) {
+		tsr := &tsapi.Recorder{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test",
+			},
+			Spec: tsapi.RecorderSpec{
+				StatefulSet: tsapi.RecorderStatefulSet{
+					Labels: map[string]string{
+						"ss-label-key": "ss-label-value",
+					},
+					Annotations: map[string]string{
+						"ss-annotation-key": "ss-annotation-value",
+					},
+					Pod: tsapi.RecorderPod{
+						Labels: map[string]string{
+							"pod-label-key": "pod-label-value",
+						},
+						Annotations: map[string]string{
+							"pod-annotation-key": "pod-annotation-value",
+						},
+						Affinity: &corev1.Affinity{
+							PodAffinity: &corev1.PodAffinity{
+								RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{
+									LabelSelector: &metav1.LabelSelector{
+										MatchLabels: map[string]string{
+											"match-label": "match-value",
+										},
+									}},
+								},
+							},
+						},
+						SecurityContext: &corev1.PodSecurityContext{
+							RunAsUser: ptr.To[int64](1000),
+						},
+						ImagePullSecrets: []corev1.LocalObjectReference{{
+							Name: "img-pull",
+						}},
+						NodeSelector: map[string]string{
+							"some-node": "selector",
+						},
+						Tolerations: []corev1.Toleration{{
+							Key:               "key",
+							Value:             "value",
+							TolerationSeconds: ptr.To[int64](60),
+						}},
+						Container: tsapi.RecorderContainer{
+							Env: []tsapi.Env{{
+								Name:  "some_env",
+								Value: "env_value",
+							}},
+							Image:           "custom-image",
+							ImagePullPolicy: corev1.PullAlways,
+							SecurityContext: &corev1.SecurityContext{
+								Capabilities: &corev1.Capabilities{
+									Add: []corev1.Capability{
+										"NET_ADMIN",
+									},
+								},
+							},
+							Resources: corev1.ResourceRequirements{
+								Limits: corev1.ResourceList{
+									corev1.ResourceCPU: resource.MustParse("100m"),
+								},
+								Requests: corev1.ResourceList{
+									corev1.ResourceCPU: resource.MustParse("50m"),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		ss := tsrStatefulSet(tsr, tsNamespace)
+
+		// StatefulSet-level.
+		if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Annotations, tsr.Spec.StatefulSet.Pod.Annotations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+
+		// Pod-level.
+		if diff := cmp.Diff(ss.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.SecurityContext, tsr.Spec.StatefulSet.Pod.SecurityContext); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.ImagePullSecrets, tsr.Spec.StatefulSet.Pod.ImagePullSecrets); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.NodeSelector, tsr.Spec.StatefulSet.Pod.NodeSelector); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Tolerations, tsr.Spec.StatefulSet.Pod.Tolerations); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+
+		// Container-level.
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr)); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].SecurityContext, tsr.Spec.StatefulSet.Pod.Container.SecurityContext); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+		if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" {
+			t.Errorf("(-got +want):\n%s", diff)
+		}
+	})
+}

cmd/k8s-operator/tsrecorder_test.go

@@ -0,0 +1,162 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+)
+
+const tsNamespace = "tailscale"
+
+func TestRecorder(t *testing.T) {
+	tsr := &tsapi.Recorder{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Finalizers: []string{"tailscale.com/finalizer"},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(tsr).
+		WithStatusSubresource(tsr).
+		Build()
+	tsClient := &fakeTSClient{}
+	zl, _ := zap.NewDevelopment()
+	fr := record.NewFakeRecorder(1)
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	reconciler := &RecorderReconciler{
+		tsNamespace: tsNamespace,
+		Client:      fc,
+		tsClient:    tsClient,
+		recorder:    fr,
+		l:           zl.Sugar(),
+		clock:       cl,
+	}
+
+	t.Run("invalid spec gives an error condition", func(t *testing.T) {
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar())
+		expectEqual(t, fc, tsr, nil)
+		if expected := 0; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		expectRecorderResources(t, fc, tsr, false)
+
+		expectedEvent := "Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
+		expectEvents(t, fr, []string{expectedEvent})
+	})
+
+	t.Run("observe Ready=true status condition for a valid spec", func(t *testing.T) {
+		tsr.Spec.EnableUI = true
+		mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
+			t.Spec = tsr.Spec
+		})
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar())
+		expectEqual(t, fc, tsr, nil)
+		if expected := 1; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		expectRecorderResources(t, fc, tsr, true)
+	})
+
+	t.Run("populate node info in state secret, and see it appear in status", func(t *testing.T) {
+		bytes, err := json.Marshal(map[string]any{
+			"Config": map[string]any{
+				"NodeID": "nodeid-123",
+				"UserProfile": map[string]any{
+					"LoginName": "test-0.example.ts.net",
+				},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		const key = "profile-abc"
+		mustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) {
+			s.Data = map[string][]byte{
+				currentProfileKey: []byte(key),
+				key:               bytes,
+			}
+		})
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+		tsr.Status.Devices = []tsapi.RecorderTailnetDevice{
+			{
+				Hostname:   "hostname-nodeid-123",
+				TailnetIPs: []string{"1.2.3.4", "::1"},
+				URL:        "https://test-0.example.ts.net",
+			},
+		}
+		expectEqual(t, fc, tsr, nil)
+	})
+
+	t.Run("delete the Recorder and observe cleanup", func(t *testing.T) {
+		if err := fc.Delete(context.Background(), tsr); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", tsr.Name)
+
+		expectMissing[tsapi.Recorder](t, fc, "", tsr.Name)
+		if expected := 0; reconciler.recorders.Len() != expected {
+			t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
+		}
+		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-123"}); diff != "" {
+			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
+		}
+		// The fake client does not clean up objects whose owner has been
+		// deleted, so we can't test for the owned resources getting deleted.
+	})
+}
+
+func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) {
+	t.Helper()
+
+	auth := tsrAuthSecret(tsr, tsNamespace, "secret-authkey")
+	state := tsrStateSecret(tsr, tsNamespace)
+	role := tsrRole(tsr, tsNamespace)
+	roleBinding := tsrRoleBinding(tsr, tsNamespace)
+	serviceAccount := tsrServiceAccount(tsr, tsNamespace)
+	statefulSet := tsrStatefulSet(tsr, tsNamespace)
+
+	if shouldExist {
+		expectEqual(t, fc, auth, nil)
+		expectEqual(t, fc, state, nil)
+		expectEqual(t, fc, role, nil)
+		expectEqual(t, fc, roleBinding, nil)
+		expectEqual(t, fc, serviceAccount, nil)
+		expectEqual(t, fc, statefulSet, nil)
+	} else {
+		expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name)
+		expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name)
+		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
+		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
+		expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
+		expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
+	}
+}

cmd/natc/natc.go

@@ -456,6 +456,11 @@ func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool {
 }
 
 func proxyTCPConn(c net.Conn, dest string) {
+	if c.RemoteAddr() == nil {
+		log.Printf("proxyTCPConn: nil RemoteAddr")
+		c.Close()
+		return
+	}
 	addrPortStr := c.LocalAddr().String()
 	_, port, err := net.SplitHostPort(addrPortStr)
 	if err != nil {
@@ -489,6 +494,9 @@ type perPeerState struct {
 func (ps *perPeerState) domainForIP(ip netip.Addr) (_ string, ok bool) {
 	ps.mu.Lock()
 	defer ps.mu.Unlock()
+	if ps.addrToDomain == nil {
+		return "", false
+	}
 	return ps.addrToDomain.Lookup(ip)
 }
 

cmd/stund/depaware.txt

@@ -50,6 +50,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/envknob                                        from tailscale.com/tsweb+
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/net/stunserver+
         tailscale.com/net/netaddr                                    from tailscale.com/net/tsaddr
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
@@ -81,15 +82,16 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/version/distro                                 from tailscale.com/envknob
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls
+        golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/net/dns/dnsmessage                              from net
+        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
+        golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
@@ -153,6 +155,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from google.golang.org/protobuf/internal/impl
+        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/tailcfg+
@@ -168,7 +171,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -195,3 +198,4 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
+        unique                                                       from net/netip

cmd/stunstamp/stunstamp.go

@@ -1,13 +1,14 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-// The stunstamp binary measures STUN round-trip latency with DERPs.
+// The stunstamp binary measures round-trip latency with DERPs.
 package main
 
 import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -22,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"runtime"
 	"slices"
 	"strconv"
 	"strings"
@@ -31,8 +33,10 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/prompb"
+	"github.com/tcnksm/go-httpstat"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/stun"
+	"tailscale.com/net/tcpinfo"
 	"tailscale.com/tailcfg"
 )
 
@@ -42,11 +46,23 @@ var (
 	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
 	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
 	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
-	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+	flagSTUNDstPorts   = flag.String("stun-dst-ports", "", "comma-separated list of STUN destination ports to monitor")
+	flagHTTPSDstPorts  = flag.String("https-dst-ports", "", "comma-separated list of HTTPS destination ports to monitor")
+	flagTCPDstPorts    = flag.String("tcp-dst-ports", "", "comma-separated list of TCP destination ports to monitor")
+	flagICMP           = flag.Bool("icmp", false, "probe ICMP")
 )
 
 const (
-	minInterval       = time.Second
+	// maxTxJitter is the upper bounds for jitter introduced across probes
+	maxTXJitter = time.Millisecond * 400
+	// minInterval is the minimum allowed probe interval/step
+	minInterval = time.Second * 10
+	// txRxTimeout is the timeout value used for kernel timestamping loopback,
+	// and packet receive operations
+	txRxTimeout = time.Second * 2
+	// maxBufferDuration is the maximum duration (maxBufferDuration /
+	// *flagInterval steps worth) of buffered data that can be held in memory
+	// before data loss occurs around prometheus unavailability.
 	maxBufferDuration = time.Hour
 )
 
@@ -89,12 +105,22 @@ func (t timestampSource) String() string {
 	}
 }
 
+type protocol string
+
+const (
+	protocolSTUN  protocol = "stun"
+	protocolICMP  protocol = "icmp"
+	protocolHTTPS protocol = "https"
+	protocolTCP   protocol = "tcp"
+)
+
 // resultKey contains the stable dimensions and their values for a given
 // timeseries, i.e. not time and not rtt/timeout.
 type resultKey struct {
 	meta            nodeMeta
 	timestampSource timestampSource
 	connStability   connStability
+	protocol        protocol
 	dstPort         int
 }
 
@@ -104,19 +130,218 @@ type result struct {
 	rtt *time.Duration // nil signifies failure, e.g. timeout
 }
 
-func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+type lportsPool struct {
+	sync.Mutex
+	ports []int
+}
+
+func (l *lportsPool) get() int {
+	l.Lock()
+	defer l.Unlock()
+	ret := l.ports[0]
+	l.ports = append(l.ports[:0], l.ports[1:]...)
+	return ret
+}
+
+func (l *lportsPool) put(i int) {
+	l.Lock()
+	defer l.Unlock()
+	l.ports = append(l.ports, int(i))
+}
+
+var (
+	lports *lportsPool
+)
+
+const (
+	lportPoolSize = 16000
+	lportBase     = 2048
+)
+
+func init() {
+	lports = &lportsPool{
+		ports: make([]int, 0, lportPoolSize),
+	}
+	for i := lportBase; i < lportBase+lportPoolSize; i++ {
+		lports.ports = append(lports.ports, i)
+	}
+}
+
+// lportForTCPConn satisfies io.ReadWriteCloser, but is really just used to pass
+// around a persistent laddr for stableConn purposes. The underlying TCP
+// connection is not created until measurement time as in some cases we need to
+// measure dial time.
+type lportForTCPConn int
+
+func (l *lportForTCPConn) Close() error {
+	if *l == 0 {
+		return nil
+	}
+	lports.put(int(*l))
+	return nil
+}
+
+func (l *lportForTCPConn) Write([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func (l *lportForTCPConn) Read([]byte) (int, error) {
+	return 0, errors.New("unimplemented")
+}
+
+func addrInUse(err error, lport *lportForTCPConn) bool {
+	if errors.Is(err, syscall.EADDRINUSE) {
+		old := int(*lport)
+		// abandon port, don't return it to pool
+		*lport = lportForTCPConn(lports.get()) // get a new port
+		log.Printf("EADDRINUSE: %v old: %d new: %d", err, old, *lport)
+		return true
+	}
+	return false
+}
+
+func tcpDial(ctx context.Context, lport *lportForTCPConn, dst netip.AddrPort) (net.Conn, error) {
+	for {
+		var opErr error
+		dialer := &net.Dialer{
+			LocalAddr: &net.TCPAddr{
+				Port: int(*lport),
+			},
+			Control: func(network, address string, c syscall.RawConn) error {
+				return c.Control(func(fd uintptr) {
+					// we may restart faster than TIME_WAIT can clear
+					opErr = setSOReuseAddr(fd)
+				})
+			},
+		}
+		if opErr != nil {
+			panic(opErr)
+		}
+		tcpConn, err := dialer.DialContext(ctx, "tcp", dst.String())
+		if err != nil {
+			if addrInUse(err, lport) {
+				continue
+			}
+			return nil, err
+		}
+		return tcpConn, nil
+	}
+}
+
+type tempError struct {
+	error
+}
+
+func (t tempError) Temporary() bool {
+	return true
+}
+
+func measureTCPRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	// Set a dial timeout < 1s (TCP_TIMEOUT_INIT on Linux) as a means to avoid
+	// SYN retries, which can contribute to tcpi->rtt below. This simply limits
+	// retries from the initiator, but SYN+ACK on the reverse path can also
+	// time out and be retransmitted.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*750)
+	defer cancel()
+	tcpConn, err := tcpDial(ctx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	// This is an unreliable method to measure TCP RTT. The Linux kernel
+	// describes it as such in tcp_rtt_estimator(). We take some care in how we
+	// hold tcp_info->rtt here, e.g. clamping dial timeout, but if we are to
+	// actually use this elsewhere as an input to some decision it warrants a
+	// deeper study and consideration for alternative methods. Its usefulness
+	// here is as a point of comparison against the other methods.
+	rtt, err = tcpinfo.RTT(tcpConn)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	return rtt, nil
+}
+
+func measureHTTPSRTT(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	lport, ok := conn.(*lportForTCPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	var httpResult httpstat.Result
+	// 5s mirrors net/netcheck.overallProbeTimeout used in net/netcheck.Client.measureHTTPSLatency.
+	reqCtx, cancel := context.WithTimeout(httpstat.WithHTTPStat(context.Background(), &httpResult), time.Second*5)
+	defer cancel()
+	reqURL := "https://" + dst.String() + "/derp/latency-check"
+	req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil)
+	if err != nil {
+		return 0, err
+	}
+	client := &http.Client{}
+	// 1.5s mirrors derp/derphttp.dialnodeTimeout used in derp/derphttp.DialNode().
+	dialCtx, dialCancel := context.WithTimeout(reqCtx, time.Millisecond*1500)
+	defer dialCancel()
+	tcpConn, err := tcpDial(dialCtx, lport, dst)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	defer tcpConn.Close()
+	tlsConn := tls.Client(tcpConn, &tls.Config{
+		ServerName: hostname,
+	})
+	// Mirror client/netcheck behavior, which handshakes before handing the
+	// tlsConn over to the http.Client via http.Transport
+	err = tlsConn.Handshake()
+	if err != nil {
+		return 0, tempError{err}
+	}
+	tlsConnCh := make(chan net.Conn, 1)
+	tlsConnCh <- tlsConn
+	tr := &http.Transport{
+		DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
+			select {
+			case tlsConn := <-tlsConnCh:
+				return tlsConn, nil
+			default:
+				return nil, errors.New("unexpected second call of DialTLSContext")
+			}
+		},
+	}
+	client.Transport = tr
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, tempError{err}
+	}
+	if resp.StatusCode/100 != 2 {
+		return 0, tempError{fmt.Errorf("unexpected status code: %d", resp.StatusCode)}
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(io.Discard, io.LimitReader(resp.Body, 8<<10))
+	if err != nil {
+		return 0, tempError{err}
+	}
+	httpResult.End(time.Now())
+	return httpResult.ServerProcessing, nil
+}
+
+func measureSTUNRTT(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	uconn, ok := conn.(*net.UDPConn)
 	if !ok {
 		return 0, fmt.Errorf("unexpected conn type: %T", conn)
 	}
-	err = uconn.SetReadDeadline(time.Now().Add(time.Second * 2))
+	err = uconn.SetReadDeadline(time.Now().Add(txRxTimeout))
 	if err != nil {
 		return 0, fmt.Errorf("error setting read deadline: %w", err)
 	}
 	txID := stun.NewTxID()
 	req := stun.Request(txID)
 	txAt := time.Now()
-	_, err = uconn.WriteToUDP(req, dst)
+	_, err = uconn.WriteToUDP(req, &net.UDPAddr{
+		IP:   dst.Addr().AsSlice(),
+		Port: int(dst.Port()),
+	})
 	if err != nil {
 		return 0, fmt.Errorf("error writing to udp socket: %w", err)
 	}
@@ -153,29 +378,7 @@ type nodeMeta struct {
 	addr       netip.Addr
 }
 
-type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
-
-// probe measures STUN round trip time for the node described by meta over
-// conn against dstPort. It may return a nil duration and nil error if the
-// STUN request timed out. A non-nil error indicates an unrecoverable or
-// non-temporary error.
-func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
-	ua := &net.UDPAddr{
-		IP:   net.IP(meta.addr.AsSlice()),
-		Port: dstPort,
-	}
-
-	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
-	rtt, err := fn(conn, ua)
-	if err != nil {
-		if isTemporaryOrTimeoutErr(err) {
-			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
-			return nil, nil
-		}
-		return nil, err
-	}
-	return &rtt, nil
-}
+type measureFn func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error)
 
 // nodeMetaFromDERPMap parses the provided DERP map in order to update nodeMeta
 // in the provided nodeMetaByAddr. It returns a slice of nodeMeta containing
@@ -237,43 +440,144 @@ func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]node
 	return stale, nil
 }
 
-func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
-	conns := [2]io.ReadWriteCloser{}
-	byDstPort, ok := stableConns[addr]
-	if ok {
-		conns, ok = byDstPort[dstPort]
-		if ok {
-			return conns, nil
-		}
-	}
-	if supportsKernelTS() {
-		kconn, err := getConnKernelTimestamp()
-		if err != nil {
-			return conns, err
-		}
-		conns[timestampSourceKernel] = kconn
-	}
-	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
-	if err != nil {
-		if supportsKernelTS() {
-			conns[timestampSourceKernel].Close()
-		}
-		return conns, err
-	}
-	conns[timestampSourceUserspace] = uconn
-	if byDstPort == nil {
-		byDstPort = make(map[int][2]io.ReadWriteCloser)
-	}
-	byDstPort[dstPort] = conns
-	stableConns[addr] = byDstPort
-	return conns, nil
+type connAndMeasureFn struct {
+	conn io.ReadWriteCloser
+	fn   measureFn
 }
 
-// probeNodes measures the round-trip time for STUN binding requests against the
-// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
-// UDP sockets that should be recycled across runs. It returns the results or
-// an error if one occurs.
-func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+// newConnAndMeasureFn returns a connAndMeasureFn or an error. It may return
+// nil for both if some combination of the supplied timestampSource, protocol,
+// or connStability is unsupported.
+func newConnAndMeasureFn(forDst netip.Addr, source timestampSource, protocol protocol, stable connStability) (*connAndMeasureFn, error) {
+	info := getProtocolSupportInfo(protocol)
+	if !info.stableConn && bool(stable) {
+		return nil, nil
+	}
+	if !info.userspaceTS && source == timestampSourceUserspace {
+		return nil, nil
+	}
+	if !info.kernelTS && source == timestampSourceKernel {
+		return nil, nil
+	}
+	switch protocol {
+	case protocolSTUN:
+		if source == timestampSourceKernel {
+			conn, err := getUDPConnKernelTimestamp()
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTTKernel,
+			}, nil
+		} else {
+			conn, err := net.ListenUDP("udp", &net.UDPAddr{})
+			if err != nil {
+				return nil, err
+			}
+			return &connAndMeasureFn{
+				conn: conn,
+				fn:   measureSTUNRTT,
+			}, nil
+		}
+	case protocolICMP:
+		conn, err := getICMPConn(forDst, source)
+		if err != nil {
+			return nil, err
+		}
+		return &connAndMeasureFn{
+			conn: conn,
+			fn:   mkICMPMeasureFn(source),
+		}, nil
+	case protocolHTTPS:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureHTTPSRTT,
+		}, nil
+	case protocolTCP:
+		localPort := 0
+		if stable {
+			localPort = lports.get()
+		}
+		conn := lportForTCPConn(localPort)
+		return &connAndMeasureFn{
+			conn: &conn,
+			fn:   measureTCPRTT,
+		}, nil
+	}
+	return nil, errors.New("unknown protocol")
+}
+
+type stableConnKey struct {
+	node     netip.Addr
+	protocol protocol
+	port     int
+}
+
+type protocolSupportInfo struct {
+	kernelTS    bool
+	userspaceTS bool
+	stableConn  bool
+}
+
+func getConns(
+	stableConns map[stableConnKey][2]*connAndMeasureFn,
+	addr netip.Addr,
+	protocol protocol,
+	dstPort int,
+) (stable, unstable [2]*connAndMeasureFn, err error) {
+	key := stableConnKey{addr, protocol, dstPort}
+	defer func() {
+		if err != nil {
+			for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+				c := stable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+				c = unstable[source]
+				if c != nil {
+					c.conn.Close()
+				}
+			}
+		}
+	}()
+
+	var ok bool
+	stable, ok = stableConns[key]
+	if !ok {
+		for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+			var cf *connAndMeasureFn
+			cf, err = newConnAndMeasureFn(addr, source, protocol, stableConn)
+			if err != nil {
+				return
+			}
+			stable[source] = cf
+		}
+		stableConns[key] = stable
+	}
+
+	for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+		var cf *connAndMeasureFn
+		cf, err = newConnAndMeasureFn(addr, source, protocol, unstableConn)
+		if err != nil {
+			return
+		}
+		unstable[source] = cf
+	}
+	return stable, unstable, nil
+}
+
+// probeNodes measures the round-trip time for the protocols and ports described
+// by portsByProtocol against the DERP nodes described by nodeMetaByAddr.
+// stableConns are used to recycle connections across calls to probeNodes.
+// probeNodes is also responsible for trimming stableConns based on node
+// lifetime in nodeMetaByAddr. It returns the results or an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[stableConnKey][2]*connAndMeasureFn, portsByProtocol map[protocol][]int) ([]result, error) {
 	wg := sync.WaitGroup{}
 	results := make([]result, 0)
 	resultsCh := make(chan result)
@@ -283,49 +587,36 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 	at := time.Now()
 	addrsToProbe := make(map[netip.Addr]bool)
 
-	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+	doProbe := func(cf *connAndMeasureFn, meta nodeMeta, source timestampSource, stable connStability, protocol protocol, dstPort int) {
 		defer wg.Done()
 		r := result{
 			key: resultKey{
 				meta:            meta,
 				timestampSource: source,
+				connStability:   stable,
 				dstPort:         dstPort,
+				protocol:        protocol,
 			},
 			at: at,
 		}
-		if conn == nil {
-			var err error
-			if source == timestampSourceKernel {
-				conn, err = getConnKernelTimestamp()
+		time.Sleep(rand.N(maxTXJitter)) // jitter across tx
+		addrPort := netip.AddrPortFrom(meta.addr, uint16(dstPort))
+		rtt, err := cf.fn(cf.conn, meta.hostname, addrPort)
+		if err != nil {
+			if isTemporaryOrTimeoutErr(err) {
+				r.rtt = nil
+				log.Printf("%s: temp error measuring RTT to %s(%s): %v", protocol, meta.hostname, addrPort, err)
 			} else {
-				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
-			}
-			if err != nil {
 				select {
 				case <-doneCh:
 					return
-				case errCh <- err:
+				case errCh <- fmt.Errorf("%s: %v", protocol, err):
 					return
 				}
 			}
-			defer conn.Close()
 		} else {
-			r.key.connStability = stableConn
+			r.rtt = &rtt
 		}
-		fn := measureRTT
-		if source == timestampSourceKernel {
-			fn = measureRTTKernel
-		}
-		rtt, err := probe(meta, conn, fn, dstPort)
-		if err != nil {
-			select {
-			case <-doneCh:
-				return
-			case errCh <- err:
-				return
-			}
-		}
-		r.rtt = rtt
 		select {
 		case <-doneCh:
 		case resultsCh <- r:
@@ -334,37 +625,42 @@ func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Ad
 
 	for _, meta := range nodeMetaByAddr {
 		addrsToProbe[meta.addr] = true
-		for _, port := range dstPorts {
-			stable, err := getStableConns(stableConns, meta.addr, port)
-			if err != nil {
-				close(doneCh)
-				wg.Wait()
-				return nil, err
-			}
+		for p, ports := range portsByProtocol {
+			for _, port := range ports {
+				stable, unstable, err := getConns(stableConns, meta.addr, p, port)
+				if err != nil {
+					close(doneCh)
+					wg.Wait()
+					return nil, err
+				}
 
-			wg.Add(2)
-			numProbes += 2
-			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
-			go doProbe(nil, meta, timestampSourceUserspace, port)
-			if supportsKernelTS() {
-				wg.Add(2)
-				numProbes += 2
-				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
-				go doProbe(nil, meta, timestampSourceKernel, port)
+				for i, cf := range stable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), stableConn, p, port)
+					}
+				}
+
+				for i, cf := range unstable {
+					if cf != nil {
+						wg.Add(1)
+						numProbes++
+						go doProbe(cf, meta, timestampSource(i), unstableConn, p, port)
+					}
+				}
 			}
 		}
 	}
 
 	// cleanup conns we no longer need
-	for k, byDstPort := range stableConns {
-		if !addrsToProbe[k] {
-			for _, conns := range byDstPort {
-				if conns[timestampSourceKernel] != nil {
-					conns[timestampSourceKernel].Close()
-				}
-				conns[timestampSourceUserspace].Close()
-				delete(stableConns, k)
+	for k, cf := range stableConns {
+		if !addrsToProbe[k.node] {
+			if cf[timestampSourceKernel] != nil {
+				cf[timestampSourceKernel].conn.Close()
 			}
+			cf[timestampSourceUserspace].conn.Close()
+			delete(stableConns, k)
 		}
 	}
 
@@ -391,11 +687,11 @@ const (
 )
 
 const (
-	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
-	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+	rttMetricName      = "stunstamp_derp_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_timeouts_total"
 )
 
-func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, protocol protocol, dstPort int) []prompb.Label {
 	addressFamily := "ipv4"
 	if meta.addr.Is6() {
 		addressFamily = "ipv6"
@@ -425,6 +721,10 @@ func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source
 		Name:  "hostname",
 		Value: meta.hostname,
 	})
+	labels = append(labels, prompb.Label{
+		Name:  "protocol",
+		Value: string(protocol),
+	})
 	labels = append(labels, prompb.Label{
 		Name:  "dst_port",
 		Value: strconv.Itoa(dstPort),
@@ -453,53 +753,35 @@ const (
 	staleNaN uint64 = 0x7ff0000000000002
 )
 
-func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, portsByProtocol map[protocol][]int) []prompb.TimeSeries {
 	staleMarkers := make([]prompb.TimeSeries, 0)
 	now := time.Now()
-	for _, s := range stale {
-		for _, dstPort := range dstPorts {
-			samples := []prompb.Sample{
-				{
-					Timestamp: now.UnixMilli(),
-					Value:     math.Float64frombits(staleNaN),
-				},
-			}
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
-				Samples: samples,
-			})
-			staleMarkers = append(staleMarkers, prompb.TimeSeries{
-				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
-				Samples: samples,
-			})
-			if supportsKernelTS() {
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
-					Samples: samples,
-				})
-				staleMarkers = append(staleMarkers, prompb.TimeSeries{
-					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
-					Samples: samples,
-				})
+
+	for p, ports := range portsByProtocol {
+		for _, port := range ports {
+			for _, s := range stale {
+				samples := []prompb.Sample{
+					{
+						Timestamp: now.UnixMilli(),
+						Value:     math.Float64frombits(staleNaN),
+					},
+				}
+				// We send stale markers for all combinations in the interest
+				// of simplicity.
+				for _, name := range []string{rttMetricName, timeoutsMetricName} {
+					for _, source := range []timestampSource{timestampSourceUserspace, timestampSourceKernel} {
+						for _, stable := range []connStability{unstableConn, stableConn} {
+							staleMarkers = append(staleMarkers, prompb.TimeSeries{
+								Labels:  timeSeriesLabels(name, s, instance, source, stable, p, port),
+								Samples: samples,
+							})
+						}
+					}
+				}
 			}
 		}
 	}
+
 	return staleMarkers
 }
 
@@ -513,7 +795,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 	for _, r := range results {
 		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
 		seenKeys[r.key] = true
-		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		rttSamples := make([]prompb.Sample, 1)
 		rttSamples[0].Timestamp = r.at.UnixMilli()
 		if r.rtt != nil {
@@ -528,7 +810,7 @@ func resultsToPromTimeSeries(results []result, instance string, timeouts map[res
 		}
 		all = append(all, rttTS)
 		timeouts[r.key] = timeoutsCount
-		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.protocol, r.key.dstPort)
 		timeoutsSamples := make([]prompb.Sample, 1)
 		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
 		timeoutsSamples[0].Value = float64(timeoutsCount)
@@ -620,22 +902,59 @@ func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSer
 	}
 }
 
-func main() {
-	flag.Parse()
-	if len(*flagDstPorts) == 0 {
-		log.Fatal("dst-ports flag is unset")
+func getPortsFromFlag(f string) ([]int, error) {
+	if len(f) == 0 {
+		return nil, nil
 	}
-	dstPortsSplit := strings.Split(*flagDstPorts, ",")
-	slices.Sort(dstPortsSplit)
-	dstPortsSplit = slices.Compact(dstPortsSplit)
-	dstPorts := make([]int, 0, len(dstPortsSplit))
-	for _, d := range dstPortsSplit {
-		i, err := strconv.ParseUint(d, 10, 16)
+	split := strings.Split(f, ",")
+	slices.Sort(split)
+	split = slices.Compact(split)
+	ports := make([]int, 0)
+	for _, portStr := range split {
+		port, err := strconv.ParseUint(portStr, 10, 16)
 		if err != nil {
-			log.Fatal("invalid dst-ports")
+			return nil, err
 		}
-		dstPorts = append(dstPorts, int(i))
+		ports = append(ports, int(port))
 	}
+	return ports, nil
+}
+
+func main() {
+	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
+		log.Fatal("unsupported platform")
+	}
+	flag.Parse()
+
+	portsByProtocol := make(map[protocol][]int)
+	stunPorts, err := getPortsFromFlag(*flagSTUNDstPorts)
+	if err != nil {
+		log.Fatalf("invalid stun-dst-ports flag value: %v", err)
+	}
+	if len(stunPorts) > 0 {
+		portsByProtocol[protocolSTUN] = stunPorts
+	}
+	httpsPorts, err := getPortsFromFlag(*flagHTTPSDstPorts)
+	if err != nil {
+		log.Fatalf("invalid https-dst-ports flag value: %v", err)
+	}
+	if len(httpsPorts) > 0 {
+		portsByProtocol[protocolHTTPS] = httpsPorts
+	}
+	tcpPorts, err := getPortsFromFlag(*flagTCPDstPorts)
+	if err != nil {
+		log.Fatalf("invalid tcp-dst-ports flag value: %v", err)
+	}
+	if len(tcpPorts) > 0 {
+		portsByProtocol[protocolTCP] = tcpPorts
+	}
+	if *flagICMP {
+		portsByProtocol[protocolICMP] = []int{0}
+	}
+	if len(portsByProtocol) == 0 {
+		log.Fatal("nothing to probe")
+	}
+
 	if len(*flagDERPMap) < 1 {
 		log.Fatal("derp-map flag is unset")
 	}
@@ -645,7 +964,7 @@ func main() {
 	if len(*flagRemoteWriteURL) < 1 {
 		log.Fatal("rw-url flag is unset")
 	}
-	_, err := url.Parse(*flagRemoteWriteURL)
+	_, err = url.Parse(*flagRemoteWriteURL)
 	if err != nil {
 		log.Fatalf("invalid rw-url flag value: %v", err)
 	}
@@ -707,7 +1026,7 @@ func main() {
 		for _, v := range nodeMetaByAddr {
 			staleMeta = append(staleMeta, v)
 		}
-		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 		if len(staleMarkers) > 0 {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 			rwc.write(ctx, staleMarkers)
@@ -723,8 +1042,8 @@ func main() {
 	// in a higher probability of the packets traversing the same underlay path.
 	// Comparison of stable and unstable 5-tuple results can shed light on
 	// differences between paths where hashing (multipathing/load balancing)
-	// comes into play.
-	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+	// comes into play. The inner 2 element array index is timestampSource.
+	stableConns := make(map[stableConnKey][2]*connAndMeasureFn)
 
 	// timeouts holds counts of timeout events. Values are persisted for the
 	// lifetime of the related node in the DERP map.
@@ -738,7 +1057,7 @@ func main() {
 	for {
 		select {
 		case <-probeTicker.C:
-			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			results, err := probeNodes(nodeMetaByAddr, stableConns, portsByProtocol)
 			if err != nil {
 				log.Printf("unrecoverable error while probing: %v", err)
 				shutdown()
@@ -761,7 +1080,7 @@ func main() {
 				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
 				continue
 			}
-			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, portsByProtocol)
 			if len(staleMarkers) < 1 {
 				continue
 			}
@@ -780,7 +1099,7 @@ func main() {
 				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 				defer cancel()
 				updatedDM, err := getDERPMap(ctx, *flagDERPMap)
-				if err != nil {
+				if err == nil {
 					dmCh <- updatedDM
 				}
 			}()

cmd/stunstamp/stunstamp_default.go

@@ -8,18 +8,58 @@ package main
 import (
 	"errors"
 	"io"
-	"net"
+	"net/netip"
 	"time"
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	return nil, errors.New("unimplemented")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	return 0, errors.New("unimplemented")
 }
 
-func supportsKernelTS() bool {
-	return false
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	case protocolICMP:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: false,
+			stableConn:  false,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func getICMPConn(forDst netip.Addr, source timestampSource) (io.ReadWriteCloser, error) {
+	return nil, errors.New("platform unsupported")
+}
+
+func mkICMPMeasureFn(source timestampSource) measureFn {
+	return func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+		return 0, errors.New("platform unsupported")
+	}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	return nil
 }

cmd/stunstamp/stunstamp_linux.go

@@ -10,21 +10,27 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
+	"math"
+	"math/rand/v2"
+	"net/netip"
+	"syscall"
 	"time"
 
 	"github.com/mdlayher/socket"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/stun"
 )
 
 const (
-	flags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
+	timestampingFlags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
 		unix.SOF_TIMESTAMPING_RX_SOFTWARE | // rx timestamp generation in the kernel
 		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
 )
 
-func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+func getUDPConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
 	if err != nil {
 		return nil, err
@@ -34,7 +40,7 @@ func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, flags)
+	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, timestampingFlags)
 	if err != nil {
 		return nil, err
 	}
@@ -56,24 +62,144 @@ func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
 	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
 }
 
-func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+func mkICMPMeasureFn(source timestampSource) measureFn {
+	return func(conn io.ReadWriteCloser, hostname string, dst netip.AddrPort) (rtt time.Duration, err error) {
+		return measureICMPRTT(source, conn, hostname, dst)
+	}
+}
+
+func measureICMPRTT(source timestampSource, conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
+	sconn, ok := conn.(*socket.Conn)
+	if !ok {
+		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
+	}
+	txBody := &icmp.Echo{
+		// The kernel overrides this and routes appropriately so there is no
+		// point in setting or verifying.
+		ID: 0,
+		// Make this sufficiently random so that we do not account a late
+		// arriving reply in a future probe window.
+		Seq: int(rand.Int32N(math.MaxUint16)),
+		// Fingerprint ourselves.
+		Data: []byte("stunstamp"),
+	}
+	txMsg := icmp.Message{
+		Body: txBody,
+	}
+	var to unix.Sockaddr
+	if dst.Addr().Is4() {
+		txMsg.Type = ipv4.ICMPTypeEcho
+		to = &unix.SockaddrInet4{}
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
+	} else {
+		txMsg.Type = ipv6.ICMPTypeEchoRequest
+		to = &unix.SockaddrInet6{}
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
+	}
+	txBuf, err := txMsg.Marshal(nil)
+	if err != nil {
+		return 0, err
+	}
+	txAt := time.Now()
+	err = sconn.Sendto(context.Background(), txBuf, 0, to)
+	if err != nil {
+		return 0, fmt.Errorf("sendto error: %v", err)
+	}
+
+	if source == timestampSourceKernel {
+		txCtx, txCancel := context.WithTimeout(context.Background(), txRxTimeout)
+		defer txCancel()
+
+		buf := make([]byte, 1024)
+		oob := make([]byte, 1024)
+
+		for {
+			n, oobn, _, _, err := sconn.Recvmsg(txCtx, buf, oob, unix.MSG_ERRQUEUE)
+			if err != nil {
+				return 0, fmt.Errorf("recvmsg (MSG_ERRQUEUE) error: %v", err) // don't wrap
+			}
+
+			buf = buf[:n]
+			// Spin until we find the message we sent. We get the full packet
+			// looped including eth header so match against the tail.
+			if n < len(txBuf) {
+				continue
+			}
+			txLoopedMsg, err := icmp.ParseMessage(txMsg.Type.Protocol(), buf[len(buf)-len(txBuf):])
+			if err != nil {
+				continue
+			}
+			txLoopedBody, ok := txLoopedMsg.Body.(*icmp.Echo)
+			if !ok || txLoopedBody.Seq != txBody.Seq || txLoopedMsg.Code != txMsg.Code ||
+				txLoopedMsg.Type != txLoopedMsg.Type || !bytes.Equal(txLoopedBody.Data, txBody.Data) {
+				continue
+			}
+			txAt, err = parseTimestampFromCmsgs(oob[:oobn])
+			if err != nil {
+				return 0, fmt.Errorf("failed to get tx timestamp: %v", err) // don't wrap
+			}
+			break
+		}
+	}
+
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), txRxTimeout)
+	defer rxCancel()
+
+	rxBuf := make([]byte, 1024)
+	oob := make([]byte, 1024)
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(rxCtx, rxBuf, oob, 0)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg error: %w", err)
+		}
+		rxAt := time.Now()
+		rxMsg, err := icmp.ParseMessage(txMsg.Type.Protocol(), rxBuf[:n])
+		if err != nil {
+			continue
+		}
+		if txMsg.Type == ipv4.ICMPTypeEcho {
+			if rxMsg.Type != ipv4.ICMPTypeEchoReply {
+				continue
+			}
+		} else {
+			if rxMsg.Type != ipv6.ICMPTypeEchoReply {
+				continue
+			}
+		}
+		if rxMsg.Code != txMsg.Code {
+			continue
+		}
+		rxBody, ok := rxMsg.Body.(*icmp.Echo)
+		if !ok || rxBody.Seq != txBody.Seq || !bytes.Equal(rxBody.Data, txBody.Data) {
+			continue
+		}
+		if source == timestampSourceKernel {
+			rxAt, err = parseTimestampFromCmsgs(oob[:oobn])
+			if err != nil {
+				return 0, fmt.Errorf("failed to get rx timestamp: %v", err)
+			}
+		}
+		return rxAt.Sub(txAt), nil
+	}
+}
+
+func measureSTUNRTTKernel(conn io.ReadWriteCloser, _ string, dst netip.AddrPort) (rtt time.Duration, err error) {
 	sconn, ok := conn.(*socket.Conn)
 	if !ok {
 		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
 	}
 
 	var to unix.Sockaddr
-	to4 := dst.IP.To4()
-	if to4 != nil {
+	if dst.Addr().Is4() {
 		to = &unix.SockaddrInet4{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+		copy(to.(*unix.SockaddrInet4).Addr[:], dst.Addr().AsSlice())
 	} else {
 		to = &unix.SockaddrInet6{
-			Port: dst.Port,
+			Port: int(dst.Port()),
 		}
-		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.Addr().AsSlice())
 	}
 
 	txID := stun.NewTxID()
@@ -84,7 +210,7 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 		return 0, fmt.Errorf("sendto error: %v", err) // don't wrap
 	}
 
-	txCtx, txCancel := context.WithTimeout(context.Background(), time.Second*2)
+	txCtx, txCancel := context.WithTimeout(context.Background(), txRxTimeout)
 	defer txCancel()
 
 	buf := make([]byte, 1024)
@@ -110,7 +236,7 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 		break
 	}
 
-	rxCtx, rxCancel := context.WithTimeout(context.Background(), time.Second*2)
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), txRxTimeout)
 	defer rxCancel()
 
 	for {
@@ -138,6 +264,54 @@ func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Durat
 
 }
 
-func supportsKernelTS() bool {
-	return true
+func getICMPConn(forDst netip.Addr, source timestampSource) (io.ReadWriteCloser, error) {
+	domain := unix.AF_INET
+	proto := unix.IPPROTO_ICMP
+	if forDst.Is6() {
+		domain = unix.AF_INET6
+		proto = unix.IPPROTO_ICMPV6
+	}
+	conn, err := socket.Socket(domain, unix.SOCK_DGRAM, proto, "icmp", nil)
+	if err != nil {
+		return nil, err
+	}
+	if source == timestampSourceKernel {
+		err = conn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, timestampingFlags)
+	}
+	return conn, err
+}
+
+func getProtocolSupportInfo(p protocol) protocolSupportInfo {
+	switch p {
+	case protocolSTUN:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolHTTPS:
+		return protocolSupportInfo{
+			kernelTS:    false,
+			userspaceTS: true,
+			stableConn:  true,
+		}
+	case protocolTCP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: false,
+			stableConn:  true,
+		}
+	case protocolICMP:
+		return protocolSupportInfo{
+			kernelTS:    true,
+			userspaceTS: true,
+			stableConn:  false,
+		}
+	}
+	return protocolSupportInfo{}
+}
+
+func setSOReuseAddr(fd uintptr) error {
+	// we may restart faster than TIME_WAIT can clear
+	return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
 }