util/set: add SetOfFunc

Fixes #12901

Signed-off-by: Marwan Sulaiman <marwan@tailscale.com>

.github/workflows/checklocks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@v4
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/checkout@v4
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: "actions/checkout@v4"
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,18 +23,18 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: false
 
       - name: golangci-lint
-        # Note: this is the 'v6.1.0' tag as of 2024-08-21
-        uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86
+        # Note: this is the 'v3' tag as of 2023-08-14
+        uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299
         with:
-          version: v1.60
+          version: v1.56
 
           # Show only new issues if it's a pull request.
           only-new-issues: true

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -24,7 +24,7 @@ jobs:
 
       - name: Post to slack
         if: failure() && github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@v1.24.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
         with:

.github/workflows/installer.yml

@@ -98,7 +98,7 @@ jobs:
       # We cannot use v4, as it requires a newer glibc version than some of the
       # tested images provide. See
       # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      uses: actions/checkout@v3
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -78,9 +78,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -150,16 +150,16 @@ jobs:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
 
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@v4
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -190,7 +190,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +202,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +214,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +258,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -295,7 +295,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -317,9 +317,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -350,7 +350,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -365,9 +365,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -399,7 +399,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -456,22 +456,18 @@ jobs:
         fuzz-seconds: 300
         dry-run: false
         language: go
-    - name: Set artifacts_path in env (workaround for actions/upload-artifact#176)
-      if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
-      run: |
-        echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      uses: actions/upload-artifact@v3
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
-        path: ${{ env.artifacts_path }}/out/artifacts
+        path: ./out/artifacts
 
   depaware:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
@@ -481,7 +477,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -494,7 +490,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -506,7 +502,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -522,7 +518,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck
@@ -563,7 +559,7 @@ jobs:
       # By having the job always run, but skipping its only step as needed, we
       # let the CI output collapse nicely in PRs.
       if: failure() && github.event_name == 'push'
-      uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+      uses: ruby/action-slack@v3.2.1
       with:
         payload: |
           {
@@ -578,7 +574,6 @@ jobs:
           }
       env:
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   check_mergeability:
     if: always()
@@ -601,6 +596,6 @@ jobs:
     steps:
     - name: Decide if change is okay to merge
       if: github.event_name != 'push'
-      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
+      uses: re-actors/alls-green@release/v1
       with:
         jobs: ${{ toJSON(needs) }}

.github/workflows/update-flake.yml

@@ -21,22 +21,21 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Run update-flakes
         run: ./update-flake.sh
 
       - name: Get access token
-        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
         id: generate-token
         with:
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_retrieval_mode: "id"
-          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Run go get
         run: |
@@ -23,19 +23,18 @@ jobs:
           ./tool/go mod tidy
 
       - name: Get access token
-        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
         id: generate-token
         with:
           # TODO(will): this should use the code updater app rather than licensing.
           # It has the same permissions, so not a big deal, but still.
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_retrieval_mode: "id"
-          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 #v5.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

.gitignore

@@ -43,9 +43,3 @@ client/web/build/assets
 
 /gocross
 /dist
-
-# Ignore xcode userstate and workspace data
-*.xcuserstate
-*.xcworkspacedata
-/tstest/tailmac/bin
-/tstest/tailmac/build

Dockerfile

@@ -1,13 +1,6 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
-# Note that this Dockerfile is currently NOT used to build any of the published
-# Tailscale container images and may have drifted from the image build mechanism
-# we use.
-# Tailscale images are currently built using https://github.com/tailscale/mkctr,
-# and the build script can be found in ./build_docker.sh.
-#
-#
 # This Dockerfile includes all the tailscale binaries.
 #
 # To build the Dockerfile:
@@ -27,7 +20,7 @@
 #     $ docker exec tailscaled tailscale status
 
 
-FROM golang:1.23-alpine AS build-env
+FROM golang:1.22-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
@@ -42,7 +35,7 @@ RUN go install \
     gvisor.dev/gvisor/pkg/tcpip/stack \
     golang.org/x/crypto/ssh \
     golang.org/x/crypto/acme \
-    github.com/coder/websocket \
+    nhooyr.io/websocket \
     github.com/mdlayher/netlink
 
 COPY . .

Makefile

@@ -117,8 +117,7 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
-	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
 
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"

README.md

@@ -37,7 +37,7 @@ not open source.
 
 ## Building
 
-We always require the latest Go release, currently Go 1.23. (While we build
+We always require the latest Go release, currently Go 1.22. (While we build
 releases with our [Go fork](https://github.com/tailscale/go/), its use is not
 required.)
 

VERSION.txt

@@ -1 +1 @@
-1.77.0
+1.71.0

api.md

@@ -1,2 +1,104 @@
 > [!IMPORTANT]
 > The Tailscale API documentation has moved to https://tailscale.com/api
+
+# Tailscale API
+
+The Tailscale API documentation is located in **[tailscale/publicapi](./publicapi/readme.md#tailscale-api)**.
+
+# APIs
+
+**[Overview](./publicapi/readme.md)**
+
+**[Device](./publicapi/device.md#device)**
+
+<a href="device-delete"></a>
+<a href="expire-device-key"></a>
+<a href="device-routes-get">
+<a href="device-routes-post"></a>
+<a href="#device-authorized-post"></a>
+<a href="device-tags-post"></a>
+<a href="device-key-post"></a>
+<a href="tailnet-acl-get"></a>
+
+- Get a device: [`GET /api/v2/device/{deviceid}`](./publicapi/device.md#get-device)
+- Delete a device: [`DELETE /api/v2/device/{deviceID}`](./publicapi/device.md#delete-device)
+- Expire device key: [`POST /api/v2/device/{deviceID}/expire`](./publicapi/device.md#expire-device-key)
+- [**Routes**](./publicapi/device.md#routes)
+  - Get device routes: [`GET /api/v2/device/{deviceID}/routes`](./publicapi/device.md#get-device-routes)
+  - Set device routes: [`POST /api/v2/device/{deviceID}/routes`](./publicapi/device.md#set-device-routes)
+- [**Authorize**](./publicapi/device.md#authorize)
+  - Authorize a device: [`POST /api/v2/device/{deviceID}/authorized`](./publicapi/device.md#authorize-device)
+- [**Tags**](./publicapi/device.md#tags)
+  - Update tags: [`POST /api/v2/device/{deviceID}/tags`](./publicapi/device.md#update-device-tags)
+- [**Keys**](./publicapi/device.md#keys)
+  - Update device key: [`POST /api/v2/device/{deviceID}/key`](./publicapi/device.md#update-device-key)
+- [**IP Addresses**](./publicapi/device.md#ip-addresses)
+  - Set device IPv4 address: [`POST /api/v2/device/{deviceID}/ip`](./publicapi/device.md#set-device-ipv4-address)
+- [**Device posture attributes**](./publicapi/device.md#device-posture-attributes)
+  - Get device posture attributes: [`GET /api/v2/device/{deviceID}/attributes`](./publicapi/device.md#get-device-posture-attributes)
+  - Set custom device posture attributes: [`POST /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#set-device-posture-attributes)
+  - Delete custom device posture attributes: [`DELETE /api/v2/device/{deviceID}/attributes/{attributeKey}`](./publicapi/device.md#delete-custom-device-posture-attributes)
+- [**Device invites**](./publicapi/device.md#invites-to-a-device)
+  - List device invites: [`GET /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#list-device-invites)
+  - Create device invites: [`POST /api/v2/device/{deviceID}/device-invites`](./publicapi/device.md#create-device-invites)
+
+**[Tailnet](./publicapi/tailnet.md#tailnet)**
+
+<a href="tailnet-acl-post"></a>
+<a href="tailnet-acl-preview-post"></a>
+<a href="tailnet-acl-validate-post"></a>
+<a href="tailnet-devices"></a>
+<a href="tailnet-keys-get"></a>
+<a href="tailnet-keys-post"></a>
+<a href="tailnet-keys-key-get"></a>
+<a href="tailnet-keys-key-delete"></a>
+<a href="tailnet-dns"></a>
+<a href="tailnet-dns-nameservers-get"></a>
+<a href="tailnet-dns-nameservers-post"></a>
+<a href="tailnet-dns-preferences-get"></a>
+<a href="tailnet-dns-preferences-post"></a>
+<a href="tailnet-dns-searchpaths-get"></a>
+<a href="tailnet-dns-searchpaths-post"></a>
+
+- [**Policy File**](./publicapi/tailnet.md#policy-file)
+  - Get policy file: [`GET /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#get-policy-file)
+  - Update policy file: [`POST /api/v2/tailnet/{tailnet}/acl`](./publicapi/tailnet.md#update-policy-file)
+  - Preview rule matches: [`POST /api/v2/tailnet/{tailnet}/acl/preview`](./publicapi/tailnet.md#preview-policy-file-rule-matches)
+  - Validate and test policy file: [`POST /api/v2/tailnet/{tailnet}/acl/validate`](./publicapi/tailnet.md#validate-and-test-policy-file)
+- [**Devices**](./publicapi/tailnet.md#devices)
+  - List tailnet devices: [`GET /api/v2/tailnet/{tailnet}/devices`](./publicapi/tailnet.md#list-tailnet-devices)
+- [**Keys**](./publicapi/tailnet.md#tailnet-keys)
+  - List tailnet keys: [`GET /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#list-tailnet-keys)
+  - Create an auth key: [`POST /api/v2/tailnet/{tailnet}/keys`](./publicapi/tailnet.md#create-auth-key)
+  - Get a key: [`GET /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#get-key)
+  - Delete a key: [`DELETE /api/v2/tailnet/{tailnet}/keys/{keyid}`](./publicapi/tailnet.md#delete-key)
+- [**DNS**](./publicapi/tailnet.md#dns)
+  - [**Nameservers**](./publicapi/tailnet.md#nameservers)
+    - Get nameservers: [`GET /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#get-nameservers)
+    - Set nameservers: [`POST /api/v2/tailnet/{tailnet}/dns/nameservers`](./publicapi/tailnet.md#set-nameservers)
+  - [**Preferences**](./publicapi/tailnet.md#preferences)
+    - Get DNS preferences: [`GET /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#get-dns-preferences)
+    - Set DNS preferences: [`POST /api/v2/tailnet/{tailnet}/dns/preferences`](./publicapi/tailnet.md#set-dns-preferences)
+  - [**Search Paths**](./publicapi/tailnet.md#search-paths)
+    - Get search paths: [`GET /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#get-search-paths)
+    - Set search paths: [`POST /api/v2/tailnet/{tailnet}/dns/searchpaths`](./publicapi/tailnet.md#set-search-paths)
+  - [**Split DNS**](./publicapi/tailnet.md#split-dns)
+    - Get split DNS: [`GET /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#get-split-dns)
+    - Update split DNS: [`PATCH /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#update-split-dns)
+    - Set split DNS: [`PUT /api/v2/tailnet/{tailnet}/dns/split-dns`](./publicapi/tailnet.md#set-split-dns)
+- [**User invites**](./publicapi/tailnet.md#tailnet-user-invites)
+  - List user invites: [`GET /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#list-user-invites)
+  - Create user invites: [`POST /api/v2/tailnet/{tailnet}/user-invites`](./publicapi/tailnet.md#create-user-invites)
+
+**[User invites](./publicapi/userinvites.md#user-invites)**
+
+- Get user invite: [`GET /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#get-user-invite)
+- Delete user invite: [`DELETE /api/v2/user-invites/{userInviteId}`](./publicapi/userinvites.md#delete-user-invite)
+- Resend user invite (by email): [`POST /api/v2/user-invites/{userInviteId}/resend`](#resend-user-invite)
+
+**[Device invites](./publicapi/deviceinvites.md#device-invites)**
+
+- Get device invite: [`GET /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#get-device-invite)
+- Delete device invite: [`DELETE /api/v2/device-invites/{deviceInviteId}`](./publicapi/deviceinvites.md#delete-device-invite)
+- Resend device invite (by email): [`POST /api/v2/device-invites/{deviceInviteId}/resend`](./publicapi/deviceinvites.md#resend-device-invite)
+- Accept device invite [`POST /api/v2/device-invites/-/accept`](#accept-device-invite)

assert_ts_toolchain_match.go

@@ -1,27 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build tailscale_go
-
-package tailscaleroot
-
-import (
-	"fmt"
-	"os"
-	"strings"
-)
-
-func init() {
-	tsRev, ok := tailscaleToolchainRev()
-	if !ok {
-		panic("binary built with tailscale_go build tag but failed to read build info or find tailscale.toolchain.rev in build info")
-	}
-	want := strings.TrimSpace(GoToolchainRev)
-	if tsRev != want {
-		if os.Getenv("TS_PERMIT_TOOLCHAIN_MISMATCH") == "1" {
-			fmt.Fprintf(os.Stderr, "tailscale.toolchain.rev = %q, want %q; but ignoring due to TS_PERMIT_TOOLCHAIN_MISMATCH=1\n", tsRev, want)
-			return
-		}
-		panic(fmt.Sprintf("binary built with tailscale_go build tag but Go toolchain %q doesn't match github.com/tailscale/tailscale expected value %q; override this failure with TS_PERMIT_TOOLCHAIN_MISMATCH=1", tsRev, want))
-	}
-}

build_docker.sh

@@ -1,11 +1,21 @@
 #!/usr/bin/env sh
+
 #
-# This script builds Tailscale container images using
-# github.com/tailscale/mkctr.
-# By default the images will be tagged with the current version and git
-# hash of this repository as produced by ./cmd/mkversion.
-# This is the image build mechanim used to build the official Tailscale
-# container images.
+# Runs `go build` with flags configured for docker distribution. All
+# it does differently from `go build` is burn git commit and version
+# information into the binaries inside docker, so that we can track down user
+# issues.
+#
+############################################################################
+#
+# WARNING: Tailscale is not yet officially supported in container
+# environments, such as Docker and Kubernetes. Though it should work, we
+# don't regularly test it, and we know there are some feature limitations.
+#
+# See current bugs tagged "containers":
+#    https://github.com/tailscale/tailscale/labels/containers
+#
+############################################################################
 
 set -eu
 
@@ -56,7 +66,6 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
-      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
@@ -73,7 +82,6 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
-      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \

client/tailscale/acl.go

@@ -19,7 +19,6 @@ import (
 // Only one of Src/Dst or Users/Ports may be specified.
 type ACLRow struct {
 	Action string   `json:"action,omitempty"` // valid values: "accept"
-	Proto  string   `json:"proto,omitempty"`  // protocol
 	Users  []string `json:"users,omitempty"`  // old name for src
 	Ports  []string `json:"ports,omitempty"`  // old name for dst
 	Src    []string `json:"src,omitempty"`
@@ -32,7 +31,6 @@ type ACLRow struct {
 type ACLTest struct {
 	Src    string   `json:"src,omitempty"`    // source
 	User   string   `json:"user,omitempty"`   // old name for source
-	Proto  string   `json:"proto,omitempty"`  // protocol
 	Accept []string `json:"accept,omitempty"` // expected destination ip:port that user can access
 	Deny   []string `json:"deny,omitempty"`   // expected destination ip:port that user cannot access
 
@@ -288,9 +286,6 @@ type UserRuleMatch struct {
 	Users      []string `json:"users"`
 	Ports      []string `json:"ports"`
 	LineNumber int      `json:"lineNumber"`
-	// Via is the list of targets through which Users can access Ports.
-	// See https://tailscale.com/kb/1378/via for more information.
-	Via []string `json:"via,omitempty"`
 
 	// Postures is a list of posture policies that are
 	// associated with this match. The rules can be looked

client/tailscale/apitype/apitype.go

@@ -4,10 +4,7 @@
 // Package apitype contains types for the Tailscale LocalAPI and control plane API.
 package apitype
 
-import (
-	"tailscale.com/tailcfg"
-	"tailscale.com/types/dnstype"
-)
+import "tailscale.com/tailcfg"
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
@@ -60,19 +57,3 @@ type ExitNodeSuggestionResponse struct {
 	Name     string
 	Location tailcfg.LocationView `json:",omitempty"`
 }
-
-// DNSOSConfig mimics dns.OSConfig without forcing us to import the entire dns package
-// into the CLI.
-type DNSOSConfig struct {
-	Nameservers   []string
-	SearchDomains []string
-	MatchDomains  []string
-}
-
-// DNSQueryResponse is the response to a DNS query request sent via LocalAPI.
-type DNSQueryResponse struct {
-	// Bytes is the raw DNS response bytes.
-	Bytes []byte
-	// Resolvers is the list of resolvers that the forwarder deemed able to resolve the query.
-	Resolvers []*dnstype.Resolver
-}

client/tailscale/localclient.go

@@ -37,10 +37,8 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
-	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
-	"tailscale.com/util/syspolicy/setting"
 )
 
 // defaultLocalClient is the default LocalClient when using the legacy
@@ -71,14 +69,6 @@ type LocalClient struct {
 	// connecting to the GUI client variants.
 	UseSocketOnly bool
 
-	// OmitAuth, if true, omits sending the local Tailscale daemon any
-	// authentication token that might be required by the platform.
-	//
-	// As of 2024-08-12, only macOS uses an authentication token. OmitAuth is
-	// meant for when Dial is set and the LocalAPI is being proxied to a
-	// different operating system, such as in integration tests.
-	OmitAuth bool
-
 	// tsClient does HTTP requests to the local Tailscale daemon.
 	// It's lazily initialized on first use.
 	tsClient     *http.Client
@@ -134,10 +124,8 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
 			},
 		}
 	})
-	if !lc.OmitAuth {
-		if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
-			req.SetBasicAuth("", token)
-		}
+	if _, token, err := safesocket.LocalTCPPortAndToken(); err == nil {
+		req.SetBasicAuth("", token)
 	}
 	return lc.tsClient.Do(req)
 }
@@ -355,12 +343,6 @@ func (lc *LocalClient) DaemonMetrics(ctx context.Context) ([]byte, error) {
 	return lc.get200(ctx, "/localapi/v0/metrics")
 }
 
-// UserMetrics returns the user metrics in
-// the Prometheus text exposition format.
-func (lc *LocalClient) UserMetrics(ctx context.Context) ([]byte, error) {
-	return lc.get200(ctx, "/localapi/v0/usermetrics")
-}
-
 // IncrementCounter increments the value of a Tailscale daemon's counter
 // metric by the given delta. If the metric has yet to exist, a new counter
 // metric is created and initialized to delta.
@@ -815,62 +797,6 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
-// GetEffectivePolicy returns the effective policy for the specified scope.
-func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
-	scopeID, err := scope.MarshalText()
-	if err != nil {
-		return nil, err
-	}
-	body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID))
-	if err != nil {
-		return nil, err
-	}
-	return decodeJSON[*setting.Snapshot](body)
-}
-
-// ReloadEffectivePolicy reloads the effective policy for the specified scope
-// by reading and merging policy settings from all applicable policy sources.
-func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
-	scopeID, err := scope.MarshalText()
-	if err != nil {
-		return nil, err
-	}
-	body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody)
-	if err != nil {
-		return nil, err
-	}
-	return decodeJSON[*setting.Snapshot](body)
-}
-
-// GetDNSOSConfig returns the system DNS configuration for the current device.
-// That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
-func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
-	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
-	if err != nil {
-		return nil, err
-	}
-	var osCfg apitype.DNSOSConfig
-	if err := json.Unmarshal(body, &osCfg); err != nil {
-		return nil, fmt.Errorf("invalid dns.OSConfig: %w", err)
-	}
-	return &osCfg, nil
-}
-
-// QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
-// It returns the raw DNS response bytes and the resolvers that were used to answer the query
-// (often just one, but can be more if we raced multiple resolvers).
-func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
-	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
-	if err != nil {
-		return nil, nil, err
-	}
-	var res apitype.DNSQueryResponse
-	if err := json.Unmarshal(body, &res); err != nil {
-		return nil, nil, fmt.Errorf("invalid query response: %w", err)
-	}
-	return res.Bytes, res.Resolvers, nil
-}
-
 // StartLoginInteractive starts an interactive login.
 func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)

client/tailscale/required_version.go

@@ -1,10 +1,10 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !go1.23
+//go:build !go1.21
 
 package tailscale
 
 func init() {
-	you_need_Go_1_23_to_compile_Tailscale()
+	you_need_Go_1_21_to_compile_Tailscale()
 }

client/tailscale/tailscale.go

@@ -51,9 +51,6 @@ type Client struct {
 	// HTTPClient optionally specifies an alternate HTTP client to use.
 	// If nil, http.DefaultClient is used.
 	HTTPClient *http.Client
-
-	// UserAgent optionally specifies an alternate User-Agent header
-	UserAgent string
 }
 
 func (c *Client) httpClient() *http.Client {
@@ -100,9 +97,8 @@ func (c *Client) setAuth(r *http.Request) {
 // and can be changed manually by the user.
 func NewClient(tailnet string, auth AuthMethod) *Client {
 	return &Client{
-		tailnet:   tailnet,
-		auth:      auth,
-		UserAgent: "tailscale-client-oss",
+		tailnet: tailnet,
+		auth:    auth,
 	}
 }
 
@@ -114,16 +110,17 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 		return nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
 	}
 	c.setAuth(req)
-	if c.UserAgent != "" {
-		req.Header.Set("User-Agent", c.UserAgent)
-	}
 	return c.httpClient().Do(req)
 }
 
 // sendRequest add the authentication key to the request and sends it. It
 // receives the response and reads up to 10MB of it.
 func (c *Client) sendRequest(req *http.Request) ([]byte, *http.Response, error) {
-	resp, err := c.Do(req)
+	if !I_Acknowledge_This_API_Is_Unstable {
+		return nil, nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
+	}
+	c.setAuth(req)
+	resp, err := c.httpClient().Do(req)
 	if err != nil {
 		return nil, resp, err
 	}

client/web/web.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -26,7 +27,6 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
-	"tailscale.com/envknob/featureknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -35,7 +35,6 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/views"
 	"tailscale.com/util/httpm"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -114,6 +113,11 @@ const (
 	ManageServerMode ServerMode = "manage"
 )
 
+var (
+	exitNodeRouteV4 = netip.MustParsePrefix("0.0.0.0/0")
+	exitNodeRouteV6 = netip.MustParsePrefix("::/0")
+)
+
 // ServerOpts contains options for constructing a new Server.
 type ServerOpts struct {
 	// Mode specifies the mode of web client being constructed.
@@ -279,12 +283,6 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if r.URL.Path == "/metrics" {
-		r.URL.Path = "/api/local/v0/usermetrics"
-		s.proxyRequestToLocalAPI(w, r)
-		return
-	}
-
 	if strings.HasPrefix(r.URL.Path, "/api/") {
 		switch {
 		case r.URL.Path == "/api/auth" && r.Method == httpm.GET:
@@ -923,10 +921,10 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 			return p == route
 		})
 	}
-	data.AdvertisingExitNodeApproved = routeApproved(tsaddr.AllIPv4()) || routeApproved(tsaddr.AllIPv6())
+	data.AdvertisingExitNodeApproved = routeApproved(exitNodeRouteV4) || routeApproved(exitNodeRouteV6)
 
 	for _, r := range prefs.AdvertiseRoutes {
-		if tsaddr.IsExitRoute(r) {
+		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
 			data.AdvertisingExitNode = true
 		} else {
 			data.AdvertisedRoutes = append(data.AdvertisedRoutes, subnetRoute{
@@ -961,16 +959,37 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 }
 
 func availableFeatures() map[string]bool {
+	env := hostinfo.GetEnvType()
 	features := map[string]bool{
 		"advertise-exit-node": true, // available on all platforms
 		"advertise-routes":    true, // available on all platforms
-		"use-exit-node":       featureknob.CanUseExitNode() == nil,
-		"ssh":                 featureknob.CanRunTailscaleSSH() == nil,
+		"use-exit-node":       canUseExitNode(env) == nil,
+		"ssh":                 envknob.CanRunTailscaleSSH() == nil,
 		"auto-update":         version.IsUnstableBuild() && clientupdate.CanAutoUpdate(),
 	}
+	if env == hostinfo.HomeAssistantAddOn {
+		// Setting SSH on Home Assistant causes trouble on startup
+		// (since the flag is not being passed to `tailscale up`).
+		// Although Tailscale SSH does work here,
+		// it's not terribly useful since it's running in a separate container.
+		features["ssh"] = false
+	}
 	return features
 }
 
+func canUseExitNode(env hostinfo.EnvType) error {
+	switch dist := distro.Get(); dist {
+	case distro.Synology, // see https://github.com/tailscale/tailscale/issues/1995
+		distro.QNAP,
+		distro.Unraid:
+		return fmt.Errorf("Tailscale exit nodes cannot be used on %s.", dist)
+	}
+	if env == hostinfo.HomeAssistantAddOn {
+		return errors.New("Tailscale exit nodes cannot be used on Home Assistant.")
+	}
+	return nil
+}
+
 // aclsAllowAccess returns whether tailnet ACLs (as expressed in the provided filter rules)
 // permit any devices to access the local web client.
 // This does not currently check whether a specific device can connect, just any device.
@@ -1046,7 +1065,7 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 	var currNonExitRoutes []string
 	var currAdvertisingExitNode bool
 	for _, r := range prefs.AdvertiseRoutes {
-		if tsaddr.IsExitRoute(r) {
+		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
 			currAdvertisingExitNode = true
 			continue
 		}
@@ -1067,7 +1086,12 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 		return err
 	}
 
-	if !data.UseExitNode.IsZero() && tsaddr.ContainsExitRoutes(views.SliceOf(routes)) {
+	hasExitNodeRoute := func(all []netip.Prefix) bool {
+		return slices.Contains(all, exitNodeRouteV4) ||
+			slices.Contains(all, exitNodeRouteV6)
+	}
+
+	if !data.UseExitNode.IsZero() && hasExitNodeRoute(routes) {
 		return errors.New("cannot use and advertise exit node at same time")
 	}
 

client/web/yarn.lock

@@ -5382,9 +5382,9 @@ wrappy@1:
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
 ws@^8.14.2:
-  version "8.17.1"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
-  integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
+  version "8.14.2"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f"
+  integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==
 
 xml-name-validator@^5.0.0:
   version "5.0.0"

clientupdate/clientupdate.go

@@ -27,8 +27,11 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
+	"tailscale.com/clientupdate/distsign"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
+	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
@@ -753,6 +756,164 @@ func (up *Updater) updateMacAppStore() error {
 	return nil
 }
 
+const (
+	// winMSIEnv is the environment variable that, if set, is the MSI file for
+	// the update command to install. It's passed like this so we can stop the
+	// tailscale.exe process from running before the msiexec process runs and
+	// tries to overwrite ourselves.
+	winMSIEnv = "TS_UPDATE_WIN_MSI"
+	// winExePathEnv is the environment variable that is set along with
+	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
+	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
+	// install is complete.
+	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
+)
+
+var (
+	verifyAuthenticode func(string) error // set non-nil only on Windows
+	markTempFileFunc   func(string) error // set non-nil only on Windows
+)
+
+func (up *Updater) updateWindows() error {
+	if msi := os.Getenv(winMSIEnv); msi != "" {
+		// stdout/stderr from this part of the install could be lost since the
+		// parent tailscaled is replaced. Create a temp log file to have some
+		// output to debug with in case update fails.
+		close, err := up.switchOutputToFile()
+		if err != nil {
+			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
+		} else {
+			defer close.Close()
+		}
+
+		up.Logf("installing %v ...", msi)
+		if err := up.installMSI(msi); err != nil {
+			up.Logf("MSI install failed: %v", err)
+			return err
+		}
+
+		up.Logf("success.")
+		return nil
+	}
+
+	if !winutil.IsCurrentProcessElevated() {
+		return errors.New(`update must be run as Administrator
+
+you can run the command prompt as Administrator one of these ways:
+* right-click cmd.exe, select 'Run as administrator'
+* press Windows+x, then press a
+* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
+	}
+	ver, err := requestedTailscaleVersion(up.Version, up.Track)
+	if err != nil {
+		return err
+	}
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	if !up.confirm(ver) {
+		return nil
+	}
+
+	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
+	msiDir := filepath.Join(tsDir, "MSICache")
+	if fi, err := os.Stat(tsDir); err != nil {
+		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
+	}
+	if err := os.MkdirAll(msiDir, 0700); err != nil {
+		return err
+	}
+	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
+	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
+	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
+		return err
+	}
+
+	up.Logf("verifying MSI authenticode...")
+	if err := verifyAuthenticode(msiTarget); err != nil {
+		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
+	}
+	up.Logf("authenticode verification succeeded")
+
+	up.Logf("making tailscale.exe copy to switch to...")
+	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
+	selfOrig, selfCopy, err := makeSelfCopy()
+	if err != nil {
+		return err
+	}
+	defer os.Remove(selfCopy)
+	up.Logf("running tailscale.exe copy for final install...")
+
+	cmd := exec.Command(selfCopy, "update")
+	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
+	cmd.Stdout = up.Stderr
+	cmd.Stderr = up.Stderr
+	cmd.Stdin = os.Stdin
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	// Once it's started, exit ourselves, so the binary is free
+	// to be replaced.
+	os.Exit(0)
+	panic("unreachable")
+}
+
+func (up *Updater) switchOutputToFile() (io.Closer, error) {
+	var logFilePath string
+	exePath, err := os.Executable()
+	if err != nil {
+		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
+	} else {
+		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
+	}
+
+	up.Logf("writing update output to %q", logFilePath)
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	up.Logf = func(m string, args ...any) {
+		fmt.Fprintf(logFile, m+"\n", args...)
+	}
+	up.Stdout = logFile
+	up.Stderr = logFile
+	return logFile, nil
+}
+
+func (up *Updater) installMSI(msi string) error {
+	var err error
+	for tries := 0; tries < 2; tries++ {
+		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
+		cmd.Dir = filepath.Dir(msi)
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		if err == nil {
+			break
+		}
+		up.Logf("Install attempt failed: %v", err)
+		uninstallVersion := up.currentVersion
+		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
+			uninstallVersion = v
+		}
+		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
+		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
+		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		up.Logf("msiexec uninstall: %v", err)
+	}
+	return err
+}
+
 // cleanupOldDownloads removes all files matching glob (see filepath.Glob).
 // Only regular files are removed, so the glob must match specific files and
 // not directories.
@@ -777,6 +938,53 @@ func (up *Updater) cleanupOldDownloads(glob string) {
 	}
 }
 
+func msiUUIDForVersion(ver string) string {
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	track, err := versionToTrack(ver)
+	if err != nil {
+		track = UnstableTrack
+	}
+	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
+	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
+}
+
+func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
+	selfExe, err := os.Executable()
+	if err != nil {
+		return "", "", err
+	}
+	f, err := os.Open(selfExe)
+	if err != nil {
+		return "", "", err
+	}
+	defer f.Close()
+	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
+	if err != nil {
+		return "", "", err
+	}
+	if f := markTempFileFunc; f != nil {
+		if err := f(f2.Name()); err != nil {
+			return "", "", err
+		}
+	}
+	if _, err := io.Copy(f2, f); err != nil {
+		f2.Close()
+		return "", "", err
+	}
+	return selfExe, f2.Name(), f2.Close()
+}
+
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
+	if err != nil {
+		return err
+	}
+	return c.Download(context.Background(), pathSrc, fileDst)
+}
+
 func (up *Updater) updateFreeBSD() (err error) {
 	if up.Version != "" {
 		return errors.New("installing a specific version on FreeBSD is not supported")

clientupdate/clientupdate_downloads.go

@@ -1,20 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build (linux && !android) || windows
-
-package clientupdate
-
-import (
-	"context"
-
-	"tailscale.com/clientupdate/distsign"
-)
-
-func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
-	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
-	if err != nil {
-		return err
-	}
-	return c.Download(context.Background(), pathSrc, fileDst)
-}

clientupdate/clientupdate_not_downloads.go

@@ -1,10 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !((linux && !android) || windows)
-
-package clientupdate
-
-func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
-	panic("unreachable")
-}

clientupdate/clientupdate_notwindows.go

@@ -1,10 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !windows
-
-package clientupdate
-
-func (up *Updater) updateWindows() error {
-	panic("unreachable")
-}

clientupdate/clientupdate_windows.go

@@ -7,57 +7,13 @@
 package clientupdate
 
 import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"runtime"
-	"strings"
-
-	"github.com/google/uuid"
 	"golang.org/x/sys/windows"
-	"tailscale.com/util/winutil"
 	"tailscale.com/util/winutil/authenticode"
 )
 
-const (
-	// winMSIEnv is the environment variable that, if set, is the MSI file for
-	// the update command to install. It's passed like this so we can stop the
-	// tailscale.exe process from running before the msiexec process runs and
-	// tries to overwrite ourselves.
-	winMSIEnv = "TS_UPDATE_WIN_MSI"
-	// winExePathEnv is the environment variable that is set along with
-	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
-	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
-	// install is complete.
-	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
-)
-
-func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
-	selfExe, err := os.Executable()
-	if err != nil {
-		return "", "", err
-	}
-	f, err := os.Open(selfExe)
-	if err != nil {
-		return "", "", err
-	}
-	defer f.Close()
-	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
-	if err != nil {
-		return "", "", err
-	}
-	if err := markTempFileWindows(f2.Name()); err != nil {
-		return "", "", err
-	}
-	if _, err := io.Copy(f2, f); err != nil {
-		f2.Close()
-		return "", "", err
-	}
-	return selfExe, f2.Name(), f2.Close()
+func init() {
+	markTempFileFunc = markTempFileWindows
+	verifyAuthenticode = verifyTailscale
 }
 
 func markTempFileWindows(name string) error {
@@ -67,159 +23,6 @@ func markTempFileWindows(name string) error {
 
 const certSubjectTailscale = "Tailscale Inc."
 
-func verifyAuthenticode(path string) error {
+func verifyTailscale(path string) error {
 	return authenticode.Verify(path, certSubjectTailscale)
 }
-
-func (up *Updater) updateWindows() error {
-	if msi := os.Getenv(winMSIEnv); msi != "" {
-		// stdout/stderr from this part of the install could be lost since the
-		// parent tailscaled is replaced. Create a temp log file to have some
-		// output to debug with in case update fails.
-		close, err := up.switchOutputToFile()
-		if err != nil {
-			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
-		} else {
-			defer close.Close()
-		}
-
-		up.Logf("installing %v ...", msi)
-		if err := up.installMSI(msi); err != nil {
-			up.Logf("MSI install failed: %v", err)
-			return err
-		}
-
-		up.Logf("success.")
-		return nil
-	}
-
-	if !winutil.IsCurrentProcessElevated() {
-		return errors.New(`update must be run as Administrator
-
-you can run the command prompt as Administrator one of these ways:
-* right-click cmd.exe, select 'Run as administrator'
-* press Windows+x, then press a
-* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
-	}
-	ver, err := requestedTailscaleVersion(up.Version, up.Track)
-	if err != nil {
-		return err
-	}
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	if !up.confirm(ver) {
-		return nil
-	}
-
-	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
-	msiDir := filepath.Join(tsDir, "MSICache")
-	if fi, err := os.Stat(tsDir); err != nil {
-		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
-	} else if !fi.IsDir() {
-		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
-	}
-	if err := os.MkdirAll(msiDir, 0700); err != nil {
-		return err
-	}
-	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
-	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
-	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
-	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
-		return err
-	}
-
-	up.Logf("verifying MSI authenticode...")
-	if err := verifyAuthenticode(msiTarget); err != nil {
-		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
-	}
-	up.Logf("authenticode verification succeeded")
-
-	up.Logf("making tailscale.exe copy to switch to...")
-	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
-	selfOrig, selfCopy, err := makeSelfCopy()
-	if err != nil {
-		return err
-	}
-	defer os.Remove(selfCopy)
-	up.Logf("running tailscale.exe copy for final install...")
-
-	cmd := exec.Command(selfCopy, "update")
-	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
-	cmd.Stdout = up.Stderr
-	cmd.Stderr = up.Stderr
-	cmd.Stdin = os.Stdin
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	// Once it's started, exit ourselves, so the binary is free
-	// to be replaced.
-	os.Exit(0)
-	panic("unreachable")
-}
-
-func (up *Updater) installMSI(msi string) error {
-	var err error
-	for tries := 0; tries < 2; tries++ {
-		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
-		cmd.Dir = filepath.Dir(msi)
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		if err == nil {
-			break
-		}
-		up.Logf("Install attempt failed: %v", err)
-		uninstallVersion := up.currentVersion
-		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
-			uninstallVersion = v
-		}
-		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
-		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
-		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		up.Logf("msiexec uninstall: %v", err)
-	}
-	return err
-}
-
-func msiUUIDForVersion(ver string) string {
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	track, err := versionToTrack(ver)
-	if err != nil {
-		track = UnstableTrack
-	}
-	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
-	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
-}
-
-func (up *Updater) switchOutputToFile() (io.Closer, error) {
-	var logFilePath string
-	exePath, err := os.Executable()
-	if err != nil {
-		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
-	} else {
-		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
-	}
-
-	up.Logf("writing update output to %q", logFilePath)
-	logFile, err := os.Create(logFilePath)
-	if err != nil {
-		return nil, err
-	}
-
-	up.Logf = func(m string, args ...any) {
-		fmt.Fprintf(logFile, m+"\n", args...)
-	}
-	up.Stdout = logFile
-	up.Stderr = logFile
-	return logFile, nil
-}

cmd/cloner/cloner.go

@@ -47,7 +47,7 @@ func main() {
 	it := codegen.NewImportTracker(pkg.Types)
 	buf := new(bytes.Buffer)
 	for _, typeName := range typeNames {
-		typ, ok := namedTypes[typeName].(*types.Named)
+		typ, ok := namedTypes[typeName]
 		if !ok {
 			log.Fatalf("could not find type %s", typeName)
 		}
@@ -115,7 +115,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		if !codegen.ContainsPointers(ft) || codegen.HasNoClone(t.Tag(i)) {
 			continue
 		}
-		if named, _ := codegen.NamedTypeOf(ft); named != nil {
+		if named, _ := ft.(*types.Named); named != nil {
 			if codegen.IsViewType(ft) {
 				writef("dst.%s = src.%s", fname, fname)
 				continue
@@ -161,7 +161,7 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 		case *types.Pointer:
 			base := ft.Elem()
 			hasPtrs := codegen.ContainsPointers(base)
-			if named, _ := codegen.NamedTypeOf(base); named != nil && hasPtrs {
+			if named, _ := base.(*types.Named); named != nil && hasPtrs {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}

cmd/containerboot/forwarding.go

@@ -1,262 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"net"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"tailscale.com/util/linuxfw"
-)
-
-// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
-	var (
-		v4Forwarding, v6Forwarding bool
-	)
-	if clusterProxyTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid cluster destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	if tailnetTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid tailnet destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	// Currently we only proxy traffic to the IPv4 address of the tailnet
-	// target.
-	if tailnetTargetFQDN != "" {
-		v4Forwarding = true
-	}
-	if routes != nil && *routes != "" {
-		for _, route := range strings.Split(*routes, ",") {
-			cidr, err := netip.ParsePrefix(route)
-			if err != nil {
-				return fmt.Errorf("invalid subnet route: %v", err)
-			}
-			if cidr.Addr().Is4() {
-				v4Forwarding = true
-			} else {
-				v6Forwarding = true
-			}
-		}
-	}
-	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
-}
-
-func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
-	var paths []string
-	if v4Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
-	}
-	if v6Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
-	}
-
-	// In some common configurations (e.g. default docker,
-	// kubernetes), the container environment denies write access to
-	// most sysctls, including IP forwarding controls. Check the
-	// sysctl values before trying to change them, so that we
-	// gracefully do nothing if the container's already been set up
-	// properly by e.g. a k8s initContainer.
-	for _, path := range paths {
-		bs, err := os.ReadFile(path)
-		if err != nil {
-			return fmt.Errorf("reading %q: %w", path, err)
-		}
-		if v := strings.TrimSpace(string(bs)); v != "1" {
-			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
-				return fmt.Errorf("enabling %q: %w", path, err)
-			}
-		}
-	}
-	return nil
-}
-
-func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.EnsureSNATForDst(local, dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	return nil
-}
-
-// installTSForwardingRuleForDestination accepts a destination address and a
-// list of node's tailnet addresses, sets up rules to forward traffic for
-// destination to the tailnet IP matching the destination IP family.
-// Destination can be Pod IP of this node.
-func installTSForwardingRuleForDestination(_ context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstFilter)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
-	}
-	if err := nfr.AddDNATRule(dst, local); err != nil {
-		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	proxyHasIPv4Address := false
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() {
-			proxyHasIPv4Address = true
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if proxyHasIPv4Address && dst.Is6() {
-		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.AddDNATRule(local, dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRuleForDNSTarget(_ context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	var (
-		tsv4       netip.Addr
-		tsv6       netip.Addr
-		v4Backends []netip.Addr
-		v6Backends []netip.Addr
-	)
-	for _, pfx := range tsIPs {
-		if pfx.IsSingleIP() && pfx.Addr().Is4() {
-			tsv4 = pfx.Addr()
-			continue
-		}
-		if pfx.IsSingleIP() && pfx.Addr().Is6() {
-			tsv6 = pfx.Addr()
-			continue
-		}
-	}
-	// TODO: log if more than one backend address is found and firewall is
-	// in nftables mode that only the first IP will be used.
-	for _, ip := range backendAddrs {
-		if ip.To4() != nil {
-			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
-		}
-		if ip.To16() != nil {
-			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
-		}
-	}
-
-	// Enable IP forwarding here as opposed to at the start of containerboot
-	// as the IPv4/IPv6 requirements might have changed.
-	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
-	// enabled by an init container, so in practice enabling forwarding here
-	// is only needed if this proxy has been configured by manually setting
-	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
-	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
-		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
-	}
-
-	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
-		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
-			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
-		}
-		// The backend might advertize MSS higher than that of the
-		// tailscale interfaces. Clamp MSS of packets going out via
-		// tailscale0 interface to its MTU to prevent broken connections
-		// in environments where path MTU discovery is not working.
-		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
-		}
-		return nil
-	}
-
-	if len(v4Backends) != 0 {
-		if !tsv4.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
-		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
-			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
-		}
-	}
-	if len(v6Backends) != 0 && !tsv6.IsValid() {
-		if !tsv6.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
-		} else if !nfr.HasIPV6NAT() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
-		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
-			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
-		}
-	}
-	return nil
-}

cmd/containerboot/healthz.go

@@ -1,51 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"log"
-	"net"
-	"net/http"
-	"sync"
-)
-
-// healthz is a simple health check server, if enabled it returns 200 OK if
-// this tailscale node currently has at least one tailnet IP address else
-// returns 503.
-type healthz struct {
-	sync.Mutex
-	hasAddrs bool
-}
-
-func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	h.Lock()
-	defer h.Unlock()
-	if h.hasAddrs {
-		w.Write([]byte("ok"))
-	} else {
-		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
-	}
-}
-
-// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
-// provided address. A containerized tailscale instance is considered healthy if
-// it has at least one tailnet IP address.
-func runHealthz(addr string, h *healthz) {
-	lis, err := net.Listen("tcp", addr)
-	if err != nil {
-		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
-	}
-	mux := http.NewServeMux()
-	mux.Handle("/healthz", h)
-	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
-	hs := &http.Server{Handler: mux}
-
-	go func() {
-		if err := hs.Serve(lis); err != nil {
-			log.Fatalf("failed running health endpoint: %v", err)
-		}
-	}()
-}

cmd/containerboot/kube.go

@@ -8,21 +8,21 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
-	"tailscale.com/kube/kubeapi"
-	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube"
 	"tailscale.com/tailcfg"
 )
 
 // storeDeviceID writes deviceID to 'device_id' data field of the named
 // Kubernetes Secret.
 func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
-	s := &kubeapi.Secret{
+	s := &kube.Secret{
 		Data: map[string][]byte{
 			"device_id": []byte(deviceID),
 		},
@@ -42,7 +42,7 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 		return err
 	}
 
-	s := &kubeapi.Secret{
+	s := &kube.Secret{
 		Data: map[string][]byte{
 			"device_fqdn": []byte(fqdn),
 			"device_ips":  deviceIPs,
@@ -55,14 +55,14 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 // secret. No-op if there is no authkey in the secret.
 func deleteAuthKey(ctx context.Context, secretName string) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
-	m := []kubeclient.JSONPatch{
+	m := []kube.JSONPatch{
 		{
 			Op:   "remove",
 			Path: "/data/authkey",
 		},
 	}
 	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
-		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
+		if s, ok := err.(*kube.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
 			return nil
@@ -72,16 +72,66 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kubeclient.Client
+var kc kube.Client
+
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kube.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
 
 func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.
-		kubeclient.SetRootPathForTesting(root)
+		kube.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kubeclient.New()
+	kc, err = kube.New()
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/kube_test.go

@@ -11,8 +11,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"tailscale.com/kube/kubeapi"
-	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube"
 )
 
 func TestSetupKube(t *testing.T) {
@@ -21,7 +20,7 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kubeclient.Client
+		kc      kube.Client
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
@@ -29,11 +28,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
 					return nil, nil
 				},
 			},
@@ -48,12 +47,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return nil, &kubeapi.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -67,12 +66,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return nil, &kubeapi.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
 				},
 			},
 			wantCfg: &settings{
@@ -87,12 +86,12 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return nil, &kubeapi.Status{Code: 403}
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 403}
 				},
 			},
 			wantCfg: &settings{
@@ -111,7 +110,7 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
@@ -127,12 +126,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return nil, &kubeapi.Status{Code: 404}
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
 				},
 			},
 		},
@@ -145,12 +144,12 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return &kubeapi.Secret{}, nil
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{}, nil
 				},
 			},
 		},
@@ -159,12 +158,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{
@@ -177,12 +176,12 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kube.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
-				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
-					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
 			},
 			wantCfg: &settings{

cmd/containerboot/main.go

@@ -52,12 +52,6 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
-//   - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
-//     served at /healthz at the provided address, which should be in form [<address>]:<port>.
-//     If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
-//     The health endpoint will return 200 OK if this node has at least one tailnet IP address,
-//     otherwise returns 503.
-//     NB: the health criteria might change in the future.
 //   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
 //     directory that containers tailscaled config in file. The config file needs to be
 //     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
@@ -92,7 +86,9 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -101,19 +97,24 @@ import (
 	"net"
 	"net/netip"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
+	"reflect"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"golang.org/x/sys/unix"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/conffile"
 	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -132,9 +133,34 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) {
 func main() {
 	log.SetPrefix("boot: ")
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
+	cfg := &settings{
+		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
+		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
+		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
+		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
+		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
+		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
+		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
+		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
+		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
+		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
+		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
+		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
+		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
+		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
+		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
+		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
+		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
+		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
+		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
+		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
+		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
+		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
+		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+	}
 
-	cfg, err := configFromEnv()
-	if err != nil {
+	if err := cfg.validate(); err != nil {
 		log.Fatalf("invalid configuration: %v", err)
 	}
 
@@ -249,8 +275,10 @@ authLoop:
 			switch *n.State {
 			case ipn.NeedsLogin:
 				if isOneStepConfig(cfg) {
-					// This could happen if this is the first time tailscaled was run for this
-					// device and the auth key was not passed via the configfile.
+					// This could happen if this is the
+					// first time tailscaled was run for
+					// this device and the auth key was not
+					// passed via the configfile.
 					log.Fatalf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
 				}
 				if err := authTailscale(); err != nil {
@@ -321,9 +349,6 @@ authLoop:
 
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
-
-		h             = &healthz{} // http server for the healthz endpoint
-		healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
 	)
 	if cfg.ServeConfigPath != "" {
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
@@ -348,9 +373,6 @@ authLoop:
 			}
 		})
 	)
-	// egressSvcsErrorChan will get an error sent to it if this containerboot instance is configured to expose 1+
-	// egress services in HA mode and errored.
-	var egressSvcsErrorChan = make(chan error)
 	defer t.Stop()
 	// resetTimer resets timer for when to next attempt to resolve the DNS
 	// name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The
@@ -376,7 +398,6 @@ authLoop:
 		failedResolveAttempts++
 	}
 
-	var egressSvcsNotify chan ipn.Notify
 	notifyChan := make(chan ipn.Notify)
 	errChan := make(chan error)
 	go func() {
@@ -454,11 +475,7 @@ runLoop:
 					egressAddrs = node.Addresses().AsSlice()
 					newCurentEgressIPs = deephash.Hash(&egressAddrs)
 					egressIPsHaveChanged = newCurentEgressIPs != currentEgressIPs
-					// The firewall rules get (re-)installed:
-					// - on startup
-					// - when the tailnet IPs of the tailnet target have changed
-					// - when the tailnet IPs of this node have changed
-					if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 {
+					if egressIPsHaveChanged && len(egressAddrs) != 0 {
 						var rulesInstalled bool
 						for _, egressAddr := range egressAddrs {
 							ea := egressAddr.Addr()
@@ -548,57 +565,31 @@ runLoop:
 						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
-
-				if cfg.HealthCheckAddrPort != "" {
-					h.Lock()
-					h.hasAddrs = len(addrs) != 0
-					h.Unlock()
-					healthzRunner()
-				}
-				if egressSvcsNotify != nil {
-					egressSvcsNotify <- n
-				}
 			}
 			if !startupTasksDone {
-				// For containerboot instances that act as TCP proxies (proxying traffic to an endpoint
-				// passed via one of the env vars that containerboot reads) and store state in a
-				// Kubernetes Secret, we consider startup tasks done at the point when device info has
-				// been successfully stored to state Secret. For all other containerboot instances, if
-				// we just get to this point the startup tasks can be considered done.
+				// For containerboot instances that act as TCP
+				// proxies (proxying traffic to an endpoint
+				// passed via one of the env vars that
+				// containerbot reads) and store state in a
+				// Kubernetes Secret, we consider startup tasks
+				// done at the point when device info has been
+				// successfully stored to state Secret.
+				// For all other containerboot instances, if we
+				// just get to this point the startup tasks can
+				// be considered done.
 				if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) {
 					// This log message is used in tests to detect when all
 					// post-auth configuration is done.
 					log.Println("Startup complete, waiting for shutdown signal")
 					startupTasksDone = true
 
-					// Configure egress proxy. Egress proxy will set up firewall rules to proxy
-					// traffic to tailnet targets configured in the provided configuration file. It
-					// will then continuously monitor the config file and netmap updates and
-					// reconfigure the firewall rules as needed. If any of its operations fail, it
-					// will crash this node.
-					if cfg.EgressSvcsCfgPath != "" {
-						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressSvcsCfgPath)
-						egressSvcsNotify = make(chan ipn.Notify)
-						ep := egressProxy{
-							cfgPath:      cfg.EgressSvcsCfgPath,
-							nfr:          nfr,
-							kc:           kc,
-							stateSecret:  cfg.KubeSecret,
-							netmapChan:   egressSvcsNotify,
-							podIPv4:      cfg.PodIPv4,
-							tailnetAddrs: addrs,
-						}
-						go func() {
-							if err := ep.run(ctx, n); err != nil {
-								egressSvcsErrorChan <- err
-							}
-						}()
-					}
-
-					// Wait on tailscaled process. It won't be cleaned up by default when the
-					// container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the
-					// reaper by a running cmd.Wait in a goroutine immediately after starting
-					// tailscaled?
+					// Wait on tailscaled process. It won't
+					// be cleaned up by default when the
+					// container exits as it is not PID1.
+					// TODO (irbekrm): perhaps we can
+					// replace the reaper by a running
+					// cmd.Wait in a goroutine immediately
+					// after starting tailscaled?
 					reaper := func() {
 						defer wg.Done()
 						for {
@@ -636,13 +627,226 @@ runLoop:
 			}
 			backendAddrs = newBackendAddrs
 			resetTimer(false)
-		case e := <-egressSvcsErrorChan:
-			log.Fatalf("egress proxy failed: %v", e)
 		}
 	}
 	wg.Wait()
 }
 
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+	if certDomainAtomic == nil {
+		panic("cd must not be nil")
+	}
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(path)); err != nil {
+			log.Fatalf("failed to add fsnotify watch: %v", err)
+		}
+		eventChan = w.Events
+	}
+
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-eventChan:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		if certDomain == "" {
+			continue
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("failed to read serve config: %v", err)
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		log.Printf("Applying serve config")
+		if err := lc.SetServeConfig(ctx, sc); err != nil {
+			log.Fatalf("failed to set serve config: %v", err)
+		}
+		prevServeConfig = sc
+	}
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}
+
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+	args := tailscaledArgs(cfg)
+	// tailscaled runs without context, since it needs to persist
+	// beyond the startup timeout in ctx.
+	cmd := exec.Command("tailscaled", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	log.Printf("Starting tailscaled")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
+	}
+
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
+	log.Printf("Waiting for tailscaled socket")
+	for {
+		if ctx.Err() != nil {
+			log.Fatalf("Timed out waiting for tailscaled socket")
+		}
+		_, err := os.Stat(cfg.Socket)
+		if errors.Is(err, fs.ErrNotExist) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		} else if err != nil {
+			log.Fatalf("Waiting for tailscaled socket: %v", err)
+		}
+		break
+	}
+
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process, nil
+}
+
+// tailscaledArgs uses cfg to construct the argv for tailscaled.
+func tailscaledArgs(cfg *settings) []string {
+	args := []string{"--socket=" + cfg.Socket}
+	switch {
+	case cfg.InKubernetes && cfg.KubeSecret != "":
+		args = append(args, "--state=kube:"+cfg.KubeSecret)
+		if cfg.StateDir == "" {
+			cfg.StateDir = "/tmp"
+		}
+		fallthrough
+	case cfg.StateDir != "":
+		args = append(args, "--statedir="+cfg.StateDir)
+	default:
+		args = append(args, "--state=mem:", "--statedir=/tmp")
+	}
+
+	if cfg.UserspaceMode {
+		args = append(args, "--tun=userspace-networking")
+	} else if err := ensureTunFile(cfg.Root); err != nil {
+		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
+	}
+
+	if cfg.SOCKSProxyAddr != "" {
+		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
+	}
+	if cfg.HTTPProxyAddr != "" {
+		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
+	}
+	if cfg.TailscaledConfigFilePath != "" {
+		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
+	}
+	if cfg.DaemonExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
+	}
+	return args
+}
+
+// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
+// if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleUp(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "up"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale up'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale up failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state and only if TS_AUTH_ONCE is set.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	log.Printf("Running 'tailscale set'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale set failed: %v", err)
+	}
+	return nil
+}
+
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
 func ensureTunFile(root string) error {
@@ -662,6 +866,344 @@ func ensureTunFile(root string) error {
 	return nil
 }
 
+// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
+	var (
+		v4Forwarding, v6Forwarding bool
+	)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	// Currently we only proxy traffic to the IPv4 address of the tailnet
+	// target.
+	if tailnetTargetFQDN != "" {
+		v4Forwarding = true
+	}
+	if routes != nil && *routes != "" {
+		for _, route := range strings.Split(*routes, ",") {
+			cidr, err := netip.ParsePrefix(route)
+			if err != nil {
+				return fmt.Errorf("invalid subnet route: %v", err)
+			}
+			if cidr.Addr().Is4() {
+				v4Forwarding = true
+			} else {
+				v6Forwarding = true
+			}
+		}
+	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
+
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
+	var paths []string
+	if v4Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
+	}
+	if v6Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
+	}
+
+	// In some common configurations (e.g. default docker,
+	// kubernetes), the container environment denies write access to
+	// most sysctls, including IP forwarding controls. Check the
+	// sysctl values before trying to change them, so that we
+	// gracefully do nothing if the container's already been set up
+	// properly by e.g. a k8s initContainer.
+	for _, path := range paths {
+		bs, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %q: %w", path, err)
+		}
+		if v := strings.TrimSpace(string(bs)); v != "1" {
+			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
+				return fmt.Errorf("enabling %q: %w", path, err)
+			}
+		}
+	}
+	return nil
+}
+
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	return nil
+}
+
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	proxyHasIPv4Address := false
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.AddDNATRule(local, dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRuleForDNSTarget(ctx context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}
+
+// settings is all the configuration for containerboot.
+type settings struct {
+	AuthKey  string
+	Hostname string
+	Routes   *string
+	// ProxyTargetIP is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. This is typically a
+	// Tailscale IP.
+	TailnetTargetIP string
+	// TailnetTargetFQDN is an MagicDNS name to which all incoming
+	// non-Tailscale traffic should be proxied. This must be a full Tailnet
+	// node FQDN.
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	PodIP string
+}
+
+func (s *settings) validate() error {
+	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
+		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
+		}
+	}
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
+	if s.TailnetTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
+		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
+	}
+	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
+	return nil
+}
+
 func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	// TODO (irbekrm): look at using recursive.Resolver instead to resolve
 	// the DNS names as well as retrieve TTLs. It looks though that this
@@ -684,6 +1226,57 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	return append(ip4s, ip6s...), nil
 }
 
+// defaultEnv returns the value of the given envvar name, or defVal if
+// unset.
+func defaultEnv(name, defVal string) string {
+	if v, ok := os.LookupEnv(name); ok {
+		return v
+	}
+	return defVal
+}
+
+// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being set to empty string vs unset.
+func defaultEnvStringPointer(name string) *string {
+	if v, ok := os.LookupEnv(name); ok {
+		return &v
+	}
+	return nil
+}
+
+// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being explicitly set to false vs unset.
+func defaultEnvBoolPointer(name string) *bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return nil
+	}
+	return &ret
+}
+
+func defaultEnvs(names []string, defVal string) string {
+	for _, name := range names {
+		if v, ok := os.LookupEnv(name); ok {
+			return v
+		}
+	}
+	return defVal
+}
+
+// defaultBool returns the boolean value of the given envvar name, or
+// defVal if unset or not a bool.
+func defaultBool(name string, defVal bool) bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return defVal
+	}
+	return ret
+}
+
 // contextWithExitSignalWatch watches for SIGTERM/SIGINT signals. It returns a
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
@@ -706,6 +1299,43 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 	return ctx, f
 }
 
+// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
+// in two steps and login should only happen once.
+// Step 1: run 'tailscaled'
+// Step 2):
+// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
+// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
+func isTwoStepConfigAuthOnce(cfg *settings) bool {
+	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
+// in two steps and we should log in every time it starts.
+// Step 1: run 'tailscaled'
+// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
+func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
+	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isOneStepConfig returns true if the Tailscale node should always be ran and
+// configured in a single step by running 'tailscaled <config opts>'
+func isOneStepConfig(cfg *settings) bool {
+	return cfg.TailscaledConfigFilePath != ""
+}
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
 // tailscaledConfigFilePath returns the path to the tailscaled config file that
 // should be used for the current capability version. It is determined by the
 // TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a
@@ -742,5 +1372,5 @@ func tailscaledConfigFilePath() string {
 		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
 	}
 	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer))
 }

cmd/containerboot/serve.go

@@ -1,96 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"log"
-	"os"
-	"path/filepath"
-	"reflect"
-	"sync/atomic"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"tailscale.com/client/tailscale"
-	"tailscale.com/ipn"
-)
-
-// watchServeConfigChanges watches path for changes, and when it sees one, reads
-// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
-// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
-// is written to when the certDomain changes, causing the serve config to be
-// re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
-	if certDomainAtomic == nil {
-		panic("cd must not be nil")
-	}
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
-			log.Fatalf("failed to add fsnotify watch: %v", err)
-		}
-		eventChan = w.Events
-	}
-
-	var certDomain string
-	var prevServeConfig *ipn.ServeConfig
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-cdChanged:
-			certDomain = *certDomainAtomic.Load()
-		case <-tickChan:
-		case <-eventChan:
-			// We can't do any reasonable filtering on the event because of how
-			// k8s handles these mounts. So just re-read the file and apply it
-			// if it's changed.
-		}
-		if certDomain == "" {
-			continue
-		}
-		sc, err := readServeConfig(path, certDomain)
-		if err != nil {
-			log.Fatalf("failed to read serve config: %v", err)
-		}
-		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
-			continue
-		}
-		log.Printf("Applying serve config")
-		if err := lc.SetServeConfig(ctx, sc); err != nil {
-			log.Fatalf("failed to set serve config: %v", err)
-		}
-		prevServeConfig = sc
-	}
-}
-
-// readServeConfig reads the ipn.ServeConfig from path, replacing
-// ${TS_CERT_DOMAIN} with certDomain.
-func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
-	if path == "" {
-		return nil, nil
-	}
-	j, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
-	var sc ipn.ServeConfig
-	if err := json.Unmarshal(j, &sc); err != nil {
-		return nil, err
-	}
-	return &sc, nil
-}

cmd/containerboot/services.go

@@ -1,571 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"tailscale.com/ipn"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/kube/kubeclient"
-	"tailscale.com/tailcfg"
-	"tailscale.com/util/linuxfw"
-	"tailscale.com/util/mak"
-)
-
-const tailscaleTunInterface = "tailscale0"
-
-// This file contains functionality to run containerboot as a proxy that can
-// route cluster traffic to one or more tailnet targets, based on portmapping
-// rules read from a configfile. Currently (9/2024) this is only used for the
-// Kubernetes operator egress proxies.
-
-// egressProxy knows how to configure firewall rules to route cluster traffic to
-// one or more tailnet services.
-type egressProxy struct {
-	cfgPath string // path to egress service config file
-
-	nfr linuxfw.NetfilterRunner // never nil
-
-	kc          kubeclient.Client // never nil
-	stateSecret string            // name of the kube state Secret
-
-	netmapChan chan ipn.Notify // chan to receive netmap updates on
-
-	podIPv4 string // never empty string, currently only IPv4 is supported
-
-	// tailnetFQDNs is the egress service FQDN to tailnet IP mappings that
-	// were last used to configure firewall rules for this proxy.
-	// TODO(irbekrm): target addresses are also stored in the state Secret.
-	// Evaluate whether we should retrieve them from there and not store in
-	// memory at all.
-	targetFQDNs map[string][]netip.Prefix
-
-	// used to configure firewall rules.
-	tailnetAddrs []netip.Prefix
-}
-
-// run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when:
-// - the mounted egress config has changed
-// - the proxy's tailnet IP addresses have changed
-// - tailnet IPs have changed for any backend targets specified by tailnet FQDN
-func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	// TODO (irbekrm): take a look if this can be pulled into a single func
-	// shared with serve config loader.
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil {
-			return fmt.Errorf("failed to add fsnotify watch: %w", err)
-		}
-		eventChan = w.Events
-	}
-
-	if err := ep.sync(ctx, n); err != nil {
-		return err
-	}
-	for {
-		var err error
-		select {
-		case <-ctx.Done():
-			return nil
-		case <-tickChan:
-			err = ep.sync(ctx, n)
-		case <-eventChan:
-			log.Printf("config file change detected, ensuring firewall config is up to date...")
-			err = ep.sync(ctx, n)
-		case n = <-ep.netmapChan:
-			shouldResync := ep.shouldResync(n)
-			if shouldResync {
-				log.Printf("netmap change detected, ensuring firewall config is up to date...")
-				err = ep.sync(ctx, n)
-			}
-		}
-		if err != nil {
-			return fmt.Errorf("error syncing egress service config: %w", err)
-		}
-	}
-}
-
-// sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if
-// any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current
-// firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such
-// as failed firewall update
-func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error {
-	cfgs, err := ep.getConfigs()
-	if err != nil {
-		return fmt.Errorf("error retrieving egress service configs: %w", err)
-	}
-	status, err := ep.getStatus(ctx)
-	if err != nil {
-		return fmt.Errorf("error retrieving current egress proxy status: %w", err)
-	}
-	newStatus, err := ep.syncEgressConfigs(cfgs, status, n)
-	if err != nil {
-		return fmt.Errorf("error syncing egress service configs: %w", err)
-	}
-	if !servicesStatusIsEqual(newStatus, status) {
-		if err := ep.setStatus(ctx, newStatus, n); err != nil {
-			return fmt.Errorf("error setting egress proxy status: %w", err)
-		}
-	}
-	return nil
-}
-
-// addrsHaveChanged returns true if the provided netmap update contains tailnet address change for this proxy node.
-// Netmap must not be nil.
-func (ep *egressProxy) addrsHaveChanged(n ipn.Notify) bool {
-	return !reflect.DeepEqual(ep.tailnetAddrs, n.NetMap.SelfNode.Addresses())
-}
-
-// syncEgressConfigs adds and deletes firewall rules to match the desired
-// configuration. It uses the provided status to determine what is currently
-// applied and updates the status after a successful sync.
-func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *egressservices.Status, n ipn.Notify) (*egressservices.Status, error) {
-	if !(wantsServicesConfigured(cfgs) || hasServicesConfigured(status)) {
-		return nil, nil
-	}
-
-	// Delete unnecessary services.
-	if err := ep.deleteUnnecessaryServices(cfgs, status); err != nil {
-		return nil, fmt.Errorf("error deleting services: %w", err)
-
-	}
-	newStatus := &egressservices.Status{}
-	if !wantsServicesConfigured(cfgs) {
-		return newStatus, nil
-	}
-
-	// Add new services, update rules for any that have changed.
-	rulesPerSvcToAdd := make(map[string][]rule, 0)
-	rulesPerSvcToDelete := make(map[string][]rule, 0)
-	for svcName, cfg := range *cfgs {
-		tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, n)
-		if err != nil {
-			return nil, fmt.Errorf("error determining tailnet target IPs: %w", err)
-		}
-		rulesToAdd, rulesToDelete, err := updatesForCfg(svcName, cfg, status, tailnetTargetIPs)
-		if err != nil {
-			return nil, fmt.Errorf("error validating service changes: %v", err)
-		}
-		log.Printf("syncegressservices: looking at svc %s rulesToAdd %d rulesToDelete %d", svcName, len(rulesToAdd), len(rulesToDelete))
-		if len(rulesToAdd) != 0 {
-			mak.Set(&rulesPerSvcToAdd, svcName, rulesToAdd)
-		}
-		if len(rulesToDelete) != 0 {
-			mak.Set(&rulesPerSvcToDelete, svcName, rulesToDelete)
-		}
-		if len(rulesToAdd) != 0 || ep.addrsHaveChanged(n) {
-			// For each tailnet target, set up SNAT from the local tailnet device address of the matching
-			// family.
-			for _, t := range tailnetTargetIPs {
-				var local netip.Addr
-				for _, pfx := range n.NetMap.SelfNode.Addresses().All() {
-					if !pfx.IsSingleIP() {
-						continue
-					}
-					if pfx.Addr().Is4() != t.Is4() {
-						continue
-					}
-					local = pfx.Addr()
-					break
-				}
-				if !local.IsValid() {
-					return nil, fmt.Errorf("no valid local IP: %v", local)
-				}
-				if err := ep.nfr.EnsureSNATForDst(local, t); err != nil {
-					return nil, fmt.Errorf("error setting up SNAT rule: %w", err)
-				}
-			}
-		}
-		// Update the status. Status will be written back to the state Secret by the caller.
-		mak.Set(&newStatus.Services, svcName, &egressservices.ServiceStatus{TailnetTargetIPs: tailnetTargetIPs, TailnetTarget: cfg.TailnetTarget, Ports: cfg.Ports})
-	}
-
-	// Actually apply the firewall rules.
-	if err := ensureRulesAdded(rulesPerSvcToAdd, ep.nfr); err != nil {
-		return nil, fmt.Errorf("error adding rules: %w", err)
-	}
-	if err := ensureRulesDeleted(rulesPerSvcToDelete, ep.nfr); err != nil {
-		return nil, fmt.Errorf("error deleting rules: %w", err)
-	}
-
-	return newStatus, nil
-}
-
-// updatesForCfg calculates any rules that need to be added or deleted for an individucal egress service config.
-func updatesForCfg(svcName string, cfg egressservices.Config, status *egressservices.Status, tailnetTargetIPs []netip.Addr) ([]rule, []rule, error) {
-	rulesToAdd := make([]rule, 0)
-	rulesToDelete := make([]rule, 0)
-	currentConfig, ok := lookupCurrentConfig(svcName, status)
-
-	// If no rules for service are present yet, add them all.
-	if !ok {
-		for _, t := range tailnetTargetIPs {
-			for ports := range cfg.Ports {
-				log.Printf("syncegressservices: svc %s adding port %v", svcName, ports)
-				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: t})
-			}
-		}
-		return rulesToAdd, rulesToDelete, nil
-	}
-
-	// If there are no backend targets available, delete any currently configured rules.
-	if len(tailnetTargetIPs) == 0 {
-		log.Printf("tailnet target for egress service %s does not have any backend addresses, deleting all rules", svcName)
-		for _, ip := range currentConfig.TailnetTargetIPs {
-			for ports := range currentConfig.Ports {
-				rulesToDelete = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
-			}
-		}
-		return rulesToAdd, rulesToDelete, nil
-	}
-
-	// If there are rules present for backend targets that no longer match, delete them.
-	for _, ip := range currentConfig.TailnetTargetIPs {
-		var found bool
-		for _, wantsIP := range tailnetTargetIPs {
-			if reflect.DeepEqual(ip, wantsIP) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			for ports := range currentConfig.Ports {
-				rulesToDelete = append(rulesToDelete, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
-			}
-		}
-	}
-
-	// Sync rules for the currently wanted backend targets.
-	for _, ip := range tailnetTargetIPs {
-
-		// If the backend target is not yet present in status, add all rules.
-		var found bool
-		for _, gotIP := range currentConfig.TailnetTargetIPs {
-			if reflect.DeepEqual(ip, gotIP) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			for ports := range cfg.Ports {
-				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
-			}
-			continue
-		}
-
-		// If the backend target is present in status, check that the
-		// currently applied rules are up to date.
-
-		// Delete any current portmappings that are no longer present in config.
-		for port := range currentConfig.Ports {
-			if _, ok := cfg.Ports[port]; ok {
-				continue
-			}
-			rulesToDelete = append(rulesToDelete, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
-		}
-
-		// Add any new portmappings.
-		for port := range cfg.Ports {
-			if _, ok := currentConfig.Ports[port]; ok {
-				continue
-			}
-			rulesToAdd = append(rulesToAdd, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
-		}
-	}
-	return rulesToAdd, rulesToDelete, nil
-}
-
-// deleteUnneccessaryServices ensure that any services found on status, but not
-// present in config are deleted.
-func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, status *egressservices.Status) error {
-	if !hasServicesConfigured(status) {
-		return nil
-	}
-	if !wantsServicesConfigured(cfgs) {
-		for svcName, svc := range status.Services {
-			log.Printf("service %s is no longer required, deleting", svcName)
-			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
-				return fmt.Errorf("error deleting service %s: %w", svcName, err)
-			}
-		}
-		return nil
-	}
-
-	for svcName, svc := range status.Services {
-		if _, ok := (*cfgs)[svcName]; !ok {
-			log.Printf("service %s is no longer required, deleting", svcName)
-			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
-				return fmt.Errorf("error deleting service %s: %w", svcName, err)
-			}
-			// TODO (irbekrm): also delete the SNAT rule here
-		}
-	}
-	return nil
-}
-
-// getConfigs gets the mounted egress service configuration.
-func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) {
-	j, err := os.ReadFile(ep.cfgPath)
-	if os.IsNotExist(err) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	if len(j) == 0 || string(j) == "" {
-		return nil, nil
-	}
-	cfg := &egressservices.Configs{}
-	if err := json.Unmarshal(j, &cfg); err != nil {
-		return nil, err
-	}
-	return cfg, nil
-}
-
-// getStatus gets the current status of the configured firewall. The current
-// status is stored in state Secret. Returns nil status if no status that
-// applies to the current proxy Pod was found. Uses the Pod IP to determine if a
-// status found in the state Secret applies to this proxy Pod.
-func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, error) {
-	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
-	if err != nil {
-		return nil, fmt.Errorf("error retrieving state secret: %w", err)
-	}
-	status := &egressservices.Status{}
-	raw, ok := secret.Data[egressservices.KeyEgressServices]
-	if !ok {
-		return nil, nil
-	}
-	if err := json.Unmarshal([]byte(raw), status); err != nil {
-		return nil, fmt.Errorf("error unmarshalling previous config: %w", err)
-	}
-	if reflect.DeepEqual(status.PodIPv4, ep.podIPv4) {
-		return status, nil
-	}
-	return nil, nil
-}
-
-// setStatus writes egress proxy's currently configured firewall to the state
-// Secret and updates proxy's tailnet addresses.
-func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, n ipn.Notify) error {
-	// Pod IP is used to determine if a stored status applies to THIS proxy Pod.
-	if status == nil {
-		status = &egressservices.Status{}
-	}
-	status.PodIPv4 = ep.podIPv4
-	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
-	if err != nil {
-		return fmt.Errorf("error retrieving state Secret: %w", err)
-	}
-	bs, err := json.Marshal(status)
-	if err != nil {
-		return fmt.Errorf("error marshalling service config: %w", err)
-	}
-	secret.Data[egressservices.KeyEgressServices] = bs
-	patch := kubeclient.JSONPatch{
-		Op:    "replace",
-		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
-		Value: bs,
-	}
-	if err := ep.kc.JSONPatchSecret(ctx, ep.stateSecret, []kubeclient.JSONPatch{patch}); err != nil {
-		return fmt.Errorf("error patching state Secret: %w", err)
-	}
-	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
-	return nil
-}
-
-// tailnetTargetIPsForSvc returns the tailnet IPs to which traffic for this
-// egress service should be proxied. The egress service can be configured by IP
-// or by FQDN. If it's configured by IP, just return that. If it's configured by
-// FQDN, resolve the FQDN and return the resolved IPs. It checks if the
-// netfilter runner supports IPv6 NAT and skips any IPv6 addresses if it
-// doesn't.
-func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.Notify) (addrs []netip.Addr, err error) {
-	if svc.TailnetTarget.IP != "" {
-		addr, err := netip.ParseAddr(svc.TailnetTarget.IP)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing tailnet target IP: %w", err)
-		}
-		if addr.Is6() && !ep.nfr.HasIPV6NAT() {
-			log.Printf("tailnet target is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode. This will probably not work.")
-			return addrs, nil
-		}
-		return []netip.Addr{addr}, nil
-	}
-
-	if svc.TailnetTarget.FQDN == "" {
-		return nil, errors.New("unexpected egress service config- neither tailnet target IP nor FQDN is set")
-	}
-	if n.NetMap == nil {
-		log.Printf("netmap is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN)
-		return addrs, nil
-	}
-	var (
-		node      tailcfg.NodeView
-		nodeFound bool
-	)
-	for _, nn := range n.NetMap.Peers {
-		if equalFQDNs(nn.Name(), svc.TailnetTarget.FQDN) {
-			node = nn
-			nodeFound = true
-			break
-		}
-	}
-	if nodeFound {
-		for _, addr := range node.Addresses().AsSlice() {
-			if addr.Addr().Is6() && !ep.nfr.HasIPV6NAT() {
-				log.Printf("tailnet target %v is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode, skipping.", addr.Addr().String())
-				continue
-			}
-			addrs = append(addrs, addr.Addr())
-		}
-		// Egress target endpoints configured via FQDN are stored, so
-		// that we can determine if a netmap update should trigger a
-		// resync.
-		mak.Set(&ep.targetFQDNs, svc.TailnetTarget.FQDN, node.Addresses().AsSlice())
-	}
-	return addrs, nil
-}
-
-// shouldResync parses netmap update and returns true if the update contains
-// changes for which the egress proxy's firewall should be reconfigured.
-func (ep *egressProxy) shouldResync(n ipn.Notify) bool {
-	if n.NetMap == nil {
-		return false
-	}
-
-	// If proxy's tailnet addresses have changed, resync.
-	if !reflect.DeepEqual(n.NetMap.SelfNode.Addresses().AsSlice(), ep.tailnetAddrs) {
-		log.Printf("node addresses have changed, trigger egress config resync")
-		ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
-		return true
-	}
-
-	// If the IPs for any of the egress services configured via FQDN have
-	// changed, resync.
-	for fqdn, ips := range ep.targetFQDNs {
-		for _, nn := range n.NetMap.Peers {
-			if equalFQDNs(nn.Name(), fqdn) {
-				if !reflect.DeepEqual(ips, nn.Addresses().AsSlice()) {
-					log.Printf("backend addresses for egress target %q have changed old IPs %v, new IPs %v trigger egress config resync", nn.Name(), ips, nn.Addresses().AsSlice())
-				}
-				return true
-			}
-		}
-	}
-	return false
-}
-
-// ensureServiceDeleted ensures that any rules for an egress service are removed
-// from the firewall configuration.
-func ensureServiceDeleted(svcName string, svc *egressservices.ServiceStatus, nfr linuxfw.NetfilterRunner) error {
-
-	// Note that the portmap is needed for iptables based firewall only.
-	// Nftables group rules for a service in a chain, so there is no need to
-	// specify individual portmapping based rules.
-	pms := make([]linuxfw.PortMap, 0)
-	for pm := range svc.Ports {
-		pms = append(pms, linuxfw.PortMap{MatchPort: pm.MatchPort, TargetPort: pm.TargetPort, Protocol: pm.Protocol})
-	}
-
-	if err := nfr.DeleteSvc(svcName, tailscaleTunInterface, svc.TailnetTargetIPs, pms); err != nil {
-		return fmt.Errorf("error deleting service %s: %w", svcName, err)
-	}
-	return nil
-}
-
-// ensureRulesAdded ensures that all portmapping rules are added to the firewall
-// configuration. For any rules that already exist, calling this function is a
-// no-op. In case of nftables, a service consists of one or two (one per IP
-// family) chains that conain the portmapping rules for the service and the
-// chains as needed when this function is called.
-func ensureRulesAdded(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
-	for svc, rules := range rulesPerSvc {
-		for _, rule := range rules {
-			log.Printf("ensureRulesAdded svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
-			if err := nfr.EnsurePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
-				return fmt.Errorf("error ensuring rule: %w", err)
-			}
-		}
-	}
-	return nil
-}
-
-// ensureRulesDeleted ensures that the given rules are deleted from the firewall
-// configuration. For any rules that do not exist, calling this funcion is a
-// no-op.
-func ensureRulesDeleted(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
-	for svc, rules := range rulesPerSvc {
-		for _, rule := range rules {
-			log.Printf("ensureRulesDeleted svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
-			if err := nfr.DeletePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
-				return fmt.Errorf("error deleting rule: %w", err)
-			}
-		}
-	}
-	return nil
-}
-
-func lookupCurrentConfig(svcName string, status *egressservices.Status) (*egressservices.ServiceStatus, bool) {
-	if status == nil || len(status.Services) == 0 {
-		return nil, false
-	}
-	c, ok := status.Services[svcName]
-	return c, ok
-}
-
-func equalFQDNs(s, s1 string) bool {
-	s, _ = strings.CutSuffix(s, ".")
-	s1, _ = strings.CutSuffix(s1, ".")
-	return strings.EqualFold(s, s1)
-}
-
-// rule contains configuration for an egress proxy firewall rule.
-type rule struct {
-	containerPort uint16     // port to match incoming traffic
-	tailnetPort   uint16     // tailnet service port
-	tailnetIP     netip.Addr // tailnet service IP
-	protocol      string
-}
-
-func wantsServicesConfigured(cfgs *egressservices.Configs) bool {
-	return cfgs != nil && len(*cfgs) != 0
-}
-
-func hasServicesConfigured(status *egressservices.Status) bool {
-	return status != nil && len(status.Services) != 0
-}
-
-func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
-	if st == nil && st1 == nil {
-		return true
-	}
-	if st == nil || st1 == nil {
-		return false
-	}
-	st.PodIPv4 = ""
-	st1.PodIPv4 = ""
-	return reflect.DeepEqual(*st, *st1)
-}

cmd/containerboot/services_test.go

@@ -1,175 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"net/netip"
-	"reflect"
-	"testing"
-
-	"tailscale.com/kube/egressservices"
-)
-
-func Test_updatesForSvc(t *testing.T) {
-	tailnetIPv4, tailnetIPv6 := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
-	tailnetIPv4_1, tailnetIPv6_1 := netip.MustParseAddr("100.88.88.88"), netip.MustParseAddr("fd7a:115c:a1e0::4101:512f")
-	ports := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {}}
-	ports1 := map[egressservices.PortMap]struct{}{{Protocol: "udp", MatchPort: 4004, TargetPort: 53}: {}}
-	ports2 := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {},
-		{Protocol: "tcp", MatchPort: 4005, TargetPort: 443}: {}}
-	fqdnSpec := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
-		Ports:         ports,
-	}
-	fqdnSpec1 := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
-		Ports:         ports1,
-	}
-	fqdnSpec2 := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-		Ports:         ports,
-	}
-	fqdnSpec3 := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-		Ports:         ports2,
-	}
-	r := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4}
-	r1 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6}
-	r2 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv4}
-	r3 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv6}
-	r4 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4_1}
-	r5 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6_1}
-	r6 := rule{containerPort: 4005, tailnetPort: 443, protocol: "tcp", tailnetIP: tailnetIPv4}
-
-	tests := []struct {
-		name              string
-		svcName           string
-		tailnetTargetIPs  []netip.Addr
-		podIP             string
-		spec              egressservices.Config
-		status            *egressservices.Status
-		wantRulesToAdd    []rule
-		wantRulesToDelete []rule
-	}{
-		{
-			name:              "add_fqdn_svc_that_does_not_yet_exist",
-			svcName:           "test",
-			tailnetTargetIPs:  []netip.Addr{tailnetIPv4, tailnetIPv6},
-			spec:              fqdnSpec,
-			status:            &egressservices.Status{},
-			wantRulesToAdd:    []rule{r, r1},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "fqdn_svc_already_exists",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-			spec:             fqdnSpec,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "fqdn_svc_already_exists_add_port_remove_port",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-			spec:             fqdnSpec1,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{r2, r3},
-			wantRulesToDelete: []rule{r, r1},
-		},
-		{
-			name:             "fqdn_svc_already_exists_change_fqdn_backend_ips",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4_1, tailnetIPv6_1},
-			spec:             fqdnSpec,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{r4, r5},
-			wantRulesToDelete: []rule{r, r1},
-		},
-		{
-			name:              "add_ip_service",
-			svcName:           "test",
-			tailnetTargetIPs:  []netip.Addr{tailnetIPv4},
-			spec:              fqdnSpec2,
-			status:            &egressservices.Status{},
-			wantRulesToAdd:    []rule{r},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "add_ip_service_already_exists",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
-			spec:             fqdnSpec2,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
-					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "ip_service_add_port",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
-			spec:             fqdnSpec3,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
-					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{r6},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "ip_service_delete_port",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
-			spec:             fqdnSpec,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
-					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-					Ports:            ports2,
-				}}},
-			wantRulesToAdd:    []rule{},
-			wantRulesToDelete: []rule{r6},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			gotRulesToAdd, gotRulesToDelete, err := updatesForCfg(tt.svcName, tt.spec, tt.status, tt.tailnetTargetIPs)
-			if err != nil {
-				t.Errorf("updatesForSvc() unexpected error %v", err)
-				return
-			}
-			if !reflect.DeepEqual(gotRulesToAdd, tt.wantRulesToAdd) {
-				t.Errorf("updatesForSvc() got rulesToAdd = \n%v\n want rulesToAdd \n%v", gotRulesToAdd, tt.wantRulesToAdd)
-			}
-			if !reflect.DeepEqual(gotRulesToDelete, tt.wantRulesToDelete) {
-				t.Errorf("updatesForSvc() got rulesToDelete = \n%v\n want rulesToDelete \n%v", gotRulesToDelete, tt.wantRulesToDelete)
-			}
-		})
-	}
-}

cmd/containerboot/settings.go

@@ -1,324 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log"
-	"net/netip"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	"tailscale.com/ipn/conffile"
-	"tailscale.com/kube/kubeclient"
-)
-
-// settings is all the configuration for containerboot.
-type settings struct {
-	AuthKey  string
-	Hostname string
-	Routes   *string
-	// ProxyTargetIP is the destination IP to which all incoming
-	// Tailscale traffic should be proxied. If empty, no proxying
-	// is done. This is typically a locally reachable IP.
-	ProxyTargetIP string
-	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
-	// incoming Tailscale traffic should be proxied.
-	ProxyTargetDNSName string
-	// TailnetTargetIP is the destination IP to which all incoming
-	// non-Tailscale traffic should be proxied. This is typically a
-	// Tailscale IP.
-	TailnetTargetIP string
-	// TailnetTargetFQDN is an MagicDNS name to which all incoming
-	// non-Tailscale traffic should be proxied. This must be a full Tailnet
-	// node FQDN.
-	TailnetTargetFQDN             string
-	ServeConfigPath               string
-	DaemonExtraArgs               string
-	ExtraArgs                     string
-	InKubernetes                  bool
-	UserspaceMode                 bool
-	StateDir                      string
-	AcceptDNS                     *bool
-	KubeSecret                    string
-	SOCKSProxyAddr                string
-	HTTPProxyAddr                 string
-	Socket                        string
-	AuthOnce                      bool
-	Root                          string
-	KubernetesCanPatch            bool
-	TailscaledConfigFilePath      string
-	EnableForwardingOptimizations bool
-	// If set to true and, if this containerboot instance is a Kubernetes
-	// ingress proxy, set up rules to forward incoming cluster traffic to be
-	// forwarded to the ingress target in cluster.
-	AllowProxyingClusterTrafficViaIngress bool
-	// PodIP is the IP of the Pod if running in Kubernetes. This is used
-	// when setting up rules to proxy cluster traffic to cluster ingress
-	// target.
-	// Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters
-	PodIP               string
-	PodIPv4             string
-	PodIPv6             string
-	HealthCheckAddrPort string
-	EgressSvcsCfgPath   string
-}
-
-func configFromEnv() (*settings, error) {
-	cfg := &settings{
-		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
-		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
-		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
-		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
-		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
-		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
-		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
-		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
-		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
-		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
-		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
-		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
-		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
-		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
-		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
-		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
-		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
-		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
-		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
-		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
-		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
-		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
-		PodIP:                                 defaultEnv("POD_IP", ""),
-		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
-		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
-		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
-	}
-	podIPs, ok := os.LookupEnv("POD_IPS")
-	if ok {
-		ips := strings.Split(podIPs, ",")
-		if len(ips) > 2 {
-			return nil, fmt.Errorf("POD_IPs can contain at most 2 IPs, got %d (%v)", len(ips), ips)
-		}
-		for _, ip := range ips {
-			parsed, err := netip.ParseAddr(ip)
-			if err != nil {
-				return nil, fmt.Errorf("error parsing IP address %s: %w", ip, err)
-			}
-			if parsed.Is4() {
-				cfg.PodIPv4 = parsed.String()
-				continue
-			}
-			cfg.PodIPv6 = parsed.String()
-		}
-	}
-	if err := cfg.validate(); err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v", err)
-	}
-	return cfg, nil
-}
-
-func (s *settings) validate() error {
-	if s.TailscaledConfigFilePath != "" {
-		dir, file := path.Split(s.TailscaledConfigFilePath)
-		if _, err := os.Stat(dir); err != nil {
-			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
-		}
-		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
-		}
-		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
-		}
-	}
-	if s.ProxyTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
-	}
-	if s.TailnetTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
-		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
-	}
-	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
-	}
-	if s.EnableForwardingOptimizations && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
-	}
-	if s.HealthCheckAddrPort != "" {
-		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
-			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
-		}
-	}
-	return nil
-}
-
-// setupKube is responsible for doing any necessary configuration and checks to
-// ensure that tailscale state storage and authentication mechanism will work on
-// Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
-	if cfg.KubeSecret == "" {
-		return nil
-	}
-	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
-	if err != nil {
-		return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-	}
-	cfg.KubernetesCanPatch = canPatch
-
-	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
-	if err != nil {
-		if !kubeclient.IsNotFoundErr(err) {
-			return fmt.Errorf("getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
-		}
-
-		if !canCreate {
-			return fmt.Errorf("tailscale state Secret %s does not exist and we don't have permissions to create it. "+
-				"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
-				"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
-				"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
-		}
-	}
-
-	// Return early if we already have an auth key.
-	if cfg.AuthKey != "" || isOneStepConfig(cfg) {
-		return nil
-	}
-
-	if s == nil {
-		log.Print("TS_AUTHKEY not provided and state Secret does not exist, login will be interactive if needed.")
-		return nil
-	}
-
-	keyBytes, _ := s.Data["authkey"]
-	key := string(keyBytes)
-
-	if key != "" {
-		// Enforce that we must be able to patch out the authkey after
-		// authenticating if you want to use this feature. This avoids
-		// us having to deal with the case where we might leave behind
-		// an unnecessary reusable authkey in a secret, like a rake in
-		// the grass.
-		if !cfg.KubernetesCanPatch {
-			return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the Secret to manage the authkey.")
-		}
-		cfg.AuthKey = key
-	}
-
-	log.Print("No authkey found in state Secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-
-	return nil
-}
-
-// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
-// in two steps and login should only happen once.
-// Step 1: run 'tailscaled'
-// Step 2):
-// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
-// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
-func isTwoStepConfigAuthOnce(cfg *settings) bool {
-	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
-// in two steps and we should log in every time it starts.
-// Step 1: run 'tailscaled'
-// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
-func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
-	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isOneStepConfig returns true if the Tailscale node should always be ran and
-// configured in a single step by running 'tailscaled <config opts>'
-func isOneStepConfig(cfg *settings) bool {
-	return cfg.TailscaledConfigFilePath != ""
-}
-
-// isL3Proxy returns true if the Tailscale node needs to be configured to act
-// as an L3 proxy, proxying to an endpoint provided via one of the config env
-// vars.
-func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != ""
-}
-
-// hasKubeStateStore returns true if the state must be stored in a Kubernetes
-// Secret.
-func hasKubeStateStore(cfg *settings) bool {
-	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
-}
-
-// defaultEnv returns the value of the given envvar name, or defVal if
-// unset.
-func defaultEnv(name, defVal string) string {
-	if v, ok := os.LookupEnv(name); ok {
-		return v
-	}
-	return defVal
-}
-
-// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being set to empty string vs unset.
-func defaultEnvStringPointer(name string) *string {
-	if v, ok := os.LookupEnv(name); ok {
-		return &v
-	}
-	return nil
-}
-
-// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being explicitly set to false vs unset.
-func defaultEnvBoolPointer(name string) *bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return nil
-	}
-	return &ret
-}
-
-func defaultEnvs(names []string, defVal string) string {
-	for _, name := range names {
-		if v, ok := os.LookupEnv(name); ok {
-			return v
-		}
-	}
-	return defVal
-}
-
-// defaultBool returns the boolean value of the given envvar name, or
-// defVal if unset or not a bool.
-func defaultBool(name string, defVal bool) bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return defVal
-	}
-	return ret
-}

cmd/containerboot/tailscaled.go

@@ -1,162 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log"
-	"os"
-	"os/exec"
-	"strings"
-	"syscall"
-	"time"
-
-	"tailscale.com/client/tailscale"
-)
-
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
-	args := tailscaledArgs(cfg)
-	// tailscaled runs without context, since it needs to persist
-	// beyond the startup timeout in ctx.
-	cmd := exec.Command("tailscaled", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-	log.Printf("Starting tailscaled")
-	if err := cmd.Start(); err != nil {
-		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
-	}
-
-	// Wait for the socket file to appear, otherwise API ops will racily fail.
-	log.Printf("Waiting for tailscaled socket")
-	for {
-		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
-		}
-		_, err := os.Stat(cfg.Socket)
-		if errors.Is(err, fs.ErrNotExist) {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
-		}
-		break
-	}
-
-	tsClient := &tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
-	return tsClient, cmd.Process, nil
-}
-
-// tailscaledArgs uses cfg to construct the argv for tailscaled.
-func tailscaledArgs(cfg *settings) []string {
-	args := []string{"--socket=" + cfg.Socket}
-	switch {
-	case cfg.InKubernetes && cfg.KubeSecret != "":
-		args = append(args, "--state=kube:"+cfg.KubeSecret)
-		if cfg.StateDir == "" {
-			cfg.StateDir = "/tmp"
-		}
-		fallthrough
-	case cfg.StateDir != "":
-		args = append(args, "--statedir="+cfg.StateDir)
-	default:
-		args = append(args, "--state=mem:", "--statedir=/tmp")
-	}
-
-	if cfg.UserspaceMode {
-		args = append(args, "--tun=userspace-networking")
-	} else if err := ensureTunFile(cfg.Root); err != nil {
-		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
-	}
-
-	if cfg.SOCKSProxyAddr != "" {
-		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
-	}
-	if cfg.HTTPProxyAddr != "" {
-		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
-	}
-	if cfg.TailscaledConfigFilePath != "" {
-		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
-	}
-	if cfg.DaemonExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
-	}
-	return args
-}
-
-// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
-// if TS_AUTH_ONCE is set, only the first time containerboot starts.
-func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	if cfg.AuthKey != "" {
-		args = append(args, "--authkey="+cfg.AuthKey)
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
-	}
-	return nil
-}
-
-// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
-// options that are passed in via environment variables. This is run after the
-// node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}

cmd/derper/depaware.txt

@@ -7,14 +7,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
-        github.com/coder/websocket                                   from tailscale.com/cmd/derper+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
@@ -52,7 +48,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    github.com/tailscale/go-winio/internal/stringbuffer          from github.com/tailscale/go-winio/internal/fs
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
-   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
@@ -86,6 +82,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         google.golang.org/protobuf/runtime/protoiface                from google.golang.org/protobuf/internal/impl+
         google.golang.org/protobuf/runtime/protoimpl                 from github.com/prometheus/client_model/go+
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
+        nhooyr.io/websocket                                          from tailscale.com/cmd/derper+
+        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
+        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/cmd/derper+
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -99,7 +99,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
-        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
@@ -113,7 +112,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
-        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
@@ -129,7 +127,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -148,11 +146,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
-     💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
-     💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
@@ -163,17 +159,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
-        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
-        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
-        tailscale.com/util/testenv                                   from tailscale.com/util/syspolicy+
-        tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
@@ -184,17 +171,15 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from tailscale.com/tka
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/hkdf                                     from crypto/tls+
+        golang.org/x/crypto/hkdf                                     from crypto/tls
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
-        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -255,7 +240,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper+
+        flag                                                         from tailscale.com/cmd/derper
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -266,7 +251,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
-        iter                                                         from maps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/ipn+
@@ -282,14 +266,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         net/http                                                     from expvar+
         net/http/httptrace                                           from net/http+
         net/http/internal                                            from net/http
-        net/http/pprof                                               from tailscale.com/tsweb
+        net/http/pprof                                               from tailscale.com/tsweb+
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
-   W    os/user                                                      from tailscale.com/util/winutil+
+   W    os/user                                                      from tailscale.com/util/winutil
         path                                                         from github.com/prometheus/client_golang/prometheus/internal+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
@@ -311,4 +295,3 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
-        unique                                                       from net/netip

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
+				port80mux.HandleFunc("/generate_204", serveNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,6 +378,31 @@ func main() {
 	}
 }
 
+const (
+	noContentChallengeHeader = "X-Tailscale-Challenge"
+	noContentResponseHeader  = "X-Tailscale-Response"
+)
+
+// For captive portal detection
+func serveNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(noContentResponseHeader, "response "+challenge)
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 	"testing"
 
-	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -77,20 +76,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
+				req.Header.Set(noContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			derphttp.ServeNoContent(w, req)
+			serveNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
+				if h, found := resp.Header[noContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/derper/websocket.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/coder/websocket"
+	"nhooyr.io/websocket"
 	"tailscale.com/derp"
 	"tailscale.com/net/wsconn"
 )

cmd/derpprobe/derpprobe.go

@@ -7,6 +7,8 @@ package main
 import (
 	"flag"
 	"fmt"
+	"html"
+	"io"
 	"log"
 	"net/http"
 	"sort"
@@ -68,18 +70,8 @@ func main() {
 	}
 
 	mux := http.NewServeMux()
-	d := tsweb.Debugger(mux)
-	d.Handle("probe-run", "Run a probe", tsweb.StdHandler(tsweb.ReturnHandlerFunc(p.RunHandler), tsweb.HandlerOptions{Logf: log.Printf}))
-	mux.Handle("/", tsweb.StdHandler(p.StatusHandler(
-		prober.WithTitle("DERP Prober"),
-		prober.WithPageLink("Prober metrics", "/debug/varz"),
-		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
-	), tsweb.HandlerOptions{Logf: log.Printf}))
-	mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "text/plain")
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ok\n"))
-	}))
+	tsweb.Debugger(mux)
+	mux.HandleFunc("/", http.HandlerFunc(serveFunc(p)))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }
@@ -113,3 +105,26 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 	sort.Strings(o.good)
 	return
 }
+
+func serveFunc(p *prober.Prober) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		st := getOverallStatus(p)
+		summary := "All good"
+		if (float64(len(st.bad)) / float64(len(st.bad)+len(st.good))) > 0.25 {
+			// Returning a 500 allows monitoring this server externally and configuring
+			// an alert on HTTP response code.
+			w.WriteHeader(500)
+			summary = fmt.Sprintf("%d problems", len(st.bad))
+		}
+
+		io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
+		fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
+		for _, s := range st.bad {
+			fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
+		}
+		for _, s := range st.good {
+			fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
+		}
+		io.WriteString(w, "</ul></body></html>\n")
+	}
+}

cmd/get-authkey/main.go

@@ -51,7 +51,6 @@ func main() {
 
 	ctx := context.Background()
 	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-get-authkey"
 	tsClient.HTTPClient = credentials.Client(ctx)
 	tsClient.BaseURL = baseURL
 

cmd/gitops-pusher/gitops-pusher.go

@@ -28,20 +28,19 @@ import (
 )
 
 var (
-	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
-	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
+	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
 )
 
-func modifiedExternallyError() error {
+func modifiedExternallyError() {
 	if *githubSyntax {
-		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
+		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
 	} else {
-		return fmt.Errorf("The policy file was modified externally in the admin console.")
+		fmt.Printf("The policy file was modified externally in the admin console.\n")
 	}
 }
 
@@ -66,22 +65,16 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
+		if cache.PrevETag != controlEtag {
+			modifiedExternallyError()
+		}
+
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
-		if cache.PrevETag != controlEtag {
-			if err := modifiedExternallyError(); err != nil {
-				if *failOnManualEdits {
-					return err
-				} else {
-					fmt.Println(err)
-				}
-			}
-		}
-
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -113,21 +106,15 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
+		if cache.PrevETag != controlEtag {
+			modifiedExternallyError()
+		}
+
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
-		if cache.PrevETag != controlEtag {
-			if err := modifiedExternallyError(); err != nil {
-				if *failOnManualEdits {
-					return err
-				} else {
-					fmt.Println(err)
-				}
-			}
-		}
-
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/k8s-operator/connector.go

@@ -26,7 +26,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -62,11 +61,11 @@ type ConnectorReconciler struct {
 
 var (
 	// gaugeConnectorResources tracks the overall number of Connectors currently managed by this operator instance.
-	gaugeConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorResourceCount)
+	gaugeConnectorResources = clientmetric.NewGauge("k8s_connector_resources")
 	// gaugeConnectorSubnetRouterResources tracks the number of Connectors managed by this operator instance that are subnet routers.
-	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
+	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
-	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
+	gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {

cmd/k8s-operator/connector_test.go

@@ -16,7 +16,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/util/mak"
 )
@@ -75,7 +74,6 @@ func TestConnector(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
-		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -171,7 +169,6 @@ func TestConnector(t *testing.T) {
 		parentType:   "connector",
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
-		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -257,7 +254,6 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		hostname:     "test-connector",
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
-		app:          kubetypes.AppConnector,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@@ -278,7 +274,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassReady),
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/depaware.txt

@@ -5,6 +5,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+  LD    github.com/anmitsu/go-shlex                                  from tailscale.com/tempfork/gliderlabs/ssh
    L    github.com/aws/aws-sdk-go-v2/aws                             from github.com/aws/aws-sdk-go-v2/aws/defaults+
    L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/aws/defaults                    from github.com/aws/aws-sdk-go-v2/service/ssm+
@@ -80,11 +81,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
+  LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
    W 💣 github.com/dblohm7/wingoes/com                               from tailscale.com/util/osdiag+
@@ -100,7 +98,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 github.com/fsnotify/fsnotify                                 from sigs.k8s.io/controller-runtime/pkg/certwatcher
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/gaissmai/bart                                     from tailscale.com/net/ipset+
-        github.com/go-json-experiment/json                           from tailscale.com/types/opt+
+        github.com/go-json-experiment/json                           from tailscale.com/types/opt
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext+
@@ -115,7 +113,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/go-openapi/jsonreference                          from k8s.io/kube-openapi/pkg/internal+
         github.com/go-openapi/jsonreference/internal                 from github.com/go-openapi/jsonreference
         github.com/go-openapi/swag                                   from github.com/go-openapi/jsonpointer+
-   L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+   L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns+
      💣 github.com/gogo/protobuf/proto                               from k8s.io/api/admission/v1+
         github.com/gogo/protobuf/sortkeys                            from k8s.io/api/admission/v1+
         github.com/golang/groupcache/lru                             from k8s.io/client-go/tools/record+
@@ -143,7 +141,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
-   L 💣 github.com/illarion/gonotify/v2                              from tailscale.com/net/dns
+   L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
         github.com/imdario/mergo                                     from k8s.io/client-go/tools/clientcmd
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
@@ -163,6 +161,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/klauspost/compress/zstd                           from tailscale.com/util/zstdframe
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
         github.com/kortschak/wol                                     from tailscale.com/ipn/ipnlocal
+  LD    github.com/kr/fs                                             from github.com/pkg/sftp
         github.com/mailru/easyjson/buffer                            from github.com/mailru/easyjson/jwriter
      💣 github.com/mailru/easyjson/jlexer                            from github.com/go-openapi/swag
         github.com/mailru/easyjson/jwriter                           from github.com/go-openapi/swag
@@ -171,7 +170,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/netlink/nltest                           from github.com/google/nftables
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink+
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
         github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
         github.com/modern-go/concurrent                              from github.com/json-iterator/go
@@ -184,6 +183,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/pierrec/lz4/v4/internal/lz4stream                 from github.com/pierrec/lz4/v4
    L    github.com/pierrec/lz4/v4/internal/xxh32                     from github.com/pierrec/lz4/v4/internal/lz4stream
         github.com/pkg/errors                                        from github.com/evanphx/json-patch/v5+
+  LD    github.com/pkg/sftp                                          from tailscale.com/ssh/tailssh
+  LD    github.com/pkg/sftp/internal/encoding/ssh/filexfer           from github.com/pkg/sftp
    D    github.com/prometheus-community/pro-bing                     from tailscale.com/wgengine/netstack
      💣 github.com/prometheus/client_golang/prometheus               from github.com/prometheus/client_golang/prometheus/collectors+
         github.com/prometheus/client_golang/prometheus/collectors    from sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics
@@ -206,7 +207,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    W    github.com/tailscale/go-winio/pkg/guid                       from github.com/tailscale/go-winio+
         github.com/tailscale/golang-x-crypto/acme                    from tailscale.com/ipn/ipnlocal
   LD    github.com/tailscale/golang-x-crypto/internal/poly1305       from github.com/tailscale/golang-x-crypto/ssh
-  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal
+  LD    github.com/tailscale/golang-x-crypto/ssh                     from tailscale.com/ipn/ipnlocal+
   LD    github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh
         github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2+
         github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
@@ -216,7 +217,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tailscale/hujson                                  from tailscale.com/ipn/conffile
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/net/routetable+
-   L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/peercred                                from tailscale.com/ipn/ipnauth
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
      💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
@@ -230,8 +230,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device
      💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
+  LD    github.com/u-root/u-root/pkg/termios                         from tailscale.com/ssh/tailssh
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
         go.uber.org/multierr                                         from go.uber.org/zap+
@@ -305,16 +307,16 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
         gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/link/channel                     from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
         gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
-        gvisor.dev/gvisor/pkg/tcpip/stack/gro                        from tailscale.com/wgengine/netstack/gro
         gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
         gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
@@ -421,7 +423,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/apimachinery/pkg/util/naming                          from k8s.io/apimachinery/pkg/runtime+
         k8s.io/apimachinery/pkg/util/net                             from k8s.io/apimachinery/pkg/watch+
         k8s.io/apimachinery/pkg/util/rand                            from k8s.io/apiserver/pkg/storage/names
-        k8s.io/apimachinery/pkg/util/remotecommand                   from tailscale.com/k8s-operator/sessionrecording/ws
         k8s.io/apimachinery/pkg/util/runtime                         from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
         k8s.io/apimachinery/pkg/util/sets                            from k8s.io/apimachinery/pkg/api/meta+
         k8s.io/apimachinery/pkg/util/strategicpatch                  from k8s.io/client-go/tools/record+
@@ -599,6 +600,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/utils/ptr                                             from k8s.io/client-go/tools/cache+
         k8s.io/utils/strings/slices                                  from k8s.io/apimachinery/pkg/labels
         k8s.io/utils/trace                                           from k8s.io/client-go/tools/cache
+        nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
+        nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+        nhooyr.io/websocket/internal/util                            from nhooyr.io/websocket
+        nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
         sigs.k8s.io/controller-runtime/pkg/builder                   from tailscale.com/cmd/k8s-operator
         sigs.k8s.io/controller-runtime/pkg/cache                     from sigs.k8s.io/controller-runtime/pkg/cluster+
         sigs.k8s.io/controller-runtime/pkg/cache/internal            from sigs.k8s.io/controller-runtime/pkg/cache
@@ -654,7 +659,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
-  LW    tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+  LD    tailscale.com/cmd/tailscaled/childproc                       from tailscale.com/ssh/tailssh
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
@@ -668,7 +674,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -687,14 +692,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/k8s-operator                                   from tailscale.com/cmd/k8s-operator
         tailscale.com/k8s-operator/apis                              from tailscale.com/k8s-operator/apis/v1alpha1
         tailscale.com/k8s-operator/apis/v1alpha1                     from tailscale.com/cmd/k8s-operator+
-        tailscale.com/k8s-operator/sessionrecording                  from tailscale.com/cmd/k8s-operator
-        tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
-        tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
-        tailscale.com/k8s-operator/sessionrecording/ws               from tailscale.com/k8s-operator/sessionrecording
-        tailscale.com/kube/egressservices                            from tailscale.com/cmd/k8s-operator
-        tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
-        tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
-        tailscale.com/kube/kubetypes                                 from tailscale.com/cmd/k8s-operator+
+        tailscale.com/kube                                           from tailscale.com/cmd/k8s-operator+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -703,7 +701,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
         tailscale.com/logtail/filch                                  from tailscale.com/log/sockstatlog+
         tailscale.com/metrics                                        from tailscale.com/derp+
-        tailscale.com/net/captivedetection                           from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns+
@@ -735,7 +732,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
-        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
@@ -747,19 +743,19 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
         tailscale.com/proxymap                                       from tailscale.com/tsd+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
-        tailscale.com/sessionrecording                               from tailscale.com/k8s-operator/sessionrecording+
+     💣 tailscale.com/ssh/tailssh                                    from tailscale.com/cmd/k8s-operator
         tailscale.com/syncs                                          from tailscale.com/control/controlknobs+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
+  LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-        tailscale.com/tsconst                                        from tailscale.com/net/netmon+
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon+
         tailscale.com/tsd                                            from tailscale.com/ipn/ipnlocal+
-        tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator+
+        tailscale.com/tsnet                                          from tailscale.com/cmd/k8s-operator
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
-        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
@@ -801,7 +797,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/osdiag                                    from tailscale.com/ipn/localapi
    W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
         tailscale.com/util/osshare                                   from tailscale.com/ipn/ipnlocal
-        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal
+        tailscale.com/util/osuser                                    from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/progresstracking                          from tailscale.com/ipn/localapi
         tailscale.com/util/race                                      from tailscale.com/net/dns/resolver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -811,22 +807,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
-        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
-        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
         tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
-        tailscale.com/util/usermetric                                from tailscale.com/health+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+
@@ -839,7 +828,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/tsnet
-        tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -849,7 +837,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
-  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf
+  LD    golang.org/x/crypto/blowfish                                 from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf+
         golang.org/x/crypto/chacha20                                 from github.com/tailscale/golang-x-crypto/ssh+
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
@@ -858,9 +846,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/hkdf                                     from crypto/tls+
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
+        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
+  LD    golang.org/x/crypto/ssh                                      from github.com/pkg/sftp+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from sigs.k8s.io/controller-runtime/pkg/cache+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
@@ -877,7 +865,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/net/websocket                                   from tailscale.com/k8s-operator/sessionrecording/ws
         golang.org/x/oauth2                                          from golang.org/x/oauth2/clientcredentials+
         golang.org/x/oauth2/clientcredentials                        from tailscale.com/cmd/k8s-operator
         golang.org/x/oauth2/internal                                 from golang.org/x/oauth2+
@@ -962,11 +949,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
-        iter                                                         from go/ast+
         log                                                          from expvar+
         log/internal                                                 from log+
         log/slog                                                     from github.com/go-logr/logr+
         log/slog/internal                                            from log/slog
+  LD    log/syslog                                                   from tailscale.com/ssh/tailssh
         maps                                                         from sigs.k8s.io/controller-runtime/pkg/predicate+
         math                                                         from archive/tar+
         math/big                                                     from crypto/dsa+
@@ -1000,7 +987,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from encoding/base32+
-        sort                                                         from compress/flate+
+        sort                                                         from archive/tar+
         strconv                                                      from archive/tar+
         strings                                                      from archive/tar+
         sync                                                         from archive/tar+
@@ -1013,4 +1000,3 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+
         unicode/utf8                                                 from bufio+
-        unique                                                       from net/netip

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -77,10 +77,6 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
-            {{- if .Values.proxyConfig.defaultProxyClass }}
-            - name: PROXY_DEFAULT_CLASS
-              value: {{ .Values.proxyConfig.defaultProxyClass }}
-            {{- end }}
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -14,22 +14,19 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["events", "services", "services/status"]
-  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+  verbs: ["*"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingresses", "ingresses/status"]
-  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+  verbs: ["*"]
 - apiGroups: ["networking.k8s.io"]
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["tailscale.com"]
-  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status", "proxygroups", "proxygroups/status"]
+  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
   verbs: ["get", "list", "watch", "update"]
 - apiGroups: ["tailscale.com"]
   resources: ["dnsconfigs", "dnsconfigs/status"]
   verbs: ["get", "list", "watch", "update"]
-- apiGroups: ["tailscale.com"]
-  resources: ["recorders", "recorders/status"]
-  verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,19 +49,13 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
-  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get","list","watch"]
+  verbs: ["*"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
-  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+  verbs: ["*"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
-  verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["roles", "rolebindings"]
-  verbs: ["get", "create", "patch", "update", "list", "watch"]
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -15,7 +15,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+  verbs: ["*"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -57,12 +57,12 @@ operatorConfig:
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.
-# https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
-# https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
+# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
+# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
 # Note that this section contains only a few global configuration options and
 # will not be updated with more configuration options in the future.
 # If you need more configuration options, take a look at ProxyClass:
-# https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
+# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
@@ -78,14 +78,10 @@ proxyConfig:
   # Note that if you pass multiple tags to this field via `--set` flag to helm upgrade/install commands you must escape the comma (for example, "tag:k8s-proxies\,tag:prod"). See https://github.com/helm/helm/issues/1556
   defaultTags: "tag:k8s"
   firewallMode: auto
-  # If defined, this proxy class will be used as the default proxy class for
-  # service and ingress resources that do not have a proxy class defined. It
-  # does not apply to Connector resources.
-  defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.
-# https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
+# https://tailscale.com/kb/1236/kubernetes-operator/#accessing-the-kubernetes-control-plane-using-an-api-server-proxy
 apiServerProxyConfig:
   mode: "false" # "true", "false", "noauth"
 

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -37,7 +37,7 @@ spec:
             exit node.
             Connector is a cluster-scoped resource.
             More info:
-            https://tailscale.com/kb/1441/kubernetes-operator-connector
+            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
           type: object
           required:
             - spec
@@ -115,7 +115,7 @@ spec:
                     To autoapprove the subnet routes or exit node defined by a Connector,
                     you can configure Tailscale ACLs to give these tags the necessary
                     permissions.
-                    See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
+                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
                     If you specify custom tags here, you must also make the operator an owner of these tags.
                     See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
                     Tags cannot be changed once a Connector node has been created.

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -89,14 +89,14 @@ spec:
                   type: object
                   properties:
                     image:
-                      description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
+                      description: Nameserver image.
                       type: object
                       properties:
                         repo:
                           description: Repo defaults to tailscale/k8s-nameserver.
                           type: string
                         tag:
-                          description: Tag defaults to unstable.
+                          description: Tag defaults to operator's own tag.
                           type: string
             status:
               description: |-

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -30,7 +30,7 @@ spec:
             connector.spec.proxyClass field.
             ProxyClass is a cluster scoped resource.
             More info:
-            https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
+            https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource.
           type: object
           required:
             - spec
@@ -1896,182 +1896,6 @@ spec:
                                   Value is the taint value the toleration matches to.
                                   If the operator is Exists, the value should be empty, otherwise just a regular string.
                                 type: string
-                        topologySpreadConstraints:
-                          description: |-
-                            Proxy Pod's topology spread constraints.
-                            By default Tailscale Kubernetes operator does not apply any topology spread constraints.
-                            https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
-                          type: array
-                          items:
-                            description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
-                            type: object
-                            required:
-                              - maxSkew
-                              - topologyKey
-                              - whenUnsatisfiable
-                            properties:
-                              labelSelector:
-                                description: |-
-                                  LabelSelector is used to find matching pods.
-                                  Pods that match this label selector are counted to determine the number of pods
-                                  in their corresponding topology domain.
-                                type: object
-                                properties:
-                                  matchExpressions:
-                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
-                                    type: array
-                                    items:
-                                      description: |-
-                                        A label selector requirement is a selector that contains values, a key, and an operator that
-                                        relates the key and values.
-                                      type: object
-                                      required:
-                                        - key
-                                        - operator
-                                      properties:
-                                        key:
-                                          description: key is the label key that the selector applies to.
-                                          type: string
-                                        operator:
-                                          description: |-
-                                            operator represents a key's relationship to a set of values.
-                                            Valid operators are In, NotIn, Exists and DoesNotExist.
-                                          type: string
-                                        values:
-                                          description: |-
-                                            values is an array of string values. If the operator is In or NotIn,
-                                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                            the values array must be empty. This array is replaced during a strategic
-                                            merge patch.
-                                          type: array
-                                          items:
-                                            type: string
-                                          x-kubernetes-list-type: atomic
-                                    x-kubernetes-list-type: atomic
-                                  matchLabels:
-                                    description: |-
-                                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                                    type: object
-                                    additionalProperties:
-                                      type: string
-                                x-kubernetes-map-type: atomic
-                              matchLabelKeys:
-                                description: |-
-                                  MatchLabelKeys is a set of pod label keys to select the pods over which
-                                  spreading will be calculated. The keys are used to lookup values from the
-                                  incoming pod labels, those key-value labels are ANDed with labelSelector
-                                  to select the group of existing pods over which spreading will be calculated
-                                  for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
-                                  MatchLabelKeys cannot be set when LabelSelector isn't set.
-                                  Keys that don't exist in the incoming pod labels will
-                                  be ignored. A null or empty list means only match against labelSelector.
-
-                                  This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
-                                type: array
-                                items:
-                                  type: string
-                                x-kubernetes-list-type: atomic
-                              maxSkew:
-                                description: |-
-                                  MaxSkew describes the degree to which pods may be unevenly distributed.
-                                  When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
-                                  between the number of matching pods in the target topology and the global minimum.
-                                  The global minimum is the minimum number of matching pods in an eligible domain
-                                  or zero if the number of eligible domains is less than MinDomains.
-                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
-                                  labelSelector spread as 2/2/1:
-                                  In this case, the global minimum is 1.
-                                  | zone1 | zone2 | zone3 |
-                                  |  P P  |  P P  |   P   |
-                                  - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
-                                  scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
-                                  violate MaxSkew(1).
-                                  - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
-                                  When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
-                                  to topologies that satisfy it.
-                                  It's a required field. Default value is 1 and 0 is not allowed.
-                                type: integer
-                                format: int32
-                              minDomains:
-                                description: |-
-                                  MinDomains indicates a minimum number of eligible domains.
-                                  When the number of eligible domains with matching topology keys is less than minDomains,
-                                  Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
-                                  And when the number of eligible domains with matching topology keys equals or greater than minDomains,
-                                  this value has no effect on scheduling.
-                                  As a result, when the number of eligible domains is less than minDomains,
-                                  scheduler won't schedule more than maxSkew Pods to those domains.
-                                  If value is nil, the constraint behaves as if MinDomains is equal to 1.
-                                  Valid values are integers greater than 0.
-                                  When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
-
-                                  For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
-                                  labelSelector spread as 2/2/2:
-                                  | zone1 | zone2 | zone3 |
-                                  |  P P  |  P P  |  P P  |
-                                  The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
-                                  In this situation, new pod with the same labelSelector cannot be scheduled,
-                                  because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
-                                  it will violate MaxSkew.
-                                type: integer
-                                format: int32
-                              nodeAffinityPolicy:
-                                description: |-
-                                  NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
-                                  when calculating pod topology spread skew. Options are:
-                                  - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
-                                  - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
-
-                                  If this value is nil, the behavior is equivalent to the Honor policy.
-                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
-                                type: string
-                              nodeTaintsPolicy:
-                                description: |-
-                                  NodeTaintsPolicy indicates how we will treat node taints when calculating
-                                  pod topology spread skew. Options are:
-                                  - Honor: nodes without taints, along with tainted nodes for which the incoming pod
-                                  has a toleration, are included.
-                                  - Ignore: node taints are ignored. All nodes are included.
-
-                                  If this value is nil, the behavior is equivalent to the Ignore policy.
-                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
-                                type: string
-                              topologyKey:
-                                description: |-
-                                  TopologyKey is the key of node labels. Nodes that have a label with this key
-                                  and identical values are considered to be in the same topology.
-                                  We consider each <key, value> as a "bucket", and try to put balanced number
-                                  of pods into each bucket.
-                                  We define a domain as a particular instance of a topology.
-                                  Also, we define an eligible domain as a domain whose nodes meet the requirements of
-                                  nodeAffinityPolicy and nodeTaintsPolicy.
-                                  e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
-                                  And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
-                                  It's a required field.
-                                type: string
-                              whenUnsatisfiable:
-                                description: |-
-                                  WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
-                                  the spread constraint.
-                                  - DoNotSchedule (default) tells the scheduler not to schedule it.
-                                  - ScheduleAnyway tells the scheduler to schedule the pod in any location,
-                                    but giving higher precedence to topologies that would help reduce the
-                                    skew.
-                                  A constraint is considered "Unsatisfiable" for an incoming pod
-                                  if and only if every possible node assignment for that pod would violate
-                                  "MaxSkew" on some topology.
-                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
-                                  labelSelector spread as 3/1/1:
-                                  | zone1 | zone2 | zone3 |
-                                  | P P P |   P   |   P   |
-                                  If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
-                                  to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
-                                  MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
-                                  won't make it *more* imbalanced.
-                                  It's a required field.
-                                type: string
                 tailscale:
                   description: |-
                     TailscaleConfig contains options to configure the tailscale-specific
@@ -2084,7 +1908,7 @@ spec:
                         routes advertized by other nodes on the tailnet, such as subnet
                         routes.
                         This is equivalent of passing --accept-routes flag to a tailscale Linux client.
-                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
+                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
                         Defaults to false.
                       type: boolean
             status:

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -1,187 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
-  name: proxygroups.tailscale.com
-spec:
-  group: tailscale.com
-  names:
-    kind: ProxyGroup
-    listKind: ProxyGroupList
-    plural: proxygroups
-    shortNames:
-      - pg
-    singular: proxygroup
-  scope: Cluster
-  versions:
-    - additionalPrinterColumns:
-        - description: Status of the deployed ProxyGroup resources.
-          jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
-          name: Status
-          type: string
-      name: v1alpha1
-      schema:
-        openAPIV3Schema:
-          type: object
-          required:
-            - spec
-          properties:
-            apiVersion:
-              description: |-
-                APIVersion defines the versioned schema of this representation of an object.
-                Servers should convert recognized schemas to the latest internal value, and
-                may reject unrecognized values.
-                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
-              type: string
-            kind:
-              description: |-
-                Kind is a string value representing the REST resource this object represents.
-                Servers may infer this from the endpoint the client submits requests to.
-                Cannot be updated.
-                In CamelCase.
-                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
-              type: string
-            metadata:
-              type: object
-            spec:
-              description: Spec describes the desired ProxyGroup instances.
-              type: object
-              required:
-                - type
-              properties:
-                hostnamePrefix:
-                  description: |-
-                    HostnamePrefix is the hostname prefix to use for tailnet devices created
-                    by the ProxyGroup. Each device will have the integer number from its
-                    StatefulSet pod appended to this prefix to form the full hostname.
-                    HostnamePrefix can contain lower case letters, numbers and dashes, it
-                    must not start with a dash and must be between 1 and 62 characters long.
-                  type: string
-                  pattern: ^[a-z0-9][a-z0-9-]{0,61}$
-                proxyClass:
-                  description: |-
-                    ProxyClass is the name of the ProxyClass custom resource that contains
-                    configuration options that should be applied to the resources created
-                    for this ProxyGroup. If unset, and there is no default ProxyClass
-                    configured, the operator will create resources with the default
-                    configuration.
-                  type: string
-                replicas:
-                  description: |-
-                    Replicas specifies how many replicas to create the StatefulSet with.
-                    Defaults to 2.
-                  type: integer
-                  format: int32
-                tags:
-                  description: |-
-                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
-                    If you specify custom tags here, make sure you also make the operator
-                    an owner of these tags.
-                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
-                    Tags cannot be changed once a ProxyGroup device has been created.
-                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
-                  type: array
-                  items:
-                    type: string
-                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
-                type:
-                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                  type: string
-                  enum:
-                    - egress
-            status:
-              description: |-
-                ProxyGroupStatus describes the status of the ProxyGroup resources. This is
-                set and managed by the Tailscale operator.
-              type: object
-              properties:
-                conditions:
-                  description: |-
-                    List of status conditions to indicate the status of the ProxyGroup
-                    resources. Known condition types are `ProxyGroupReady`.
-                  type: array
-                  items:
-                    description: Condition contains details for one aspect of the current state of this API Resource.
-                    type: object
-                    required:
-                      - lastTransitionTime
-                      - message
-                      - reason
-                      - status
-                      - type
-                    properties:
-                      lastTransitionTime:
-                        description: |-
-                          lastTransitionTime is the last time the condition transitioned from one status to another.
-                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
-                        type: string
-                        format: date-time
-                      message:
-                        description: |-
-                          message is a human readable message indicating details about the transition.
-                          This may be an empty string.
-                        type: string
-                        maxLength: 32768
-                      observedGeneration:
-                        description: |-
-                          observedGeneration represents the .metadata.generation that the condition was set based upon.
-                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
-                          with respect to the current state of the instance.
-                        type: integer
-                        format: int64
-                        minimum: 0
-                      reason:
-                        description: |-
-                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
-                          Producers of specific condition types may define expected values and meanings for this field,
-                          and whether the values are considered a guaranteed API.
-                          The value should be a CamelCase string.
-                          This field may not be empty.
-                        type: string
-                        maxLength: 1024
-                        minLength: 1
-                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
-                      status:
-                        description: status of the condition, one of True, False, Unknown.
-                        type: string
-                        enum:
-                          - "True"
-                          - "False"
-                          - Unknown
-                      type:
-                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
-                        type: string
-                        maxLength: 316
-                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                  x-kubernetes-list-map-keys:
-                    - type
-                  x-kubernetes-list-type: map
-                devices:
-                  description: List of tailnet devices associated with the ProxyGroup StatefulSet.
-                  type: array
-                  items:
-                    type: object
-                    required:
-                      - hostname
-                    properties:
-                      hostname:
-                        description: |-
-                          Hostname is the fully qualified domain name of the device.
-                          If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
-                          node.
-                        type: string
-                      tailnetIPs:
-                        description: |-
-                          TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
-                          assigned to the device.
-                        type: array
-                        items:
-                          type: string
-                  x-kubernetes-list-map-keys:
-                    - hostname
-                  x-kubernetes-list-type: map
-      served: true
-      storage: true
-      subresources:
-        status: {}

cmd/k8s-operator/deploy/examples/proxygroup.yaml

@@ -1,7 +0,0 @@
-apiVersion: tailscale.com/v1alpha1
-kind: ProxyGroup
-metadata:
-  name: egress-proxies
-spec:
-  type: egress
-  replicas: 3

cmd/k8s-operator/deploy/examples/recorder.yaml

@@ -1,6 +0,0 @@
-apiVersion: tailscale.com/v1alpha1
-kind: Recorder
-metadata:
-  name: recorder
-spec:
-  enableUI: true

cmd/k8s-operator/dnsrecords.go

@@ -24,7 +24,6 @@ import (
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/util/mak"
-	"tailscale.com/util/set"
 )
 
 const (
@@ -168,49 +167,36 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 		}
 	}
 
-	// Get the Pod IP addresses for the proxy from the EndpointSlices for
-	// the headless Service. The Service can have multiple EndpointSlices
-	// associated with it, for example in dual-stack clusters.
+	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
+	// headless Service.
 	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	var eps = new(discoveryv1.EndpointSliceList)
-	if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil {
-		return fmt.Errorf("error listing EndpointSlices for the proxy's headless Service: %w", err)
+	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
+	if err != nil {
+		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
 	}
-	if len(eps.Items) == 0 {
+	if eps == nil {
 		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
 		return nil
 	}
-	// Each EndpointSlice for a Service can have a list of endpoints that each
+	// An EndpointSlice for a Service can have a list of endpoints that each
 	// can have multiple addresses - these are the IP addresses of any Pods
 	// selected by that Service. Pick all the IPv4 addresses.
-	// It is also possible that multiple EndpointSlices have overlapping addresses.
-	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints
-	ips := make(set.Set[string], 0)
-	for _, slice := range eps.Items {
-		if slice.AddressType != discoveryv1.AddressTypeIPv4 {
-			logger.Infof("EndpointSlice is for AddressType %s, currently only IPv4 address type is supported", slice.AddressType)
-			continue
-		}
-		for _, ep := range slice.Endpoints {
-			if !epIsReady(&ep) {
-				logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String())
-				continue
-			}
-			for _, ip := range ep.Addresses {
-				if !net.IsIPv4String(ip) {
-					logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
-				} else {
-					ips.Add(ip)
-				}
+	ips := make([]string, 0)
+	for _, ep := range eps.Endpoints {
+		for _, ip := range ep.Addresses {
+			if !net.IsIPv4String(ip) {
+				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+			} else {
+				ips = append(ips, ip)
 			}
 		}
 	}
-	if ips.Len() == 0 {
+	if len(ips) == 0 {
 		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
 		return nil
 	}
 	updateFunc := func(rec *operatorutils.Records) {
-		mak.Set(&rec.IP4, fqdn, ips.Slice())
+		mak.Set(&rec.IP4, fqdn, ips)
 	}
 	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
 		return fmt.Errorf("error updating DNS records: %w", err)
@@ -218,17 +204,6 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS
 	return nil
 }
 
-// epIsReady reports whether the endpoint is currently in a state to receive new
-// traffic. As per kube docs, only explicitly set 'false' for 'Ready' or
-// 'Serving' conditions or explicitly set 'true' for 'Terminating' condition
-// means that the Endpoint is NOT ready.
-// https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/apis/discovery/types.go#L109-L131
-func epIsReady(ep *discoveryv1.Endpoint) bool {
-	return (ep.Conditions.Ready == nil || *ep.Conditions.Ready) &&
-		(ep.Conditions.Serving == nil || *ep.Conditions.Serving) &&
-		(ep.Conditions.Terminating == nil || !*ep.Conditions.Terminating)
-}
-
 // maybeCleanup ensures that the DNS record for the proxy has been removed from
 // dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
 // has been removed from the Service. If the record is not found in the

cmd/k8s-operator/dnsrecords_test.go

@@ -8,7 +8,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -88,16 +87,13 @@ func TestDNSRecordsReconciler(t *testing.T) {
 		},
 	}
 	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
-	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4)
-	epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6)
-
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
 	mustCreate(t, fc, egressSvcFQDN)
 	mustCreate(t, fc, headlessForEgressSvcFQDN)
 	mustCreate(t, fc, ep)
-	mustCreate(t, fc, epv6)
 	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
 	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
-	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} // IPv6 endpoint is currently ignored
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
@@ -110,7 +106,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectHostsRecords(t, fc, wantHosts)
 
 	// 3. DNS record is updated if the IP address of the proxy Pod changes.
-	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4)
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
 	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
 		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
 	})
@@ -120,7 +116,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
 
 	// 4. DNS record is created for an ingress proxy configured via Ingress
 	headlessForIngress := headlessSvcForParent(ing, "ingress")
-	ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4)
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
 	mustCreate(t, fc, headlessForIngress)
 	mustCreate(t, fc, ep)
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
@@ -144,17 +140,6 @@ func TestDNSRecordsReconciler(t *testing.T) {
 	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
 	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
 	expectHostsRecords(t, fc, wantHosts)
-
-	// 7. A not-ready Endpoint is removed from DNS config.
-	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
-		ep.Endpoints[0].Conditions.Ready = ptr.To(false)
-		ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{
-			Addresses: []string{"1.2.3.4"},
-		})
-	})
-	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
-	wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"}
-	expectHostsRecords(t, fc, wantHosts)
 }
 
 func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
@@ -177,21 +162,15 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 	}
 }
 
-func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.AddressType) *discoveryv1.EndpointSlice {
+func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
 	return &discoveryv1.EndpointSlice{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%s", svc.Name, string(fam)),
+			Name:      svc.Name,
 			Namespace: svc.Namespace,
 			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
 		},
-		AddressType: fam,
 		Endpoints: []discoveryv1.Endpoint{{
 			Addresses: []string{ip},
-			Conditions: discoveryv1.EndpointConditions{
-				Ready:       ptr.To(true),
-				Serving:     ptr.To(true),
-				Terminating: ptr.To(false),
-			},
 		}},
 	}
 }

cmd/k8s-operator/egress-eps.go

@@ -1,213 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/netip"
-	"reflect"
-	"strings"
-
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/types/ptr"
-)
-
-// egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies.
-type egressEpsReconciler struct {
-	client.Client
-	logger      *zap.SugaredLogger
-	tsNamespace string
-}
-
-// Reconcile reconciles an EndpointSlice for a tailnet service. It updates the EndpointSlice with the endpoints of
-// those ProxyGroup Pods that are ready to route traffic to the tailnet service.
-// It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired
-// configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready.
-func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
-	l := er.logger.With("Service", req.NamespacedName)
-	l.Debugf("starting reconcile")
-	defer l.Debugf("reconcile finished")
-
-	eps := new(discoveryv1.EndpointSlice)
-	err = er.Get(ctx, req.NamespacedName, eps)
-	if apierrors.IsNotFound(err) {
-		l.Debugf("EndpointSlice not found")
-		return reconcile.Result{}, nil
-	}
-	if err != nil {
-		return reconcile.Result{}, fmt.Errorf("failed to get EndpointSlice: %w", err)
-	}
-	if !eps.DeletionTimestamp.IsZero() {
-		l.Debugf("EnpointSlice is being deleted")
-		return res, nil
-	}
-
-	// Get the user-created ExternalName Service and use its status conditions to determine whether cluster
-	// resources are set up for this tailnet service.
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      eps.Labels[LabelParentName],
-			Namespace: eps.Labels[LabelParentNamespace],
-		},
-	}
-	err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
-	if apierrors.IsNotFound(err) {
-		l.Infof("ExternalName Service %s/%s not found, perhaps it was deleted", svc.Namespace, svc.Name)
-		return res, nil
-	}
-	if err != nil {
-		return res, fmt.Errorf("error retrieving ExternalName Service: %w", err)
-	}
-	if !tsoperator.EgressServiceIsValidAndConfigured(svc) {
-		l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name)
-		return res, nil
-	}
-
-	// TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is
-	// wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to
-	// determine if a reconcile is needed.
-
-	oldEps := eps.DeepCopy()
-	proxyGroupName := eps.Labels[labelProxyGroup]
-	tailnetSvc := tailnetSvcName(svc)
-	l = l.With("tailnet-service-name", tailnetSvc)
-
-	// Retrieve the desired tailnet service configuration from the ConfigMap.
-	_, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace)
-	if err != nil {
-		return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err)
-	}
-	cfg, ok := (*cfgs)[tailnetSvc]
-	if !ok {
-		l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc)
-		return res, nil
-	}
-
-	// Check which Pods in ProxyGroup are ready to route traffic to this
-	// egress service.
-	podList := &corev1.PodList{}
-	if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil {
-		return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
-	}
-	newEndpoints := make([]discoveryv1.Endpoint, 0)
-	for _, pod := range podList.Items {
-		ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, l)
-		if err != nil {
-			return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err)
-		}
-		if !ready {
-			continue // maybe next time
-		}
-		podIP, err := podIPv4(&pod) // we currently only support IPv4
-		if err != nil {
-			return res, fmt.Errorf("error determining IPv4 address for Pod: %w", err)
-		}
-		newEndpoints = append(newEndpoints, discoveryv1.Endpoint{
-			Hostname:  (*string)(&pod.UID),
-			Addresses: []string{podIP},
-			Conditions: discoveryv1.EndpointConditions{
-				Ready:       ptr.To(true),
-				Serving:     ptr.To(true),
-				Terminating: ptr.To(false),
-			},
-		})
-	}
-	// Note that Endpoints are being overwritten with the currently valid endpoints so we don't need to explicitly
-	// run a cleanup for deleted Pods etc.
-	eps.Endpoints = newEndpoints
-	if !reflect.DeepEqual(eps, oldEps) {
-		l.Infof("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods")
-		if err := er.Update(ctx, eps); err != nil {
-			return res, fmt.Errorf("error updating EndpointSlice: %w", err)
-		}
-	}
-	return res, nil
-}
-
-func podIPv4(pod *corev1.Pod) (string, error) {
-	for _, ip := range pod.Status.PodIPs {
-		parsed, err := netip.ParseAddr(ip.IP)
-		if err != nil {
-			return "", fmt.Errorf("error parsing IP address %s: %w", ip, err)
-		}
-		if parsed.Is4() {
-			return parsed.String(), nil
-		}
-	}
-	return "", nil
-}
-
-// podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to
-// route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service
-// status written there to the desired service configuration.
-func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, l *zap.SugaredLogger) (bool, error) {
-	l = l.With("proxy_pod", pod.Name)
-	l.Debugf("checking whether proxy is ready to route to egress service")
-	if !pod.DeletionTimestamp.IsZero() {
-		l.Debugf("proxy Pod is being deleted, ignore")
-		return false, nil
-	}
-	podIP, err := podIPv4(&pod)
-	if err != nil {
-		return false, fmt.Errorf("error determining Pod IP address: %v", err)
-	}
-	if podIP == "" {
-		l.Infof("[unexpected] Pod does not have an IPv4 address, and IPv6 is not currently supported")
-		return false, nil
-	}
-	stateS := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      pod.Name,
-			Namespace: pod.Namespace,
-		},
-	}
-	err = er.Get(ctx, client.ObjectKeyFromObject(stateS), stateS)
-	if apierrors.IsNotFound(err) {
-		l.Debugf("proxy does not have a state Secret, waiting...")
-		return false, nil
-	}
-	if err != nil {
-		return false, fmt.Errorf("error getting state Secret: %w", err)
-	}
-	svcStatusBS := stateS.Data[egressservices.KeyEgressServices]
-	if len(svcStatusBS) == 0 {
-		l.Debugf("proxy's state Secret does not contain egress services status, waiting...")
-		return false, nil
-	}
-	svcStatus := &egressservices.Status{}
-	if err := json.Unmarshal(svcStatusBS, svcStatus); err != nil {
-		return false, fmt.Errorf("error unmarshalling egress service status: %w", err)
-	}
-	if !strings.EqualFold(podIP, svcStatus.PodIPv4) {
-		l.Infof("proxy's egress service status is for Pod IP %s, current proxy's Pod IP %s, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP)
-		return false, nil
-	}
-	st, ok := (*svcStatus).Services[tailnetSvcName]
-	if !ok {
-		l.Infof("proxy's state Secret does not have egress service status, waiting...")
-		return false, nil
-	}
-	if !reflect.DeepEqual(cfg.TailnetTarget, st.TailnetTarget) {
-		l.Infof("proxy has configured egress service for tailnet target %v, current target is %v, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget)
-		return false, nil
-	}
-	if !reflect.DeepEqual(cfg.Ports, st.Ports) {
-		l.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports)
-		return false, nil
-	}
-	l.Debugf("proxy is ready to route traffic to egress service")
-	return true, nil
-}

cmd/k8s-operator/egress-eps_test.go

@@ -1,211 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"math/rand/v2"
-	"testing"
-
-	"github.com/AlekSi/pointer"
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/tstest"
-	"tailscale.com/util/mak"
-)
-
-func TestTailscaleEgressEndpointSlices(t *testing.T) {
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetFQDN: "foo.bar.ts.net",
-				AnnotationProxyGroup:        "foo",
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ExternalName: "placeholder",
-			Type:         corev1.ServiceTypeExternalName,
-			Selector:     nil,
-			Ports: []corev1.ServicePort{
-				{
-					Name:     "http",
-					Protocol: "TCP",
-					Port:     80,
-				},
-			},
-		},
-		Status: corev1.ServiceStatus{
-			Conditions: []metav1.Condition{
-				condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, "", "", clock),
-				condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "", "", clock),
-			},
-		},
-	}
-	port := randomPort()
-	cm := configMapForSvc(t, svc, port)
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(svc, cm).
-		WithStatusSubresource(svc).
-		Build()
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	er := &egressEpsReconciler{
-		Client:      fc,
-		logger:      zl.Sugar(),
-		tsNamespace: "operator-ns",
-	}
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "foo",
-			Namespace: "operator-ns",
-			Labels: map[string]string{
-				LabelParentName:      "test",
-				LabelParentNamespace: "default",
-				labelSvcType:         typeEgress,
-				labelProxyGroup:      "foo"},
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-	}
-	mustCreate(t, fc, eps)
-
-	t.Run("no_proxy_group_resources", func(t *testing.T) {
-		expectReconciled(t, er, "operator-ns", "foo") // should not error
-	})
-
-	t.Run("no_pods_ready_to_route_traffic", func(t *testing.T) {
-		pod, stateS := podAndSecretForProxyGroup("foo")
-		mustCreate(t, fc, pod)
-		mustCreate(t, fc, stateS)
-		expectReconciled(t, er, "operator-ns", "foo") // should not error
-	})
-
-	t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) {
-		pod, stateS := podAndSecretForProxyGroup("foo")
-		stBs := serviceStatusForPodIP(t, svc, pod.Status.PodIPs[0].IP, port)
-		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
-			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
-		})
-		expectReconciled(t, er, "operator-ns", "foo")
-		eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
-			Addresses: []string{"10.0.0.1"},
-			Hostname:  pointer.To("foo"),
-			Conditions: discoveryv1.EndpointConditions{
-				Serving:     pointer.ToBool(true),
-				Ready:       pointer.ToBool(true),
-				Terminating: pointer.ToBool(false),
-			},
-		})
-		expectEqual(t, fc, eps, nil)
-	})
-	t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
-		_, stateS := podAndSecretForProxyGroup("foo")           // replica Pod has IP 10.0.0.1
-		stBs := serviceStatusForPodIP(t, svc, "10.0.0.2", port) // status is for a Pod with IP 10.0.0.2
-		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
-			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
-		})
-		expectReconciled(t, er, "operator-ns", "foo")
-		eps.Endpoints = []discoveryv1.Endpoint{}
-		expectEqual(t, fc, eps, nil)
-	})
-}
-
-func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.ConfigMap {
-	t.Helper()
-	ports := make(map[egressservices.PortMap]struct{})
-	for _, port := range svc.Spec.Ports {
-		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
-	}
-	cfg := egressservices.Config{
-		Ports: ports,
-	}
-	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
-		cfg.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
-	}
-	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
-		cfg.TailnetTarget = egressservices.TailnetTarget{IP: ip}
-	}
-	name := tailnetSvcName(svc)
-	cfgs := egressservices.Configs{name: cfg}
-	bs, err := json.Marshal(&cfgs)
-	if err != nil {
-		t.Fatalf("error marshalling config: %v", err)
-	}
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      pgEgressCMName(svc.Annotations[AnnotationProxyGroup]),
-			Namespace: "operator-ns",
-		},
-		BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs},
-	}
-	return cm
-}
-
-func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint16) []byte {
-	t.Helper()
-	ports := make(map[egressservices.PortMap]struct{})
-	for _, port := range svc.Spec.Ports {
-		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
-	}
-	svcSt := egressservices.ServiceStatus{Ports: ports}
-	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
-		svcSt.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
-	}
-	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
-		svcSt.TailnetTarget = egressservices.TailnetTarget{IP: ip}
-	}
-	svcName := tailnetSvcName(svc)
-	st := egressservices.Status{
-		PodIPv4:  ip,
-		Services: map[string]*egressservices.ServiceStatus{svcName: &svcSt},
-	}
-	bs, err := json.Marshal(st)
-	if err != nil {
-		t.Fatalf("error marshalling service status: %v", err)
-	}
-	return bs
-}
-
-func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
-	p := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-0", pg),
-			Namespace: "operator-ns",
-			Labels:    pgLabels(pg, nil),
-			UID:       "foo",
-		},
-		Status: corev1.PodStatus{
-			PodIPs: []corev1.PodIP{
-				{IP: "10.0.0.1"},
-			},
-		},
-	}
-	s := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-0", pg),
-			Namespace: "operator-ns",
-			Labels:    pgSecretLabels(pg, "state"),
-		},
-	}
-	return p, s
-}
-
-func randomPort() uint16 {
-	return uint16(rand.Int32N(1000) + 1000)
-}

cmd/k8s-operator/egress-services-readiness.go

@@ -1,179 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-
-	"go.uber.org/zap"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/tstime"
-)
-
-const (
-	reasonReadinessCheckFailed     = "ReadinessCheckFailed"
-	reasonClusterResourcesNotReady = "ClusterResourcesNotReady"
-	reasonNoProxies                = "NoProxiesConfigured"
-	reasonNotReady                 = "NotReadyToRouteTraffic"
-	reasonReady                    = "ReadyToRouteTraffic"
-	reasonPartiallyReady           = "PartiallyReadyToRouteTraffic"
-	msgReadyToRouteTemplate        = "%d out of %d replicas are ready to route traffic"
-)
-
-type egressSvcsReadinessReconciler struct {
-	client.Client
-	logger      *zap.SugaredLogger
-	clock       tstime.Clock
-	tsNamespace string
-}
-
-// Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the
-// EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to
-// route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress
-// service to determine how many replicas are currently able to route traffic.
-func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
-	l := esrr.logger.With("Service", req.NamespacedName)
-	defer l.Info("reconcile finished")
-
-	svc := new(corev1.Service)
-	if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
-		l.Info("Service not found")
-		return res, nil
-	} else if err != nil {
-		return res, fmt.Errorf("failed to get Service: %w", err)
-	}
-	var (
-		reason, msg string
-		st          metav1.ConditionStatus = metav1.ConditionUnknown
-	)
-	oldStatus := svc.Status.DeepCopy()
-	defer func() {
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
-		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
-			err = errors.Join(err, esrr.Status().Update(ctx, svc))
-		}
-	}()
-
-	crl := egressSvcChildResourceLabels(svc)
-	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl)
-	if err != nil {
-		err = fmt.Errorf("error getting EndpointSlice: %w", err)
-		reason = reasonReadinessCheckFailed
-		msg = err.Error()
-		return res, err
-	}
-	if eps == nil {
-		l.Infof("EndpointSlice for Service does not yet exist, waiting...")
-		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: svc.Annotations[AnnotationProxyGroup],
-		},
-	}
-	err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg)
-	if apierrors.IsNotFound(err) {
-		l.Infof("ProxyGroup for Service does not exist, waiting...")
-		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	if err != nil {
-		err = fmt.Errorf("error retrieving ProxyGroup: %w", err)
-		reason = reasonReadinessCheckFailed
-		msg = err.Error()
-		return res, err
-	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup for Service is not ready, waiting...")
-		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-
-	replicas := pgReplicas(pg)
-	if replicas == 0 {
-		l.Infof("ProxyGroup replicas set to 0")
-		reason, msg = reasonNoProxies, reasonNoProxies
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	podLabels := pgLabels(pg.Name, nil)
-	var readyReplicas int32
-	for i := range replicas {
-		podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
-		pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
-		if err != nil {
-			err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err)
-			reason = reasonReadinessCheckFailed
-			msg = err.Error()
-			return res, err
-		}
-		if pod == nil {
-			l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
-			reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-			return res, nil
-		}
-		l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
-		ready := false
-		for _, ep := range eps.Endpoints {
-			l.Infof("looking at endpoint with addresses %v", ep.Addresses)
-			if endpointReadyForPod(&ep, pod, l) {
-				l.Infof("endpoint is ready for Pod")
-				ready = true
-				break
-			}
-		}
-		if ready {
-			readyReplicas++
-		}
-	}
-	msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
-	if readyReplicas == 0 {
-		reason = reasonNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	st = metav1.ConditionTrue
-	if readyReplicas < replicas {
-		reason = reasonPartiallyReady
-	} else {
-		reason = reasonReady
-	}
-	return res, nil
-}
-
-// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic.
-// Endpoint must not be nil.
-func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
-	podIP, err := podIPv4(pod)
-	if err != nil {
-		l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
-		return false
-	}
-	// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.
-	if len(ep.Addresses) != 1 {
-		return false
-	}
-	return strings.EqualFold(ep.Addresses[0], podIP) &&
-		*ep.Conditions.Ready &&
-		*ep.Conditions.Serving &&
-		!*ep.Conditions.Terminating
-}

cmd/k8s-operator/egress-services-readiness_test.go

@@ -1,169 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/AlekSi/pointer"
-	"go.uber.org/zap"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/tstest"
-	"tailscale.com/tstime"
-)
-
-func TestEgressServiceReadiness(t *testing.T) {
-	// We need to pass a ProxyGroup object to WithStatusSubresource because of some quirks in how the fake client
-	// works. Without this code further down would not be able to update ProxyGroup status.
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithStatusSubresource(&tsapi.ProxyGroup{}).
-		Build()
-	zl, _ := zap.NewDevelopment()
-	cl := tstest.NewClock(tstest.ClockOpts{})
-	rec := &egressSvcsReadinessReconciler{
-		tsNamespace: "operator-ns",
-		Client:      fc,
-		logger:      zl.Sugar(),
-		clock:       cl,
-	}
-	tailnetFQDN := "my-app.tailnetxyz.ts.net"
-	egressSvc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "my-app",
-			Namespace: "dev",
-			Annotations: map[string]string{
-				AnnotationProxyGroup:        "dev",
-				AnnotationTailnetTargetFQDN: tailnetFQDN,
-			},
-		},
-	}
-	fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}}
-	l := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "my-app",
-			Namespace: "operator-ns",
-			Labels:    l,
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-	}
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "dev",
-		},
-	}
-	mustCreate(t, fc, egressSvc)
-	setClusterNotReady(egressSvc, cl, zl.Sugar())
-	t.Run("endpointslice_does_not_exist", func(t *testing.T) {
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // not ready
-	})
-	t.Run("proxy_group_does_not_exist", func(t *testing.T) {
-		mustCreate(t, fc, eps)
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // still not ready
-	})
-	t.Run("proxy_group_not_ready", func(t *testing.T) {
-		mustCreate(t, fc, pg)
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // still not ready
-	})
-	t.Run("no_ready_replicas", func(t *testing.T) {
-		setPGReady(pg, cl, zl.Sugar())
-		mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
-			p.Status = pg.Status
-		})
-		expectEqual(t, fc, pg, nil)
-		for i := range pgReplicas(pg) {
-			p := pod(pg, i)
-			mustCreate(t, fc, p)
-			mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
-				existing.Status.PodIPs = p.Status.PodIPs
-			})
-		}
-		expectReconciled(t, rec, "dev", "my-app")
-		setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
-		expectEqual(t, fc, egressSvc, nil) // still not ready
-	})
-	t.Run("one_ready_replica", func(t *testing.T) {
-		setEndpointForReplica(pg, 0, eps)
-		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
-			e.Endpoints = eps.Endpoints
-		})
-		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // partially ready
-	})
-	t.Run("all_replicas_ready", func(t *testing.T) {
-		for i := range pgReplicas(pg) {
-			setEndpointForReplica(pg, i, eps)
-		}
-		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
-			e.Endpoints = eps.Endpoints
-		})
-		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // ready
-	})
-}
-
-func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger) {
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, l)
-}
-
-func setNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas int32) {
-	msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas)
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, l)
-}
-
-func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas, readyReplicas int32) {
-	reason := reasonPartiallyReady
-	if readyReplicas == replicas {
-		reason = reasonReady
-	}
-	msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, l)
-}
-
-func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) {
-	tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l)
-}
-
-func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {
-	p := pod(pg, ordinal)
-	eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
-		Addresses: []string{p.Status.PodIPs[0].IP},
-		Conditions: discoveryv1.EndpointConditions{
-			Ready:       pointer.ToBool(true),
-			Serving:     pointer.ToBool(true),
-			Terminating: pointer.ToBool(false),
-		},
-	})
-}
-
-func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
-	l := pgLabels(pg.Name, nil)
-	l[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
-	ip := fmt.Sprintf("10.0.0.%d", ordinal)
-	return &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%d", pg.Name, ordinal),
-			Namespace: "operator-ns",
-			Labels:    l,
-		},
-		Status: corev1.PodStatus{
-			PodIPs: []corev1.PodIP{{IP: ip}},
-		},
-	}
-}

cmd/k8s-operator/egress-services.go

@@ -1,716 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math/rand/v2"
-	"reflect"
-	"slices"
-	"strings"
-	"sync"
-
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apiserver/pkg/storage/names"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/kube/kubetypes"
-	"tailscale.com/tstime"
-	"tailscale.com/util/clientmetric"
-	"tailscale.com/util/mak"
-	"tailscale.com/util/set"
-)
-
-const (
-	reasonEgressSvcInvalid        = "EgressSvcInvalid"
-	reasonEgressSvcValid          = "EgressSvcValid"
-	reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
-	reasonProxyGroupNotReady      = "ProxyGroupNotReady"
-
-	labelProxyGroup = "tailscale.com/proxy-group"
-
-	labelSvcType = "tailscale.com/svc-type" // ingress or egress
-	typeEgress   = "egress"
-	// maxPorts is the maximum number of ports that can be exposed on a
-	// container. In practice this will be ports in range [3000 - 4000). The
-	// high range should make it easier to distinguish container ports from
-	// the tailnet target ports for debugging purposes (i.e when reading
-	// netfilter rules). The limit of 10000 is somewhat arbitrary, the
-	// assumption is that this would not be hit in practice.
-	maxPorts = 10000
-
-	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
-)
-
-var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
-
-// egressSvcsReconciler reconciles user created ExternalName Services that specify a tailnet
-// endpoint that should be exposed to cluster workloads and an egress ProxyGroup
-// on whose proxies it should be exposed.
-type egressSvcsReconciler struct {
-	client.Client
-	logger      *zap.SugaredLogger
-	recorder    record.EventRecorder
-	clock       tstime.Clock
-	tsNamespace string
-
-	mu   sync.Mutex           // protects following
-	svcs set.Slice[types.UID] // UIDs of all currently managed egress Services for ProxyGroup
-}
-
-// Reconcile reconciles an ExternalName Service that specifies a tailnet target and a ProxyGroup on whose proxies should
-// forward cluster traffic to the target.
-// For an ExternalName Service the reconciler:
-//
-// - for each port N defined on the ExternalName Service, allocates a port X in range [3000- 4000), unique for the
-// ProxyGroup proxies. Proxies will forward cluster traffic received on port N to port M on the tailnet target
-//
-// - creates a ClusterIP Service in the operator's namespace with portmappings for all M->N port pairs. This will allow
-// cluster workloads to send traffic on the user-defined tailnet target port and get it transparently mapped to the
-// randomly selected port on proxy Pods.
-//
-// - creates an EndpointSlice in the operator's namespace with kubernetes.io/service-name label pointing to the
-// ClusterIP Service. The endpoints will get dynamically updates to proxy Pod IPs as the Pods become ready to route
-// traffic to the tailnet target. kubernetes.io/service-name label ensures that kube-proxy sets up routing rules to
-// forward cluster traffic received on ClusterIP Service's IP address to the endpoints (Pod IPs).
-//
-// - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the
-// portmappings.
-func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
-	l := esr.logger.With("Service", req.NamespacedName)
-	defer l.Info("reconcile finished")
-
-	svc := new(corev1.Service)
-	if err = esr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
-		l.Info("Service not found")
-		return res, nil
-	} else if err != nil {
-		return res, fmt.Errorf("failed to get Service: %w", err)
-	}
-
-	// Name of the 'egress service', meaning the tailnet target.
-	tailnetSvc := tailnetSvcName(svc)
-	l = l.With("tailnet-service", tailnetSvc)
-
-	// Note that resources for egress Services are only cleaned up when the
-	// Service is actually deleted (and not if, for example, user decides to
-	// remove the Tailscale annotation from it). This should be fine- we
-	// assume that the egress ExternalName Services are always created for
-	// Tailscale operator specifically.
-	if !svc.DeletionTimestamp.IsZero() {
-		l.Info("Service is being deleted, ensuring resource cleanup")
-		return res, esr.maybeCleanup(ctx, svc, l)
-	}
-
-	oldStatus := svc.Status.DeepCopy()
-	defer func() {
-		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
-			err = errors.Join(err, esr.Status().Update(ctx, svc))
-		}
-	}()
-
-	// Validate the user-created ExternalName Service and the associated ProxyGroup.
-	if ok, err := esr.validateClusterResources(ctx, svc, l); err != nil {
-		return res, fmt.Errorf("error validating cluster resources: %w", err)
-	} else if !ok {
-		return res, nil
-	}
-
-	if !slices.Contains(svc.Finalizers, FinalizerName) {
-		l.Infof("configuring tailnet service") // logged exactly once
-		svc.Finalizers = append(svc.Finalizers, FinalizerName)
-		if err := esr.Update(ctx, svc); err != nil {
-			err := fmt.Errorf("failed to add finalizer: %w", err)
-			r := svcConfiguredReason(svc, false, l)
-			tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
-			return res, err
-		}
-		esr.mu.Lock()
-		esr.svcs.Add(svc.UID)
-		gaugeEgressServices.Set(int64(esr.svcs.Len()))
-		esr.mu.Unlock()
-	}
-
-	if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, l); err != nil {
-		err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err)
-		r := svcConfiguredReason(svc, false, l)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
-		return res, err
-	}
-
-	return res, esr.maybeProvision(ctx, svc, l)
-}
-
-func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) {
-	r := svcConfiguredReason(svc, false, l)
-	st := metav1.ConditionFalse
-	defer func() {
-		msg := r
-		if st != metav1.ConditionTrue && err != nil {
-			msg = err.Error()
-		}
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, l)
-	}()
-
-	crl := egressSvcChildResourceLabels(svc)
-	clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl)
-	if err != nil {
-		err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
-		return err
-	}
-	if clusterIPSvc == nil {
-		clusterIPSvc = esr.clusterIPSvcForEgress(crl)
-	}
-	upToDate := svcConfigurationUpToDate(svc, l)
-	provisioned := true
-	if !upToDate {
-		if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, l); err != nil {
-			return err
-		}
-	}
-	if !provisioned {
-		l.Infof("unable to provision cluster resources")
-		return nil
-	}
-
-	// Update ExternalName Service to point at the ClusterIP Service.
-	clusterDomain := retrieveClusterDomain(esr.tsNamespace, l)
-	clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain)
-	if svc.Spec.ExternalName != clusterIPSvcFQDN {
-		l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
-		svc.Spec.ExternalName = clusterIPSvcFQDN
-		if err = esr.Update(ctx, svc); err != nil {
-			err = fmt.Errorf("error updating ExternalName Service: %w", err)
-			return err
-		}
-	}
-	r = svcConfiguredReason(svc, true, l)
-	st = metav1.ConditionTrue
-	return nil
-}
-
-func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, l *zap.SugaredLogger) (*corev1.Service, bool, error) {
-	l.Infof("updating configuration...")
-	usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName)
-	if err != nil {
-		return nil, false, fmt.Errorf("error calculating used ports for ProxyGroup %s: %w", proxyGroupName, err)
-	}
-
-	oldClusterIPSvc := clusterIPSvc.DeepCopy()
-	// loop over ClusterIP Service ports, remove any that are not needed.
-	for i := len(clusterIPSvc.Spec.Ports) - 1; i >= 0; i-- {
-		pm := clusterIPSvc.Spec.Ports[i]
-		found := false
-		for _, wantsPM := range svc.Spec.Ports {
-			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			l.Debugf("portmapping %s:%d -> %s:%d is no longer required, removing", pm.Protocol, pm.TargetPort.IntVal, pm.Protocol, pm.Port)
-			clusterIPSvc.Spec.Ports = slices.Delete(clusterIPSvc.Spec.Ports, i, i+1)
-		}
-	}
-
-	// loop over ExternalName Service ports, for each one not found on
-	// ClusterIP Service produce new target port and add a portmapping to
-	// the ClusterIP Service.
-	for _, wantsPM := range svc.Spec.Ports {
-		found := false
-		for _, gotPM := range clusterIPSvc.Spec.Ports {
-			if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			// Calculate a free port to expose on container and add
-			// a new PortMap to the ClusterIP Service.
-			if usedPorts.Len() == maxPorts {
-				// TODO(irbekrm): refactor to avoid extra reconciles here. Low priority as in practice,
-				// the limit should not be hit.
-				return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts)
-			}
-			p := unusedPort(usedPorts)
-			l.Debugf("mapping tailnet target port %d to container port %d", wantsPM.Port, p)
-			usedPorts.Insert(p)
-			clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
-				Name:       wantsPM.Name,
-				Protocol:   wantsPM.Protocol,
-				Port:       wantsPM.Port,
-				TargetPort: intstr.FromInt32(p),
-			})
-		}
-	}
-	if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
-		if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
-			svc.Labels = clusterIPSvc.Labels
-			svc.Spec = clusterIPSvc.Spec
-		}); err != nil {
-			return nil, false, fmt.Errorf("error ensuring ClusterIP Service: %v", err)
-		}
-	}
-
-	crl := egressSvcEpsLabels(svc, clusterIPSvc)
-	// TODO(irbekrm): support IPv6, but need to investigate how kube proxy
-	// sets up Service -> Pod routing when IPv6 is involved.
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
-			Namespace: esr.tsNamespace,
-			Labels:    crl,
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-		Ports:       epsPortsFromSvc(clusterIPSvc),
-	}
-	if eps, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
-		e.Labels = eps.Labels
-		e.AddressType = eps.AddressType
-		e.Ports = eps.Ports
-		for _, p := range e.Endpoints {
-			p.Conditions.Ready = nil
-		}
-	}); err != nil {
-		return nil, false, fmt.Errorf("error ensuring EndpointSlice: %w", err)
-	}
-
-	cm, cfgs, err := egressSvcsConfigs(ctx, esr.Client, proxyGroupName, esr.tsNamespace)
-	if err != nil {
-		return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err)
-	}
-	if cm == nil {
-		l.Info("ConfigMap not yet created, waiting..")
-		return nil, false, nil
-	}
-	tailnetSvc := tailnetSvcName(svc)
-	gotCfg := (*cfgs)[tailnetSvc]
-	wantsCfg := egressSvcCfg(svc, clusterIPSvc)
-	if !reflect.DeepEqual(gotCfg, wantsCfg) {
-		l.Debugf("updating egress services ConfigMap %s", cm.Name)
-		mak.Set(cfgs, tailnetSvc, wantsCfg)
-		bs, err := json.Marshal(cfgs)
-		if err != nil {
-			return nil, false, fmt.Errorf("error marshalling egress services configs: %w", err)
-		}
-		mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
-		if err := esr.Update(ctx, cm); err != nil {
-			return nil, false, fmt.Errorf("error updating egress services ConfigMap: %w", err)
-		}
-	}
-	l.Infof("egress service configuration has been updated")
-	return clusterIPSvc, true, nil
-}
-
-func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
-	logger.Info("ensuring that resources created for egress service are deleted")
-
-	// Delete egress service config from the ConfigMap mounted by the proxies.
-	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, logger); err != nil {
-		return fmt.Errorf("error deleting egress service config: %w", err)
-	}
-
-	// Delete the ClusterIP Service and EndpointSlice for the egress
-	// service.
-	types := []client.Object{
-		&corev1.Service{},
-		&discoveryv1.EndpointSlice{},
-	}
-	crl := egressSvcChildResourceLabels(svc)
-	for _, typ := range types {
-		if err := esr.DeleteAllOf(ctx, typ, client.InNamespace(esr.tsNamespace), client.MatchingLabels(crl)); err != nil {
-			return fmt.Errorf("error deleting %s: %w", typ, err)
-		}
-	}
-
-	ix := slices.Index(svc.Finalizers, FinalizerName)
-	if ix != -1 {
-		logger.Debug("Removing Tailscale finalizer from Service")
-		svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
-		if err := esr.Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to remove finalizer: %w", err)
-		}
-	}
-	esr.mu.Lock()
-	esr.svcs.Remove(svc.UID)
-	gaugeEgressServices.Set(int64(esr.svcs.Len()))
-	esr.mu.Unlock()
-	logger.Info("successfully cleaned up resources for egress Service")
-	return nil
-}
-
-func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) error {
-	wantsProxyGroup := svc.Annotations[AnnotationProxyGroup]
-	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
-	if cond == nil {
-		return nil
-	}
-	ss := strings.Split(cond.Reason, ":")
-	if len(ss) < 3 {
-		return nil
-	}
-	if strings.EqualFold(wantsProxyGroup, ss[2]) {
-		return nil
-	}
-	esr.logger.Infof("egress Service configured on ProxyGroup %s, wants ProxyGroup %s, cleaning up...", ss[2], wantsProxyGroup)
-	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, l); err != nil {
-		return fmt.Errorf("error deleting egress service config: %w", err)
-	}
-	return nil
-}
-
-// usedPortsForPG calculates the currently used match ports for ProxyGroup
-// containers. It does that by looking by retrieving all target ports of all
-// ClusterIP Services created for egress services exposed on this ProxyGroup's
-// proxies.
-// TODO(irbekrm): this is currently good enough because we only have a single worker and
-// because these Services are created by us, so we can always expect to get the
-// latest ClusterIP Services via the controller cache. It will not work as well
-// once we split into multiple workers- at that point we probably want to set
-// used ports on ProxyGroup's status.
-func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) {
-	svcList := &corev1.ServiceList{}
-	if err := esr.List(ctx, svcList, client.InNamespace(esr.tsNamespace), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
-		return nil, fmt.Errorf("error listing Services: %w", err)
-	}
-	usedPorts := sets.New[int32]()
-	for _, s := range svcList.Items {
-		for _, p := range s.Spec.Ports {
-			usedPorts.Insert(p.TargetPort.IntVal)
-		}
-	}
-	return usedPorts, nil
-}
-
-// clusterIPSvcForEgress returns a template for the ClusterIP Service created
-// for an egress service exposed on ProxyGroup proxies. The ClusterIP Service
-// has no selector. Traffic sent to it will be routed to the endpoints defined
-// by an EndpointSlice created for this egress service.
-func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
-	return &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: svcNameBase(crl[LabelParentName]),
-			Namespace:    esr.tsNamespace,
-			Labels:       crl,
-		},
-		Spec: corev1.ServiceSpec{
-			Type: corev1.ServiceTypeClusterIP,
-		},
-	}
-}
-
-func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
-	crl := egressSvcChildResourceLabels(svc)
-	cmName := pgEgressCMName(crl[labelProxyGroup])
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      cmName,
-			Namespace: esr.tsNamespace,
-		},
-	}
-	l := logger.With("ConfigMap", client.ObjectKeyFromObject(cm))
-	l.Debug("ensuring that egress service configuration is removed from proxy config")
-	if err := esr.Get(ctx, client.ObjectKeyFromObject(cm), cm); apierrors.IsNotFound(err) {
-		l.Debugf("ConfigMap not found")
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("error retrieving ConfigMap: %w", err)
-	}
-	bs := cm.BinaryData[egressservices.KeyEgressServices]
-	if len(bs) == 0 {
-		l.Debugf("ConfigMap does not contain egress service configs")
-		return nil
-	}
-	cfgs := &egressservices.Configs{}
-	if err := json.Unmarshal(bs, cfgs); err != nil {
-		return fmt.Errorf("error unmarshalling egress services configs")
-	}
-	tailnetSvc := tailnetSvcName(svc)
-	_, ok := (*cfgs)[tailnetSvc]
-	if !ok {
-		l.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted")
-		return nil
-	}
-	l.Infof("before deleting config %+#v", *cfgs)
-	delete(*cfgs, tailnetSvc)
-	l.Infof("after deleting config %+#v", *cfgs)
-	bs, err := json.Marshal(cfgs)
-	if err != nil {
-		return fmt.Errorf("error marshalling egress services configs: %w", err)
-	}
-	mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
-	return esr.Update(ctx, cm)
-}
-
-func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (bool, error) {
-	proxyGroupName := svc.Annotations[AnnotationProxyGroup]
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: proxyGroupName,
-		},
-	}
-	if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
-		l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	} else if err != nil {
-		err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, err
-	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-
-	if violations := validateEgressService(svc, pg); len(violations) > 0 {
-		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
-		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
-		l.Info(msg)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-	l.Debugf("egress service is valid")
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
-	return true, nil
-}
-
-func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
-	violations := validateService(svc)
-
-	// We check that only one of these two is set in the earlier validateService function.
-	if svc.Annotations[AnnotationTailnetTargetFQDN] == "" && svc.Annotations[AnnotationTailnetTargetIP] == "" {
-		violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", AnnotationTailnetTargetFQDN, AnnotationTailnetTargetIP))
-	}
-	if len(svc.Spec.Ports) == 0 {
-		violations = append(violations, "egress Service for ProxyGroup must have at least one target Port specified")
-	}
-	if svc.Spec.Type != corev1.ServiceTypeExternalName {
-		violations = append(violations, fmt.Sprintf("unexpected egress Service type %s. The only supported type is ExternalName.", svc.Spec.Type))
-	}
-	if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
-		violations = append(violations, fmt.Sprintf("egress Service references ProxyGroup of type %s, must be type %s", pg.Spec.Type, tsapi.ProxyGroupTypeEgress))
-	}
-	return violations
-}
-
-// egressSvcNameBase returns a name base that can be passed to
-// ObjectMeta.GenerateName to generate a name for the ClusterIP Service.
-// The generated name needs to be short enough so that it can later be used to
-// generate a valid Kubernetes resource name for the EndpointSlice in form
-// 'ipv4-|ipv6-<ClusterIP Service name>.
-// A valid Kubernetes resource name must not be longer than 253 chars.
-func svcNameBase(s string) string {
-	// -ipv4 - ipv6
-	const maxClusterIPSvcNameLength = 253 - 5
-	base := fmt.Sprintf("ts-%s-", s)
-	generator := names.SimpleNameGenerator
-	for {
-		generatedName := generator.GenerateName(base)
-		excess := len(generatedName) - maxClusterIPSvcNameLength
-		if excess <= 0 {
-			return base
-		}
-		base = base[:len(base)-1-excess] // cut off the excess chars
-		base = base + "-"                // re-instate the dash
-	}
-}
-
-// unusedPort returns a port in range [3000 - 4000). The caller must ensure that
-// usedPorts does not contain all ports in range [3000 - 4000).
-func unusedPort(usedPorts sets.Set[int32]) int32 {
-	foundFreePort := false
-	var suggestPort int32
-	for !foundFreePort {
-		suggestPort = rand.Int32N(maxPorts) + 3000
-		if !usedPorts.Has(suggestPort) {
-			foundFreePort = true
-		}
-	}
-	return suggestPort
-}
-
-// tailnetTargetFromSvc returns a tailnet target for the given egress Service.
-// Service must contain exactly one of tailscale.com/tailnet-ip,
-// tailscale.com/tailnet-fqdn annotations.
-func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
-	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
-		return egressservices.TailnetTarget{
-			FQDN: fqdn,
-		}
-	}
-	return egressservices.TailnetTarget{
-		IP: svc.Annotations[AnnotationTailnetTargetIP],
-	}
-}
-
-func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config {
-	tt := tailnetTargetFromSvc(externalNameSvc)
-	cfg := egressservices.Config{TailnetTarget: tt}
-	for _, svcPort := range clusterIPSvc.Spec.Ports {
-		pm := portMap(svcPort)
-		mak.Set(&cfg.Ports, pm, struct{}{})
-	}
-	return cfg
-}
-
-func portMap(p corev1.ServicePort) egressservices.PortMap {
-	// TODO (irbekrm): out of bounds check?
-	return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)}
-}
-
-func isEgressSvcForProxyGroup(obj client.Object) bool {
-	s, ok := obj.(*corev1.Service)
-	if !ok {
-		return false
-	}
-	annots := s.ObjectMeta.Annotations
-	return annots[AnnotationProxyGroup] != "" && (annots[AnnotationTailnetTargetFQDN] != "" || annots[AnnotationTailnetTargetIP] != "")
-}
-
-// egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well
-// as unmarshalled configuration from the ConfigMap.
-func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) {
-	name := pgEgressCMName(proxyGroupName)
-	cm = &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: tsNamespace,
-		},
-	}
-	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
-		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
-	}
-	cfgs = &egressservices.Configs{}
-	if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 {
-		if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], cfgs); err != nil {
-			return nil, nil, fmt.Errorf("error unmarshaling egress services config %v: %w", cm.BinaryData[egressservices.KeyEgressServices], err)
-		}
-	}
-	return cm, cfgs, nil
-}
-
-// egressSvcChildResourceLabels returns labels that should be applied to the
-// ClusterIP Service and the EndpointSlice created for the egress service.
-// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes
-// resource names (ProxyGroup, Service). Maximum allowed label length is 63
-// chars whilst the maximum allowed resource name length is 253 chars, so we
-// should probably validate and truncate (?) the names is they are too long.
-func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
-	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentType:      "svc",
-		LabelParentName:      svc.Name,
-		LabelParentNamespace: svc.Namespace,
-		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
-		labelSvcType:         typeEgress,
-	}
-}
-
-// egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service.
-func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string {
-	l := egressSvcChildResourceLabels(extNSvc)
-	// Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the
-	// endpoints defined on this EndpointSlice.
-	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	l[discoveryv1.LabelServiceName] = clusterIPSvc.Name
-	// Kubernetes recommends setting this label.
-	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management
-	l[discoveryv1.LabelManagedBy] = "tailscale.com"
-	return l
-}
-
-func svcConfigurationUpToDate(svc *corev1.Service, l *zap.SugaredLogger) bool {
-	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
-	if cond == nil {
-		return false
-	}
-	if cond.Status != metav1.ConditionTrue {
-		return false
-	}
-	wantsReadyReason := svcConfiguredReason(svc, true, l)
-	return strings.EqualFold(wantsReadyReason, cond.Reason)
-}
-
-func cfgHash(c cfg, l *zap.SugaredLogger) string {
-	bs, err := json.Marshal(c)
-	if err != nil {
-		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
-		l.Infof("error marhsalling Config: %v", err)
-		return ""
-	}
-	h := sha256.New()
-	if _, err := h.Write(bs); err != nil {
-		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
-		l.Infof("error producing Config hash: %v", err)
-		return ""
-	}
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-type cfg struct {
-	Ports         []corev1.ServicePort         `json:"ports"`
-	TailnetTarget egressservices.TailnetTarget `json:"tailnetTarget"`
-	ProxyGroup    string                       `json:"proxyGroup"`
-}
-
-func svcConfiguredReason(svc *corev1.Service, configured bool, l *zap.SugaredLogger) string {
-	var r string
-	if configured {
-		r = "ConfiguredFor:"
-	} else {
-		r = fmt.Sprintf("ConfigurationFailed:%s", r)
-	}
-	r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[AnnotationProxyGroup])
-	tt := tailnetTargetFromSvc(svc)
-	s := cfg{
-		Ports:         svc.Spec.Ports,
-		TailnetTarget: tt,
-		ProxyGroup:    svc.Annotations[AnnotationProxyGroup],
-	}
-	r += fmt.Sprintf(":Config:%s", cfgHash(s, l))
-	return r
-}
-
-// tailnetSvc accepts and ExternalName Service name and returns a name that will be used to distinguish this tailnet
-// service from other tailnet services exposed to cluster workloads.
-func tailnetSvcName(extNSvc *corev1.Service) string {
-	return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name)
-}
-
-// epsPortsFromSvc takes the ClusterIP Service created for an egress service and
-// returns its Port array in a form that can be used for an EndpointSlice.
-func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
-	for _, p := range svc.Spec.Ports {
-		ep = append(ep, discoveryv1.EndpointPort{
-			Protocol: &p.Protocol,
-			Port:     &p.TargetPort.IntVal,
-			Name:     &p.Name,
-		})
-	}
-	return ep
-}

cmd/k8s-operator/egress-services_test.go

@@ -1,268 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"testing"
-
-	"github.com/AlekSi/pointer"
-	"github.com/google/go-cmp/cmp"
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/tstest"
-	"tailscale.com/tstime"
-)
-
-func TestTailscaleEgressServices(t *testing.T) {
-	pg := &tsapi.ProxyGroup{
-		TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "foo",
-			UID:  types.UID("1234-UID"),
-		},
-		Spec: tsapi.ProxyGroupSpec{
-			Replicas: pointer.To[int32](3),
-			Type:     tsapi.ProxyGroupTypeEgress,
-		},
-	}
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      pgEgressCMName("foo"),
-			Namespace: "operator-ns",
-		},
-	}
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pg, cm).
-		WithStatusSubresource(pg).
-		Build()
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-
-	esr := &egressSvcsReconciler{
-		Client:      fc,
-		logger:      zl.Sugar(),
-		clock:       clock,
-		tsNamespace: "operator-ns",
-	}
-	tailnetTargetFQDN := "foo.bar.ts.net."
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
-				AnnotationProxyGroup:        "foo",
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ExternalName: "placeholder",
-			Type:         corev1.ServiceTypeExternalName,
-			Selector:     nil,
-			Ports: []corev1.ServicePort{
-				{
-					Name:     "http",
-					Protocol: "TCP",
-					Port:     80,
-				},
-				{
-					Name:     "https",
-					Protocol: "TCP",
-					Port:     443,
-				},
-			},
-		},
-	}
-
-	t.Run("proxy_group_not_ready", func(t *testing.T) {
-		mustCreate(t, fc, svc)
-		expectReconciled(t, esr, "default", "test")
-		// Service should have EgressSvcValid condition set to Unknown.
-		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
-		expectEqual(t, fc, svc, nil)
-	})
-
-	t.Run("proxy_group_ready", func(t *testing.T) {
-		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
-			pg.Status.Conditions = []metav1.Condition{
-				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
-			}
-		})
-		// Quirks of the fake client.
-		mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
-			svc.Status.Conditions = []metav1.Condition{}
-		})
-		expectReconciled(t, esr, "default", "test")
-		// Verify that a ClusterIP Service has been created.
-		name := findGenNameForEgressSvcResources(t, fc, svc)
-		expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
-		clusterSvc := mustGetClusterIPSvc(t, fc, name)
-		// Verify that an EndpointSlice has been created.
-		expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
-		// Verify that ConfigMap contains configuration for the new egress service.
-		mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
-		r := svcConfiguredReason(svc, true, zl.Sugar())
-		// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
-		// CluterIP Service.
-		svc.Status.Conditions = []metav1.Condition{
-			condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
-		}
-		svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
-		svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
-		expectEqual(t, fc, svc, nil)
-	})
-
-	t.Run("delete_external_name_service", func(t *testing.T) {
-		name := findGenNameForEgressSvcResources(t, fc, svc)
-		if err := fc.Delete(context.Background(), svc); err != nil {
-			t.Fatalf("error deleting ExternalName Service: %v", err)
-		}
-		expectReconciled(t, esr, "default", "test")
-		// Verify that ClusterIP Service and EndpointSlice have been deleted.
-		expectMissing[corev1.Service](t, fc, "operator-ns", name)
-		expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
-		// Verify that service config has been deleted from the ConfigMap.
-		mustNotHaveConfigForSvc(t, fc, svc, cm)
-	})
-}
-
-func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
-	return metav1.Condition{
-		Type:               string(typ),
-		Status:             st,
-		LastTransitionTime: conditionTime(clock),
-		Reason:             r,
-		Message:            msg,
-	}
-}
-
-func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
-	t.Helper()
-	labels := egressSvcChildResourceLabels(svc)
-	s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
-	if err != nil {
-		t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
-	}
-	if s == nil {
-		t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name)
-	}
-	return s.GetName()
-}
-
-func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
-	labels := egressSvcChildResourceLabels(extNSvc)
-	return &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:         name,
-			Namespace:    "operator-ns",
-			GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name),
-			Labels:       labels,
-		},
-		Spec: corev1.ServiceSpec{
-			Type:  corev1.ServiceTypeClusterIP,
-			Ports: extNSvc.Spec.Ports,
-		},
-	}
-}
-
-func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service {
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: "operator-ns",
-		},
-	}
-	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil {
-		t.Fatalf("error retrieving Service")
-	}
-	return svc
-}
-
-func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
-	labels := egressSvcChildResourceLabels(extNSvc)
-	labels[discoveryv1.LabelManagedBy] = "tailscale.com"
-	labels[discoveryv1.LabelServiceName] = name
-	return &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ipv4", name),
-			Namespace: "operator-ns",
-			Labels:    labels,
-		},
-		Ports:       portsForEndpointSlice(clusterIPSvc),
-		AddressType: discoveryv1.AddressTypeIPv4,
-	}
-}
-
-func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
-	ports := make([]discoveryv1.EndpointPort, 0)
-	for _, p := range svc.Spec.Ports {
-		ports = append(ports, discoveryv1.EndpointPort{
-			Name:     &p.Name,
-			Protocol: &p.Protocol,
-			Port:     pointer.ToInt32(p.TargetPort.IntVal),
-		})
-	}
-	return ports
-}
-
-func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
-	t.Helper()
-	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
-	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
-		t.Fatalf("Error retrieving ConfigMap: %v", err)
-	}
-	name := tailnetSvcName(extNSvc)
-	gotCfg := configFromCM(t, cm, name)
-	if gotCfg == nil {
-		t.Fatalf("No config found for service %q", name)
-	}
-	if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" {
-		t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff)
-	}
-}
-
-func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) {
-	t.Helper()
-	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
-		t.Fatalf("Error retrieving ConfigMap: %v", err)
-	}
-	name := tailnetSvcName(extNSvc)
-	gotCfg := configFromCM(t, cm, name)
-	if gotCfg != nil {
-		t.Fatalf("Config  %#+v for service %q found when it should not be present", gotCfg, name)
-	}
-}
-
-func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config {
-	t.Helper()
-	cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices]
-	if !ok {
-		return nil
-	}
-	cfgs := &egressservices.Configs{}
-	if err := json.Unmarshal(cfgBs, cfgs); err != nil {
-		t.Fatalf("error unmarshalling config: %v", err)
-	}
-	cfg, ok := (*cfgs)[svcName]
-	if ok {
-		return &cfg
-	}
-	return nil
-}

cmd/k8s-operator/generate/main.go

@@ -24,14 +24,10 @@ const (
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
-	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
-	proxyGroupCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
-	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
-	proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -115,7 +111,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector, ProxyClass, DNSConfig, Recorder) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -141,34 +137,28 @@ func generate(baseDir string) error {
 		}
 		return nil
 	}
-	for _, crd := range []struct {
-		crdPath, templatePath string
-	}{
-		{connectorCRDPath, connectorCRDHelmTemplatePath},
-		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
-		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
-		{recorderCRDPath, recorderCRDHelmTemplatePath},
-		{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
-	} {
-		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
-			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
-		}
+	if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil {
+		return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err)
+	}
+	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
+		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
+	}
+	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
+		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
 	}
 	return nil
 }
 
 func cleanup(baseDir string) error {
 	log.Print("Cleaning up CRD from Helm templates")
-	for _, path := range []string{
-		connectorCRDHelmTemplatePath,
-		proxyClassCRDHelmTemplatePath,
-		dnsConfigCRDHelmTemplatePath,
-		recorderCRDHelmTemplatePath,
-		proxyGroupCRDHelmTemplatePath,
-	} {
-		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
-			return fmt.Errorf("error cleaning up %s: %w", path, err)
-		}
+	if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up Connector CRD template: %w", err)
+	}
+	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
+	}
+	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
 	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -59,12 +59,6 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD not found in default chart install")
 	}
-	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
-		t.Errorf("Recorder CRD not found in default chart install")
-	}
-	if !strings.Contains(installContentsWithCRD.String(), "name: proxygroups.tailscale.com") {
-		t.Errorf("ProxyGroup CRD not found in default chart install")
-	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -83,10 +77,4 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
 		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
 	}
-	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
-		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
-	}
-	if strings.Contains(installContentsWithoutCRD.String(), "name: proxygroups.tailscale.com") {
-		t.Errorf("ProxyGroup CRD found in chart install that should not contain a CRD")
-	}
 }

cmd/k8s-operator/ingress.go

@@ -23,7 +23,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/ipn"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -47,14 +46,12 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
-
-	defaultProxyClass string
 }
 
 var (
 	// gaugeIngressResources tracks the number of ingress resources that we're
 	// currently managing.
-	gaugeIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressResourceCount)
+	gaugeIngressResources = clientmetric.NewGauge("k8s_ingress_resources")
 )
 
 func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
@@ -136,7 +133,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing, a.defaultProxyClass)
+	proxyClass := proxyClassForObject(ing)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/ingress_test.go

@@ -17,7 +17,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
@@ -94,7 +93,6 @@ func TestTailscaleIngress(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
-		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -226,7 +224,6 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		namespace:  "default",
 		parentType: "ingress",
 		hostname:   "default-test",
-		app:        kubetypes.AppIngressResource,
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -253,7 +250,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassReady),
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/nameserver.go

@@ -28,7 +28,6 @@ import (
 	"sigs.k8s.io/yaml"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
@@ -63,7 +62,9 @@ type NameserverReconciler struct {
 	managedNameservers set.Slice[types.UID] // one or none
 }
 
-var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
+var (
+	gaugeNameserverResources = clientmetric.NewGauge("k8s_nameserver_resources")
+)
 
 func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := a.logger.With("dnsConfig", req.Name)

cmd/k8s-operator/nameserver_test.go

@@ -33,7 +33,7 @@ func TestNameserverReconciler(t *testing.T) {
 		},
 		Spec: tsapi.DNSConfigSpec{
 			Nameserver: &tsapi.Nameserver{
-				Image: &tsapi.NameserverImage{
+				Image: &tsapi.Image{
 					Repo: "test",
 					Tag:  "v0.0.1",
 				},

cmd/k8s-operator/operator.go

@@ -22,7 +22,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -40,7 +39,6 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/kubestore"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
 	"tailscale.com/types/logger"
@@ -53,8 +51,7 @@ import (
 // Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
 //go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
 
-// Generate CRD API docs.
-//go:generate go run github.com/elastic/crd-ref-docs --renderer=markdown --source-path=../../k8s-operator/apis/ --config=../../k8s-operator/api-docs-config.yaml --output-path=../../k8s-operator/api.md
+// TODO (irbekrm): generate CRD docs from the yamls
 
 func main() {
 	// Required to use our client API. We're fine with the instability since the
@@ -68,7 +65,6 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
-		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
@@ -89,9 +85,9 @@ func main() {
 	// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
 	mode := parseAPIProxyMode()
 	if mode == apiserverProxyModeDisabled {
-		hostinfo.SetApp(kubetypes.AppOperator)
+		hostinfo.SetApp("k8s-operator")
 	} else {
-		hostinfo.SetApp(kubetypes.AppAPIServerProxy)
+		hostinfo.SetApp("k8s-operator-proxy")
 	}
 
 	s, tsClient := initTSNet(zlog)
@@ -109,7 +105,6 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
-		defaultProxyClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -143,7 +138,6 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
 	}
 	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-k8s-operator"
 	tsClient.HTTPClient = credentials.Client(context.Background())
 
 	s := &tsnet.Server{
@@ -239,13 +233,10 @@ func runReconcilers(opts reconcilerOpts) {
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.Secret{}:             nsFilter,
 				&corev1.ServiceAccount{}:     nsFilter,
-				&corev1.Pod{}:                nsFilter,
 				&corev1.ConfigMap{}:          nsFilter,
 				&appsv1.StatefulSet{}:        nsFilter,
 				&appsv1.Deployment{}:         nsFilter,
 				&discoveryv1.EndpointSlice{}: nsFilter,
-				&rbacv1.Role{}:               nsFilter,
-				&rbacv1.RoleBinding{}:        nsFilter,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
@@ -287,7 +278,6 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
-			defaultProxyClass:     opts.defaultProxyClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -306,11 +296,10 @@ func runReconcilers(opts reconcilerOpts) {
 		Watches(&corev1.Service{}, svcHandlerForIngress).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress).
 		Complete(&IngressReconciler{
-			ssr:               ssr,
-			recorder:          eventRecorder,
-			Client:            mgr.GetClient(),
-			logger:            opts.log.Named("ingress-reconciler"),
-			defaultProxyClass: opts.defaultProxyClass,
+			ssr:      ssr,
+			recorder: eventRecorder,
+			Client:   mgr.GetClient(),
+			logger:   opts.log.Named("ingress-reconciler"),
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -355,65 +344,6 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create nameserver reconciler: %v", err)
 	}
-
-	egressSvcFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsHandler)
-	egressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsFromEgressProxyGroup(mgr.GetClient(), opts.log))
-	err = builder.
-		ControllerManagedBy(mgr).
-		Named("egress-svcs-reconciler").
-		Watches(&corev1.Service{}, egressSvcFilter).
-		Watches(&tsapi.ProxyGroup{}, egressProxyGroupFilter).
-		Complete(&egressSvcsReconciler{
-			Client:      mgr.GetClient(),
-			tsNamespace: opts.tailscaleNamespace,
-			recorder:    eventRecorder,
-			clock:       tstime.DefaultClock{},
-			logger:      opts.log.Named("egress-svcs-reconciler"),
-		})
-	if err != nil {
-		startlog.Fatalf("could not create egress Services reconciler: %v", err)
-	}
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexEgressProxyGroup, indexEgressServices); err != nil {
-		startlog.Fatalf("failed setting up indexer for egress Services: %v", err)
-	}
-
-	egressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(egressSvcFromEps)
-	err = builder.
-		ControllerManagedBy(mgr).
-		Named("egress-svcs-readiness-reconciler").
-		Watches(&corev1.Service{}, egressSvcFilter).
-		Watches(&discoveryv1.EndpointSlice{}, egressSvcFromEpsFilter).
-		Complete(&egressSvcsReadinessReconciler{
-			Client:      mgr.GetClient(),
-			tsNamespace: opts.tailscaleNamespace,
-			clock:       tstime.DefaultClock{},
-			logger:      opts.log.Named("egress-svcs-readiness-reconciler"),
-		})
-	if err != nil {
-		startlog.Fatalf("could not create egress Services readiness reconciler: %v", err)
-	}
-
-	epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
-	podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace))
-	secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace))
-	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
-
-	err = builder.
-		ControllerManagedBy(mgr).
-		Named("egress-eps-reconciler").
-		Watches(&discoveryv1.EndpointSlice{}, epsFilter).
-		Watches(&corev1.Pod{}, podsFilter).
-		Watches(&corev1.Secret{}, secretsFilter).
-		Watches(&corev1.Service{}, epsFromExtNSvcFilter).
-		Complete(&egressEpsReconciler{
-			Client:      mgr.GetClient(),
-			tsNamespace: opts.tailscaleNamespace,
-			logger:      opts.log.Named("egress-eps-reconciler"),
-		})
-	if err != nil {
-		startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
-	}
-
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
 		Complete(&ProxyClassReconciler{
@@ -453,56 +383,6 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create DNS records reconciler: %v", err)
 	}
-
-	// Recorder reconciler.
-	recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
-	err = builder.ControllerManagedBy(mgr).
-		For(&tsapi.Recorder{}).
-		Watches(&appsv1.StatefulSet{}, recorderFilter).
-		Watches(&corev1.ServiceAccount{}, recorderFilter).
-		Watches(&corev1.Secret{}, recorderFilter).
-		Watches(&rbacv1.Role{}, recorderFilter).
-		Watches(&rbacv1.RoleBinding{}, recorderFilter).
-		Complete(&RecorderReconciler{
-			recorder:    eventRecorder,
-			tsNamespace: opts.tailscaleNamespace,
-			Client:      mgr.GetClient(),
-			l:           opts.log.Named("recorder-reconciler"),
-			clock:       tstime.DefaultClock{},
-			tsClient:    opts.tsClient,
-		})
-	if err != nil {
-		startlog.Fatalf("could not create Recorder reconciler: %v", err)
-	}
-
-	// Recorder reconciler.
-	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
-	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
-	err = builder.ControllerManagedBy(mgr).
-		For(&tsapi.ProxyGroup{}).
-		Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
-		Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
-		Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
-		Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
-		Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
-		Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
-		Complete(&ProxyGroupReconciler{
-			recorder: eventRecorder,
-			Client:   mgr.GetClient(),
-			l:        opts.log.Named("proxygroup-reconciler"),
-			clock:    tstime.DefaultClock{},
-			tsClient: opts.tsClient,
-
-			tsNamespace:       opts.tailscaleNamespace,
-			proxyImage:        opts.proxyImage,
-			defaultTags:       strings.Split(opts.proxyTags, ","),
-			tsFirewallMode:    opts.proxyFirewallMode,
-			defaultProxyClass: opts.defaultProxyClass,
-		})
-	if err != nil {
-		startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)
-	}
-
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
@@ -543,10 +423,6 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
-	// defaultProxyClass is the name of the ProxyClass to use as the default
-	// class for proxies that do not have a ProxyClass set.
-	// this is defined by an operator env variable.
-	defaultProxyClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
@@ -639,7 +515,6 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
-	Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error
 }
 
@@ -735,27 +610,6 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass,
-// returns a list of reconcile requests for all Connectors that have
-// .spec.proxyClass set.
-func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(ctx context.Context, o client.Object) []reconcile.Request {
-		pgList := new(tsapi.ProxyGroupList)
-		if err := cl.List(ctx, pgList); err != nil {
-			logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		proxyClassName := o.GetName()
-		for _, pg := range pgList.Items {
-			if pg.Spec.ProxyClass == proxyClassName {
-				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
-			}
-		}
-		return reqs
-	}
-}
-
 // serviceHandlerForIngress returns a handler for Service events for ingress
 // reconciler that ensures that if the Service associated with an event is of
 // interest to the reconciler, the associated Ingress(es) gets be reconciled.
@@ -797,10 +651,6 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl
 }
 
 func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
-	if _, ok := o.GetAnnotations()[AnnotationProxyGroup]; ok {
-		// Do not reconcile Services for ProxyGroup.
-		return nil
-	}
 	if isManagedByType(o, "svc") {
 		// If this is a Service managed by a Service we want to enqueue its parent
 		return []reconcile.Request{{NamespacedName: parentFromObjectLabels(o)}}
@@ -826,195 +676,3 @@ func isMagicDNSName(name string) bool {
 	validMagicDNSName := regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`)
 	return validMagicDNSName.MatchString(name)
 }
-
-// egressSvcsHandler returns accepts a Kubernetes object and returns a reconcile
-// request for it , if the object is a Tailscale egress Service meant to be
-// exposed on a ProxyGroup.
-func egressSvcsHandler(_ context.Context, o client.Object) []reconcile.Request {
-	if !isEgressSvcForProxyGroup(o) {
-		return nil
-	}
-	return []reconcile.Request{
-		{
-			NamespacedName: types.NamespacedName{
-				Namespace: o.GetNamespace(),
-				Name:      o.GetName(),
-			},
-		},
-	}
-}
-
-// egressEpsHandler returns accepts an EndpointSlice and, if the EndpointSlice
-// is for an egress service, returns a reconcile request for it.
-func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
-	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
-		return nil
-	}
-	return []reconcile.Request{
-		{
-			NamespacedName: types.NamespacedName{
-				Namespace: o.GetNamespace(),
-				Name:      o.GetName(),
-			},
-		},
-	}
-}
-
-// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
-// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
-func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
-			return nil
-		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
-		if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
-			return nil
-		}
-		pg, ok := o.GetLabels()[LabelParentName]
-		if !ok {
-			return nil
-		}
-		return reconcileRequestsForPG(pg, cl, ns)
-	}
-}
-
-// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is,
-// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
-func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
-			return nil
-		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
-		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
-			return nil
-		}
-		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
-			return nil
-		}
-		pg, ok := o.GetLabels()[LabelParentName]
-		if !ok {
-			return nil
-		}
-		return reconcileRequestsForPG(pg, cl, ns)
-	}
-}
-
-// egressSvcFromEps is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service
-// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service.
-func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
-	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
-		return nil
-	}
-	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
-		return nil
-	}
-	svcName, ok := o.GetLabels()[LabelParentName]
-	if !ok {
-		return nil
-	}
-	svcNs, ok := o.GetLabels()[LabelParentNamespace]
-	if !ok {
-		return nil
-	}
-	return []reconcile.Request{
-		{
-			NamespacedName: types.NamespacedName{
-				Namespace: svcNs,
-				Name:      svcName,
-			},
-		},
-	}
-}
-
-func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request {
-	epsList := discoveryv1.EndpointSliceList{}
-	if err := cl.List(context.Background(), &epsList,
-		client.InNamespace(ns),
-		client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
-		return nil
-	}
-	reqs := make([]reconcile.Request, 0)
-	for _, ep := range epsList.Items {
-		reqs = append(reqs, reconcile.Request{
-			NamespacedName: types.NamespacedName{
-				Namespace: ep.Namespace,
-				Name:      ep.Name,
-			},
-		})
-	}
-	return reqs
-}
-
-// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
-// user-created ExternalName Services that should be exposed on this ProxyGroup.
-func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		pg, ok := o.(*tsapi.ProxyGroup)
-		if !ok {
-			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
-			return nil
-		}
-		if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
-			return nil
-		}
-		svcList := &corev1.ServiceList{}
-		if err := cl.List(context.Background(), svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
-			logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, svc := range svcList.Items {
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: svc.Namespace,
-					Name:      svc.Name,
-				},
-			})
-		}
-		return reqs
-	}
-}
-
-// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
-// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
-func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		svc, ok := o.(*corev1.Service)
-		if !ok {
-			logger.Infof("[unexpected] Service handler triggered for an object that is not a Service")
-			return nil
-		}
-		if !isEgressSvcForProxyGroup(svc) {
-			return nil
-		}
-		epsList := &discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
-			client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
-			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, eps := range epsList.Items {
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: eps.Namespace,
-					Name:      eps.Name,
-				},
-			})
-		}
-		return reqs
-	}
-}
-
-// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
-// index is used a list filter.
-func indexEgressServices(o client.Object) []string {
-	if !isEgressSvcForProxyGroup(o) {
-		return nil
-	}
-	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
-}

cmd/k8s-operator/operator_test.go

@@ -22,7 +22,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime"
@@ -124,7 +123,6 @@ func TestLoadBalancerClass(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
@@ -262,7 +260,6 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		parentType:        "svc",
 		tailnetTargetFQDN: tailnetTargetFQDN,
 		hostname:          "default-test",
-		app:               kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -374,7 +371,6 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		parentType:      "svc",
 		tailnetTargetIP: tailnetTargetIP,
 		hostname:        "default-test",
-		app:             kubetypes.AppEgressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -432,148 +428,6 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 }
 
-func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:   zl.Sugar(),
-		clock:    clock,
-		recorder: record.NewFakeRecorder(100),
-	}
-	tailnetTargetIP := "invalid-ip"
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-
-			UID: types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	t0 := conditionTime(clock)
-
-	want := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-		Status: corev1.ServiceStatus{
-			Conditions: []metav1.Condition{{
-				Type:               string(tsapi.ProxyReady),
-				Status:             metav1.ConditionFalse,
-				LastTransitionTime: t0,
-				Reason:             reasonProxyInvalid,
-				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "invalid-ip" could not be parsed as a valid IP Address, error: ParseAddr("invalid-ip"): unable to parse IP`,
-			}},
-		},
-	}
-
-	expectEqual(t, fc, want, nil)
-}
-
-func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:   zl.Sugar(),
-		clock:    clock,
-		recorder: record.NewFakeRecorder(100),
-	}
-	tailnetTargetIP := "999.999.999.999"
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-
-			UID: types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	t0 := conditionTime(clock)
-
-	want := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-		Status: corev1.ServiceStatus{
-			Conditions: []metav1.Condition{{
-				Type:               string(tsapi.ProxyReady),
-				Status:             metav1.ConditionFalse,
-				LastTransitionTime: t0,
-				Reason:             reasonProxyInvalid,
-				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "999.999.999.999" could not be parsed as a valid IP Address, error: ParseAddr("999.999.999.999"): IPv4 field has value >255`,
-			}},
-		},
-	}
-
-	expectEqual(t, fc, want, nil)
-}
-
 func TestAnnotations(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
@@ -625,7 +479,6 @@ func TestAnnotations(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -731,7 +584,6 @@ func TestAnnotationIntoLB(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -861,7 +713,6 @@ func TestLBIntoAnnotation(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -1001,7 +852,6 @@ func TestCustomHostname(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "reindeer-flotilla",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
@@ -1114,7 +964,6 @@ func TestCustomPriorityClassName(t *testing.T) {
 		hostname:          "tailscale-critical",
 		priorityClassName: "custom-priority-class-name",
 		clusterTargetIP:   "10.20.30.40",
-		app:               kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@@ -1183,7 +1032,6 @@ func TestProxyClassForService(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
@@ -1206,7 +1054,7 @@ func TestProxyClassForService(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassReady),
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
@@ -1277,7 +1125,6 @@ func TestDefaultLoadBalancer(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -1334,7 +1181,6 @@ func TestProxyFirewallMode(t *testing.T) {
 		hostname:        "default-test",
 		firewallMode:    "nftables",
 		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 }
@@ -1389,7 +1235,6 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
 		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
-		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 
@@ -1629,72 +1474,6 @@ func Test_clusterDomainFromResolverConf(t *testing.T) {
 		})
 	}
 }
-func Test_authKeyRemoval(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// 1. A new Service that should be exposed via Tailscale gets created, a Secret with a config that contains auth
-	// key is generated.
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger: zl.Sugar(),
-		clock:  clock,
-	}
-
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
-	opts := configOpts{
-		stsName:         shortName,
-		secretName:      fullName,
-		namespace:       "default",
-		parentType:      "svc",
-		hostname:        "default-test",
-		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
-	}
-
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
-
-	// 2. Apply update to the Secret that imitates the proxy setting device_id.
-	s := expectedSecret(t, fc, opts)
-	mustUpdate(t, fc, s.Namespace, s.Name, func(s *corev1.Secret) {
-		mak.Set(&s.Data, "device_id", []byte("dkkdi4CNTRL"))
-	})
-
-	// 3. Config should no longer contain auth key
-	expectReconciled(t, sr, "default", "test")
-	opts.shouldRemoveAuthKey = true
-	opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-}
 
 func Test_externalNameService(t *testing.T) {
 	fc := fake.NewFakeClient()
@@ -1750,7 +1529,6 @@ func Test_externalNameService(t *testing.T) {
 		parentType:       "svc",
 		hostname:         "default-test",
 		clusterTargetDNS: "foo.com",
-		app:              kubetypes.AppIngressProxy,
 	}
 
 	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)

cmd/k8s-operator/proxy.go

@@ -22,8 +22,8 @@ import (
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
-	ksr "tailscale.com/k8s-operator/sessionrecording"
-	"tailscale.com/kube/kubetypes"
+	tskube "tailscale.com/kube"
+	"tailscale.com/ssh/tailssh"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -31,10 +31,17 @@ import (
 	"tailscale.com/util/set"
 )
 
+var whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
+
 var (
 	// counterNumRequestsproxies counts the number of API server requests proxied via this proxy.
 	counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
-	whoIsKey                  = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
+
+	// counterSessionRecordingsAttempted counts the number of session recording attempts.
+	counterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy__session_recordings_attempted")
+
+	// counterSessionRecordingsUploaded counts the number of successfully uploaded session recordings.
+	counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded")
 )
 
 type apiServerProxyMode int
@@ -166,8 +173,7 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", ap.serveDefault)
-	mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
-	mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
+	mux.HandleFunc("/api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExec)
 
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
@@ -208,25 +214,9 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
 	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
-// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
-// optionally configuring the kubectl exec sessions to be recorded.
-func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
-	ap.execForProto(w, r, ksr.SPDYProtocol)
-}
-
-// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
-// optionally configuring the kubectl exec sessions to be recorded.
-func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
-	ap.execForProto(w, r, ksr.WSProtocol)
-}
-
-func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
-	const (
-		podNameKey       = "pod"
-		namespaceNameKey = "namespace"
-		upgradeHeaderKey = "Upgrade"
-	)
-
+// serveExec serves 'kubectl exec' requests, optionally configuring the kubectl
+// exec sessions to be recorded.
+func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
 	who, err := ap.whoIs(r)
 	if err != nil {
 		ap.authError(w, err)
@@ -242,17 +232,15 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p
 		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 		return
 	}
-	ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
+	counterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
 	if !failOpen && len(addrs) == 0 {
 		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
 		ap.log.Error(msg)
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-
-	wantsHeader := upgradeHeaderForProto[proto]
-	if h := r.Header.Get(upgradeHeaderKey); h != wantsHeader {
-		msg := fmt.Sprintf("[unexpected] unable to verify that streaming protocol is %s, wants Upgrade header %q, got: %q", proto, wantsHeader, h)
+	if r.Method != "POST" || r.Header.Get("Upgrade") != "SPDY/3.1" {
+		msg := "'kubectl exec' session recording is configured, but the request is not over SPDY. Session recording is currently only supported for SPDY based clients"
 		if failOpen {
 			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
 			ap.log.Warn(msg)
@@ -264,22 +252,20 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p
 		http.Error(w, msg, http.StatusForbidden)
 		return
 	}
-
-	opts := ksr.HijackerOpts{
-		Req:       r,
-		W:         w,
-		Proto:     proto,
-		TS:        ap.ts,
-		Who:       who,
-		Addrs:     addrs,
-		FailOpen:  failOpen,
-		Pod:       r.PathValue(podNameKey),
-		Namespace: r.PathValue(namespaceNameKey),
-		Log:       ap.log,
+	spdyH := &spdyHijacker{
+		ts:                ap.ts,
+		req:               r,
+		who:               who,
+		ResponseWriter:    w,
+		log:               ap.log,
+		pod:               r.PathValue("pod"),
+		ns:                r.PathValue("namespace"),
+		addrs:             addrs,
+		failOpen:          failOpen,
+		connectToRecorder: tailssh.ConnectToRecorder,
 	}
-	h := ksr.New(opts)
 
-	ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
 func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
@@ -314,11 +300,9 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
 		log.Printf("failed to add impersonation headers: " + err.Error())
 	}
 }
-
 func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
 	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
 }
-
 func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
 	ap.log.Errorf("failed to authenticate caller: %v", err)
 	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
@@ -339,10 +323,10 @@ const (
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)
@@ -392,7 +376,7 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 		return false, nil, errors.New("[unexpected] cannot determine caller")
 	}
 	failOpen = true
-	rules, err := tailcfg.UnmarshalCapJSON[kubetypes.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if err != nil {
 		return failOpen, nil, fmt.Errorf("failed to unmarshal Kubernetes capability: %w", err)
 	}
@@ -414,8 +398,3 @@ func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorde
 	}
 	return failOpen, recorderAddresses, nil
 }
-
-var upgradeHeaderForProto = map[ksr.Protocol]string{
-	ksr.SPDYProtocol: "SPDY/3.1",
-	ksr.WSProtocol:   "websocket",
-}

cmd/k8s-operator/proxyclass.go

@@ -98,9 +98,9 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	if errs := pcr.validate(pc); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
 		pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
-		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
+		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
 	} else {
-		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
+		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
 	}
 	if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) {
 		if err := pcr.Client.Status().Update(ctx, pc); err != nil {

cmd/k8s-operator/proxyclass_test.go

@@ -69,7 +69,7 @@ func TestProxyClass(t *testing.T) {
 	// 1. A valid ProxyClass resource gets its status updated to Ready.
 	expectReconciled(t, pcr, "", "test")
 	pc.Status.Conditions = append(pc.Status.Conditions, metav1.Condition{
-		Type:               string(tsapi.ProxyClassReady),
+		Type:               string(tsapi.ProxyClassready),
 		Status:             metav1.ConditionTrue,
 		Reason:             reasonProxyClassValid,
 		Message:            reasonProxyClassValid,
@@ -85,7 +85,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
 	expectEvents(t, fr, []string{expectedEvent})
@@ -99,7 +99,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
 	expectEvents(t, fr, []string{expectedEvent})
@@ -118,7 +118,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
 	expectEvents(t, fr, []string{expectedEvent})

cmd/k8s-operator/proxygroup.go

@@ -1,533 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"slices"
-	"sync"
-
-	"github.com/pkg/errors"
-	"go.uber.org/zap"
-	xslices "golang.org/x/exp/slices"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"tailscale.com/client/tailscale"
-	"tailscale.com/ipn"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
-	"tailscale.com/tailcfg"
-	"tailscale.com/tstime"
-	"tailscale.com/types/ptr"
-	"tailscale.com/util/clientmetric"
-	"tailscale.com/util/mak"
-	"tailscale.com/util/set"
-)
-
-const (
-	reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
-	reasonProxyGroupReady          = "ProxyGroupReady"
-	reasonProxyGroupCreating       = "ProxyGroupCreating"
-	reasonProxyGroupInvalid        = "ProxyGroupInvalid"
-)
-
-var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupCount)
-
-// ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
-type ProxyGroupReconciler struct {
-	client.Client
-	l        *zap.SugaredLogger
-	recorder record.EventRecorder
-	clock    tstime.Clock
-	tsClient tsClient
-
-	// User-specified defaults from the helm installation.
-	tsNamespace       string
-	proxyImage        string
-	defaultTags       []string
-	tsFirewallMode    string
-	defaultProxyClass string
-
-	mu          sync.Mutex           // protects following
-	proxyGroups set.Slice[types.UID] // for proxygroups gauge
-}
-
-func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
-	return r.l.With("ProxyGroup", name)
-}
-
-func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := r.logger(req.Name)
-	logger.Debugf("starting reconcile")
-	defer logger.Debugf("reconcile finished")
-
-	pg := new(tsapi.ProxyGroup)
-	err = r.Get(ctx, req.NamespacedName, pg)
-	if apierrors.IsNotFound(err) {
-		logger.Debugf("ProxyGroup not found, assuming it was deleted")
-		return reconcile.Result{}, nil
-	} else if err != nil {
-		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyGroup: %w", err)
-	}
-	if markedForDeletion(pg) {
-		logger.Debugf("ProxyGroup is being deleted, cleaning up resources")
-		ix := xslices.Index(pg.Finalizers, FinalizerName)
-		if ix < 0 {
-			logger.Debugf("no finalizer, nothing to do")
-			return reconcile.Result{}, nil
-		}
-
-		if done, err := r.maybeCleanup(ctx, pg); err != nil {
-			return reconcile.Result{}, err
-		} else if !done {
-			logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...")
-			return reconcile.Result{RequeueAfter: shortRequeue}, nil
-		}
-
-		pg.Finalizers = slices.Delete(pg.Finalizers, ix, ix+1)
-		if err := r.Update(ctx, pg); err != nil {
-			return reconcile.Result{}, err
-		}
-		return reconcile.Result{}, nil
-	}
-
-	oldPGStatus := pg.Status.DeepCopy()
-	setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldPGStatus, pg.Status) {
-			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
-		}
-		return reconcile.Result{}, err
-	}
-
-	if !slices.Contains(pg.Finalizers, FinalizerName) {
-		// This log line is printed exactly once during initial provisioning,
-		// because once the finalizer is in place this block gets skipped. So,
-		// this is a nice place to log that the high level, multi-reconcile
-		// operation is underway.
-		logger.Infof("ensuring ProxyGroup is set up")
-		pg.Finalizers = append(pg.Finalizers, FinalizerName)
-		if err = r.Update(ctx, pg); err != nil {
-			err = fmt.Errorf("error adding finalizer: %w", err)
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, reasonProxyGroupCreationFailed)
-		}
-	}
-
-	if err = r.validate(pg); err != nil {
-		message := fmt.Sprintf("ProxyGroup is invalid: %s", err)
-		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupInvalid, message)
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupInvalid, message)
-	}
-
-	proxyClassName := r.defaultProxyClass
-	if pg.Spec.ProxyClass != "" {
-		proxyClassName = pg.Spec.ProxyClass
-	}
-
-	var proxyClass *tsapi.ProxyClass
-	if proxyClassName != "" {
-		proxyClass = new(tsapi.ProxyClass)
-		err := r.Get(ctx, types.NamespacedName{Name: proxyClassName}, proxyClass)
-		if apierrors.IsNotFound(err) {
-			err = nil
-			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s does not (yet) exist", proxyClassName)
-			logger.Info(message)
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-		}
-		if err != nil {
-			err = fmt.Errorf("error getting ProxyGroup's ProxyClass %s: %s", proxyClassName, err)
-			r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
-		}
-		if !tsoperator.ProxyClassIsReady(proxyClass) {
-			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName)
-			logger.Info(message)
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-		}
-	}
-
-	if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
-		err = fmt.Errorf("error provisioning ProxyGroup resources: %w", err)
-		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
-	}
-
-	desiredReplicas := int(pgReplicas(pg))
-	if len(pg.Status.Devices) < desiredReplicas {
-		message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
-		logger.Debug(message)
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-	}
-
-	if len(pg.Status.Devices) > desiredReplicas {
-		message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas)
-		logger.Debug(message)
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-	}
-
-	logger.Info("ProxyGroup resources synced")
-	return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady)
-}
-
-func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
-	logger := r.logger(pg.Name)
-	r.mu.Lock()
-	r.proxyGroups.Add(pg.UID)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
-	r.mu.Unlock()
-
-	cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass)
-	if err != nil {
-		return fmt.Errorf("error provisioning config Secrets: %w", err)
-	}
-	// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
-	stateSecrets := pgStateSecrets(pg, r.tsNamespace)
-	for _, sec := range stateSecrets {
-		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
-			s.ObjectMeta.Labels = sec.ObjectMeta.Labels
-			s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
-			s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
-		}); err != nil {
-			return fmt.Errorf("error provisioning state Secrets: %w", err)
-		}
-	}
-	sa := pgServiceAccount(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
-		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
-	}); err != nil {
-		return fmt.Errorf("error provisioning ServiceAccount: %w", err)
-	}
-	role := pgRole(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
-		r.ObjectMeta.Labels = role.ObjectMeta.Labels
-		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
-		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
-		r.Rules = role.Rules
-	}); err != nil {
-		return fmt.Errorf("error provisioning Role: %w", err)
-	}
-	roleBinding := pgRoleBinding(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
-		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
-		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
-		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
-		r.RoleRef = roleBinding.RoleRef
-		r.Subjects = roleBinding.Subjects
-	}); err != nil {
-		return fmt.Errorf("error provisioning RoleBinding: %w", err)
-	}
-	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-		cm := pgEgressCM(pg, r.tsNamespace)
-		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
-			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
-			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
-		}); err != nil {
-			return fmt.Errorf("error provisioning ConfigMap: %w", err)
-		}
-	}
-	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
-	if err != nil {
-		return fmt.Errorf("error generating StatefulSet spec: %w", err)
-	}
-	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
-		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
-		s.Spec = ss.Spec
-	}); err != nil {
-		return fmt.Errorf("error provisioning StatefulSet: %w", err)
-	}
-
-	if err := r.cleanupDanglingResources(ctx, pg); err != nil {
-		return fmt.Errorf("error cleaning up dangling resources: %w", err)
-	}
-
-	devices, err := r.getDeviceInfo(ctx, pg)
-	if err != nil {
-		return fmt.Errorf("failed to get device info: %w", err)
-	}
-
-	pg.Status.Devices = devices
-
-	return nil
-}
-
-// cleanupDanglingResources ensures we don't leak config secrets, state secrets, and
-// tailnet devices when the number of replicas specified is reduced.
-func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error {
-	logger := r.logger(pg.Name)
-	metadata, err := r.getNodeMetadata(ctx, pg)
-	if err != nil {
-		return err
-	}
-
-	for _, m := range metadata {
-		if m.ordinal+1 <= int(pgReplicas(pg)) {
-			continue
-		}
-
-		// Dangling resource, delete the config + state Secrets, as well as
-		// deleting the device from the tailnet.
-		if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
-			return err
-		}
-		if err := r.Delete(ctx, m.stateSecret); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err)
-			}
-		}
-		configSecret := m.stateSecret.DeepCopy()
-		configSecret.Name += "-config"
-		if err := r.Delete(ctx, configSecret); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err)
-			}
-		}
-	}
-
-	return nil
-}
-
-// maybeCleanup just deletes the device from the tailnet. All the kubernetes
-// resources linked to a ProxyGroup will get cleaned up via owner references
-// (which we can use because they are all in the same namespace).
-func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.ProxyGroup) (bool, error) {
-	logger := r.logger(pg.Name)
-
-	metadata, err := r.getNodeMetadata(ctx, pg)
-	if err != nil {
-		return false, err
-	}
-
-	for _, m := range metadata {
-		if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
-			return false, err
-		}
-	}
-
-	logger.Infof("cleaned up ProxyGroup resources")
-	r.mu.Lock()
-	r.proxyGroups.Remove(pg.UID)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
-	r.mu.Unlock()
-	return true, nil
-}
-
-func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error {
-	logger.Debugf("deleting device %s from control", string(id))
-	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
-		errResp := &tailscale.ErrResponse{}
-		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
-			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
-		} else {
-			return fmt.Errorf("error deleting device: %w", err)
-		}
-	} else {
-		logger.Debugf("device %s deleted from control", string(id))
-	}
-
-	return nil
-}
-
-func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) {
-	logger := r.logger(pg.Name)
-	var allConfigs []tailscaledConfigs
-	for i := range pgReplicas(pg) {
-		cfgSecret := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:            fmt.Sprintf("%s-%d-config", pg.Name, i),
-				Namespace:       r.tsNamespace,
-				Labels:          pgSecretLabels(pg.Name, "config"),
-				OwnerReferences: pgOwnerReference(pg),
-			},
-		}
-
-		var existingCfgSecret *corev1.Secret // unmodified copy of secret
-		if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
-			logger.Debugf("secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
-			existingCfgSecret = cfgSecret.DeepCopy()
-		} else if !apierrors.IsNotFound(err) {
-			return "", err
-		}
-
-		var authKey string
-		if existingCfgSecret == nil {
-			logger.Debugf("creating authkey for new ProxyGroup proxy")
-			tags := pg.Spec.Tags.Stringify()
-			if len(tags) == 0 {
-				tags = r.defaultTags
-			}
-			authKey, err = newAuthKey(ctx, r.tsClient, tags)
-			if err != nil {
-				return "", err
-			}
-		}
-
-		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret)
-		if err != nil {
-			return "", fmt.Errorf("error creating tailscaled config: %w", err)
-		}
-		allConfigs = append(allConfigs, configs)
-
-		for cap, cfg := range configs {
-			cfgJSON, err := json.Marshal(cfg)
-			if err != nil {
-				return "", fmt.Errorf("error marshalling tailscaled config: %w", err)
-			}
-			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
-		}
-
-		if existingCfgSecret != nil {
-			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
-			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
-				return "", err
-			}
-		} else {
-			logger.Debugf("creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
-			if err := r.Create(ctx, cfgSecret); err != nil {
-				return "", err
-			}
-		}
-	}
-
-	sum := sha256.New()
-	b, err := json.Marshal(allConfigs)
-	if err != nil {
-		return "", err
-	}
-	if _, err := sum.Write(b); err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%x", sum.Sum(nil)), nil
-}
-
-func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
-	conf := &ipn.ConfigVAlpha{
-		Version:      "alpha0",
-		AcceptDNS:    "false",
-		AcceptRoutes: "false", // AcceptRoutes defaults to true
-		Locked:       "false",
-		Hostname:     ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)),
-	}
-
-	if pg.Spec.HostnamePrefix != "" {
-		conf.Hostname = ptr.To(fmt.Sprintf("%s%d", pg.Spec.HostnamePrefix, idx))
-	}
-
-	if shouldAcceptRoutes(class) {
-		conf.AcceptRoutes = "true"
-	}
-
-	deviceAuthed := false
-	for _, d := range pg.Status.Devices {
-		if d.Hostname == *conf.Hostname {
-			deviceAuthed = true
-			break
-		}
-	}
-
-	if authKey != "" {
-		conf.AuthKey = &authKey
-	} else if !deviceAuthed {
-		key, err := authKeyFromSecret(oldSecret)
-		if err != nil {
-			return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
-		}
-		conf.AuthKey = key
-	}
-	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
-	capVerConfigs[106] = *conf
-	return capVerConfigs, nil
-}
-
-func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error {
-	return nil
-}
-
-// getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by
-// querying their state Secrets. It may not return the same number of items as
-// specified in the ProxyGroup spec if e.g. it is getting scaled up or down, or
-// some pods have failed to write state.
-func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) {
-	// List all state secrets owned by this ProxyGroup.
-	secrets := &corev1.SecretList{}
-	if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil {
-		return nil, fmt.Errorf("failed to list state Secrets: %w", err)
-	}
-	for _, secret := range secrets.Items {
-		var ordinal int
-		if _, err := fmt.Sscanf(secret.Name, pg.Name+"-%d", &ordinal); err != nil {
-			return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err)
-		}
-
-		id, dnsName, ok, err := getNodeMetadata(ctx, &secret)
-		if err != nil {
-			return nil, err
-		}
-		if !ok {
-			continue
-		}
-
-		metadata = append(metadata, nodeMetadata{
-			ordinal:     ordinal,
-			stateSecret: &secret,
-			tsID:        id,
-			dnsName:     dnsName,
-		})
-	}
-
-	return metadata, nil
-}
-
-func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
-	metadata, err := r.getNodeMetadata(ctx, pg)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, m := range metadata {
-		device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret)
-		if err != nil {
-			return nil, err
-		}
-		if !ok {
-			continue
-		}
-		devices = append(devices, tsapi.TailnetDevice{
-			Hostname:   device.Hostname,
-			TailnetIPs: device.TailnetIPs,
-		})
-	}
-
-	return devices, nil
-}
-
-type nodeMetadata struct {
-	ordinal     int
-	stateSecret *corev1.Secret
-	tsID        tailcfg.StableNodeID
-	dnsName     string
-}

cmd/k8s-operator/proxygroup_specs.go

@@ -1,294 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"fmt"
-
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/yaml"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/types/ptr"
-)
-
-// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
-// applied over the top after.
-func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) {
-	ss := new(appsv1.StatefulSet)
-	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
-	}
-	// Validate some base assumptions.
-	if len(ss.Spec.Template.Spec.InitContainers) != 1 {
-		return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers))
-	}
-	if len(ss.Spec.Template.Spec.Containers) != 1 {
-		return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers))
-	}
-
-	// StatefulSet config.
-	ss.ObjectMeta = metav1.ObjectMeta{
-		Name:            pg.Name,
-		Namespace:       namespace,
-		Labels:          pgLabels(pg.Name, nil),
-		OwnerReferences: pgOwnerReference(pg),
-	}
-	ss.Spec.Replicas = ptr.To(pgReplicas(pg))
-	ss.Spec.Selector = &metav1.LabelSelector{
-		MatchLabels: pgLabels(pg.Name, nil),
-	}
-
-	// Template config.
-	tmpl := &ss.Spec.Template
-	tmpl.ObjectMeta = metav1.ObjectMeta{
-		Name:                       pg.Name,
-		Namespace:                  namespace,
-		Labels:                     pgLabels(pg.Name, nil),
-		DeletionGracePeriodSeconds: ptr.To[int64](10),
-		Annotations: map[string]string{
-			podAnnotationLastSetConfigFileHash: cfgHash,
-		},
-	}
-	tmpl.Spec.ServiceAccountName = pg.Name
-	tmpl.Spec.InitContainers[0].Image = image
-	tmpl.Spec.Volumes = func() []corev1.Volume {
-		var volumes []corev1.Volume
-		for i := range pgReplicas(pg) {
-			volumes = append(volumes, corev1.Volume{
-				Name: fmt.Sprintf("tailscaledconfig-%d", i),
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i),
-					},
-				},
-			})
-		}
-
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			volumes = append(volumes, corev1.Volume{
-				Name: pgEgressCMName(pg.Name),
-				VolumeSource: corev1.VolumeSource{
-					ConfigMap: &corev1.ConfigMapVolumeSource{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: pgEgressCMName(pg.Name),
-						},
-					},
-				},
-			})
-		}
-
-		return volumes
-	}()
-
-	// Main container config.
-	c := &ss.Spec.Template.Spec.Containers[0]
-	c.Image = image
-	c.VolumeMounts = func() []corev1.VolumeMount {
-		var mounts []corev1.VolumeMount
-		for i := range pgReplicas(pg) {
-			mounts = append(mounts, corev1.VolumeMount{
-				Name:      fmt.Sprintf("tailscaledconfig-%d", i),
-				ReadOnly:  true,
-				MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
-			})
-		}
-
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			mounts = append(mounts, corev1.VolumeMount{
-				Name:      pgEgressCMName(pg.Name),
-				MountPath: "/etc/proxies",
-				ReadOnly:  true,
-			})
-		}
-
-		return mounts
-	}()
-	c.Env = func() []corev1.EnvVar {
-		envs := []corev1.EnvVar{
-			{
-				// TODO(irbekrm): verify that .status.podIPs are always set, else read in .status.podIP as well.
-				Name: "POD_IPS", // this will be a comma separate list i.e 10.136.0.6,2600:1900:4011:161:0:e:0:6
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						FieldPath: "status.podIPs",
-					},
-				},
-			},
-			{
-				Name: "POD_NAME",
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						// Secret is named after the pod.
-						FieldPath: "metadata.name",
-					},
-				},
-			},
-			{
-				Name:  "TS_KUBE_SECRET",
-				Value: "$(POD_NAME)",
-			},
-			{
-				Name:  "TS_STATE",
-				Value: "kube:$(POD_NAME)",
-			},
-			{
-				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
-				Value: "/etc/tsconfig/$(POD_NAME)",
-			},
-			{
-				Name:  "TS_USERSPACE",
-				Value: "false",
-			},
-		}
-
-		if tsFirewallMode != "" {
-			envs = append(envs, corev1.EnvVar{
-				Name:  "TS_DEBUG_FIREWALL_MODE",
-				Value: tsFirewallMode,
-			})
-		}
-
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			envs = append(envs, corev1.EnvVar{
-				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
-				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
-			})
-		}
-
-		return envs
-	}()
-
-	return ss, nil
-}
-
-func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount {
-	return &corev1.ServiceAccount{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pg.Name,
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-	}
-}
-
-func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
-	return &rbacv1.Role{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pg.Name,
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-		Rules: []rbacv1.PolicyRule{
-			{
-				APIGroups: []string{""},
-				Resources: []string{"secrets"},
-				Verbs: []string{
-					"get",
-					"patch",
-					"update",
-				},
-				ResourceNames: func() (secrets []string) {
-					for i := range pgReplicas(pg) {
-						secrets = append(secrets,
-							fmt.Sprintf("%s-%d-config", pg.Name, i), // Config with auth key.
-							fmt.Sprintf("%s-%d", pg.Name, i),        // State.
-						)
-					}
-					return secrets
-				}(),
-			},
-		},
-	}
-}
-
-func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
-	return &rbacv1.RoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pg.Name,
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      pg.Name,
-				Namespace: namespace,
-			},
-		},
-		RoleRef: rbacv1.RoleRef{
-			Kind: "Role",
-			Name: pg.Name,
-		},
-	}
-}
-
-func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) {
-	for i := range pgReplicas(pg) {
-		secrets = append(secrets, &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:            fmt.Sprintf("%s-%d", pg.Name, i),
-				Namespace:       namespace,
-				Labels:          pgSecretLabels(pg.Name, "state"),
-				OwnerReferences: pgOwnerReference(pg),
-			},
-		})
-	}
-
-	return secrets
-}
-
-func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
-	return &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pgEgressCMName(pg.Name),
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-	}
-}
-
-func pgSecretLabels(pgName, typ string) map[string]string {
-	return pgLabels(pgName, map[string]string{
-		labelSecretType: typ, // "config" or "state".
-	})
-}
-
-func pgLabels(pgName string, customLabels map[string]string) map[string]string {
-	l := make(map[string]string, len(customLabels)+3)
-	for k, v := range customLabels {
-		l[k] = v
-	}
-
-	l[LabelManaged] = "true"
-	l[LabelParentType] = "proxygroup"
-	l[LabelParentName] = pgName
-
-	return l
-}
-
-func pgOwnerReference(owner *tsapi.ProxyGroup) []metav1.OwnerReference {
-	return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("ProxyGroup"))}
-}
-
-func pgReplicas(pg *tsapi.ProxyGroup) int32 {
-	if pg.Spec.Replicas != nil {
-		return *pg.Spec.Replicas
-	}
-
-	return 2
-}
-
-func pgEgressCMName(pg string) string {
-	return fmt.Sprintf("%s-egress-config", pg)
-}

cmd/k8s-operator/proxygroup_test.go

@@ -1,267 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"go.uber.org/zap"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	"tailscale.com/client/tailscale"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/tstest"
-	"tailscale.com/types/ptr"
-)
-
-const testProxyImage = "tailscale/tailscale:test"
-
-var defaultProxyClassAnnotations = map[string]string{
-	"some-annotation": "from-the-proxy-class",
-}
-
-func TestProxyGroup(t *testing.T) {
-	pc := &tsapi.ProxyClass{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "default-pc",
-		},
-		Spec: tsapi.ProxyClassSpec{
-			StatefulSet: &tsapi.StatefulSet{
-				Annotations: defaultProxyClassAnnotations,
-			},
-		},
-	}
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:       "test",
-			Finalizers: []string{"tailscale.com/finalizer"},
-		},
-	}
-
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pg, pc).
-		WithStatusSubresource(pg, pc).
-		Build()
-	tsClient := &fakeTSClient{}
-	zl, _ := zap.NewDevelopment()
-	fr := record.NewFakeRecorder(1)
-	cl := tstest.NewClock(tstest.ClockOpts{})
-	reconciler := &ProxyGroupReconciler{
-		tsNamespace:       tsNamespace,
-		proxyImage:        testProxyImage,
-		defaultTags:       []string{"tag:test-tag"},
-		tsFirewallMode:    "auto",
-		defaultProxyClass: "default-pc",
-
-		Client:   fc,
-		tsClient: tsClient,
-		recorder: fr,
-		l:        zl.Sugar(),
-		clock:    cl,
-	}
-
-	t.Run("proxyclass_not_ready", func(t *testing.T) {
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-	})
-
-	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
-		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []metav1.Condition{{
-				Type:               string(tsapi.ProxyClassReady),
-				Status:             metav1.ConditionTrue,
-				Reason:             reasonProxyClassValid,
-				Message:            reasonProxyClassValid,
-				LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
-			}},
-		}
-		if err := fc.Status().Update(context.Background(), pc); err != nil {
-			t.Fatal(err)
-		}
-
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-		if expected := 1; reconciler.proxyGroups.Len() != expected {
-			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
-		}
-		expectProxyGroupResources(t, fc, pg, true)
-		keyReq := tailscale.KeyCapabilities{
-			Devices: tailscale.KeyDeviceCapabilities{
-				Create: tailscale.KeyDeviceCreateCapabilities{
-					Reusable:      false,
-					Ephemeral:     false,
-					Preauthorized: true,
-					Tags:          []string{"tag:test-tag"},
-				},
-			},
-		}
-		if diff := cmp.Diff(tsClient.KeyRequests(), []tailscale.KeyCapabilities{keyReq, keyReq}); diff != "" {
-			t.Fatalf("unexpected secrets (-got +want):\n%s", diff)
-		}
-	})
-
-	t.Run("simulate_successful_device_auth", func(t *testing.T) {
-		addNodeIDToStateSecrets(t, fc, pg)
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		pg.Status.Devices = []tsapi.TailnetDevice{
-			{
-				Hostname:   "hostname-nodeid-0",
-				TailnetIPs: []string{"1.2.3.4", "::1"},
-			},
-			{
-				Hostname:   "hostname-nodeid-1",
-				TailnetIPs: []string{"1.2.3.4", "::1"},
-			},
-		}
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
-	})
-
-	t.Run("scale_up_to_3", func(t *testing.T) {
-		pg.Spec.Replicas = ptr.To[int32](3)
-		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
-			p.Spec = pg.Spec
-		})
-		expectReconciled(t, reconciler, "", pg.Name)
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-
-		addNodeIDToStateSecrets(t, fc, pg)
-		expectReconciled(t, reconciler, "", pg.Name)
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
-		pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{
-			Hostname:   "hostname-nodeid-2",
-			TailnetIPs: []string{"1.2.3.4", "::1"},
-		})
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
-	})
-
-	t.Run("scale_down_to_1", func(t *testing.T) {
-		pg.Spec.Replicas = ptr.To[int32](1)
-		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
-			p.Spec = pg.Spec
-		})
-		expectReconciled(t, reconciler, "", pg.Name)
-		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
-		expectEqual(t, fc, pg, nil)
-
-		expectProxyGroupResources(t, fc, pg, true)
-	})
-
-	t.Run("delete_and_cleanup", func(t *testing.T) {
-		if err := fc.Delete(context.Background(), pg); err != nil {
-			t.Fatal(err)
-		}
-
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		expectMissing[tsapi.Recorder](t, fc, "", pg.Name)
-		if expected := 0; reconciler.proxyGroups.Len() != expected {
-			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
-		}
-		// 2 nodes should get deleted as part of the scale down, and then finally
-		// the first node gets deleted with the ProxyGroup cleanup.
-		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" {
-			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
-		}
-		// The fake client does not clean up objects whose owner has been
-		// deleted, so we can't test for the owned resources getting deleted.
-	})
-}
-
-func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool) {
-	t.Helper()
-
-	role := pgRole(pg, tsNamespace)
-	roleBinding := pgRoleBinding(pg, tsNamespace)
-	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	statefulSet.Annotations = defaultProxyClassAnnotations
-
-	if shouldExist {
-		expectEqual(t, fc, role, nil)
-		expectEqual(t, fc, roleBinding, nil)
-		expectEqual(t, fc, serviceAccount, nil)
-		expectEqual(t, fc, statefulSet, func(ss *appsv1.StatefulSet) {
-			ss.Spec.Template.Annotations[podAnnotationLastSetConfigFileHash] = ""
-		})
-	} else {
-		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
-		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
-		expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
-		expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
-	}
-
-	var expectedSecrets []string
-	for i := range pgReplicas(pg) {
-		expectedSecrets = append(expectedSecrets,
-			fmt.Sprintf("%s-%d", pg.Name, i),
-			fmt.Sprintf("%s-%d-config", pg.Name, i),
-		)
-	}
-	expectSecrets(t, fc, expectedSecrets)
-}
-
-func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) {
-	t.Helper()
-
-	secrets := &corev1.SecretList{}
-	if err := fc.List(context.Background(), secrets); err != nil {
-		t.Fatal(err)
-	}
-
-	var actual []string
-	for _, secret := range secrets.Items {
-		actual = append(actual, secret.Name)
-	}
-
-	if diff := cmp.Diff(actual, expected); diff != "" {
-		t.Fatalf("unexpected secrets (-got +want):\n%s", diff)
-	}
-}
-
-func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) {
-	const key = "profile-abc"
-	for i := range pgReplicas(pg) {
-		bytes, err := json.Marshal(map[string]any{
-			"Config": map[string]any{
-				"NodeID": fmt.Sprintf("nodeid-%d", i),
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) {
-			s.Data = map[string][]byte{
-				currentProfileKey: []byte(key),
-				key:               bytes,
-			}
-		})
-	}
-}

cmd/k8s-operator/recorder.go

@@ -0,0 +1,88 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"tailscale.com/tstime"
+)
+
+// recorder knows how to send the provided bytes to the configured tsrecorder
+// instance in asciinema format.
+type recorder struct {
+	start time.Time
+	clock tstime.Clock
+
+	// failOpen specifies whether the session should be allowed to
+	// continue if writing to the recording fails.
+	failOpen bool
+
+	// backOff is set to true if  we've failed open and should stop
+	// attempting to write to tsrecorder.
+	backOff bool
+
+	mu   sync.Mutex     // guards writes to conn
+	conn io.WriteCloser // connection to a tsrecorder instance
+}
+
+// Write appends timestamp to the provided bytes and sends them to the
+// configured tsrecorder.
+func (rec *recorder) Write(p []byte) (err error) {
+	if len(p) == 0 {
+		return nil
+	}
+	if rec.backOff {
+		return nil
+	}
+	j, err := json.Marshal([]any{
+		rec.clock.Now().Sub(rec.start).Seconds(),
+		"o",
+		string(p),
+	})
+	if err != nil {
+		return fmt.Errorf("error marhalling payload: %w", err)
+	}
+	j = append(j, '\n')
+	if err := rec.writeCastLine(j); err != nil {
+		if !rec.failOpen {
+			return fmt.Errorf("error writing payload to recorder: %w", err)
+		}
+		rec.backOff = true
+	}
+	return nil
+}
+
+func (rec *recorder) Close() error {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if rec.conn == nil {
+		return nil
+	}
+	err := rec.conn.Close()
+	rec.conn = nil
+	return err
+}
+
+// writeCastLine sends bytes to the tsrecorder. The bytes should be in
+// asciinema format.
+func (rec *recorder) writeCastLine(j []byte) error {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if rec.conn == nil {
+		return errors.New("recorder closed")
+	}
+	_, err := rec.conn.Write(j)
+	if err != nil {
+		return fmt.Errorf("recorder write error: %w", err)
+	}
+	return nil
+}

cmd/k8s-operator/spdy-frame.go

@@ -3,7 +3,7 @@
 
 //go:build !plan9
 
-package spdy
+package main
 
 import (
 	"bytes"

cmd/k8s-operator/spdy-frame_test.go

@@ -3,21 +3,17 @@
 
 //go:build !plan9
 
-package spdy
+package main
 
 import (
 	"bytes"
 	"compress/zlib"
 	"encoding/binary"
-	"fmt"
 	"io"
 	"net/http"
 	"reflect"
 	"strings"
 	"testing"
-	"time"
-
-	"math/rand"
 
 	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
@@ -204,29 +200,6 @@ func Test_spdyFrame_parseHeaders(t *testing.T) {
 	}
 }
 
-// Test_spdyFrame_ParseRand calls spdyFrame.Parse with randomly generated bytes
-// to test that it doesn't panic.
-func Test_spdyFrame_ParseRand(t *testing.T) {
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	for i := range 100 {
-		n := r.Intn(4096)
-		b := make([]byte, n)
-		_, err := r.Read(b)
-		if err != nil {
-			t.Fatalf("error generating random byte slice: %v", err)
-		}
-		sf := &spdyFrame{}
-		f := func() {
-			sf.Parse(b, zl.Sugar())
-		}
-		testPanic(t, f, fmt.Sprintf("[%d] Parse panicked running with byte slice of length %d: %v", i, n, r))
-	}
-}
-
 // payload takes a control frame type and a map with 0 or more header keys and
 // values and returns a SPDY control frame payload with the header as SPDY zlib
 // compressed header name/value block. The payload is padded with arbitrary
@@ -318,13 +291,3 @@ func header(hs map[string]string) http.Header {
 	}
 	return h
 }
-
-func testPanic(t *testing.T, f func(), msg string) {
-	t.Helper()
-	defer func() {
-		if r := recover(); r != nil {
-			t.Fatal(msg, r)
-		}
-	}()
-	f()
-}

cmd/k8s-operator/spdy-hijacker.go

@@ -3,9 +3,7 @@
 
 //go:build !plan9
 
-// Package sessionrecording contains functionality for recording Kubernetes API
-// server proxy 'kubectl exec' sessions.
-package sessionrecording
+package main
 
 import (
 	"bufio"
@@ -15,75 +13,23 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/http/httptrace"
 	"net/netip"
 	"strings"
 
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"tailscale.com/client/tailscale/apitype"
-	"tailscale.com/k8s-operator/sessionrecording/spdy"
-	"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
-	"tailscale.com/k8s-operator/sessionrecording/ws"
-	"tailscale.com/sessionrecording"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
-	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/multierr"
 )
 
-const (
-	SPDYProtocol Protocol = "SPDY"
-	WSProtocol   Protocol = "WebSocket"
-)
-
-// Protocol is the streaming protocol of the hijacked session. Supported
-// protocols are SPDY and WebSocket.
-type Protocol string
-
-var (
-	// CounterSessionRecordingsAttempted counts the number of session recording attempts.
-	CounterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_attempted")
-
-	// counterSessionRecordingsUploaded counts the number of successfully uploaded session recordings.
-	counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded")
-)
-
-func New(opts HijackerOpts) *Hijacker {
-	return &Hijacker{
-		ts:                opts.TS,
-		req:               opts.Req,
-		who:               opts.Who,
-		ResponseWriter:    opts.W,
-		pod:               opts.Pod,
-		ns:                opts.Namespace,
-		addrs:             opts.Addrs,
-		failOpen:          opts.FailOpen,
-		proto:             opts.Proto,
-		log:               opts.Log,
-		connectToRecorder: sessionrecording.ConnectToRecorder,
-	}
-}
-
-type HijackerOpts struct {
-	TS        *tsnet.Server
-	Req       *http.Request
-	W         http.ResponseWriter
-	Who       *apitype.WhoIsResponse
-	Addrs     []netip.AddrPort
-	Log       *zap.SugaredLogger
-	Pod       string
-	Namespace string
-	FailOpen  bool
-	Proto     Protocol
-}
-
-// Hijacker implements [net/http.Hijacker] interface.
+// spdyHijacker implements [net/http.Hijacker] interface.
 // It must be configured with an http request for a 'kubectl exec' session that
 // needs to be recorded. It knows how to hijack the connection and configure for
 // the session contents to be sent to a tsrecorder instance.
-type Hijacker struct {
+type spdyHijacker struct {
 	http.ResponseWriter
 	ts                *tsnet.Server
 	req               *http.Request
@@ -94,7 +40,6 @@ type Hijacker struct {
 	addrs             []netip.AddrPort // tsrecorder addresses
 	failOpen          bool             // whether to fail open if recording fails
 	connectToRecorder RecorderDialFn
-	proto             Protocol // streaming protocol
 }
 
 // RecorderDialFn dials the specified netip.AddrPorts that should be tsrecorder
@@ -106,7 +51,7 @@ type RecorderDialFn func(context.Context, []netip.AddrPort, func(context.Context
 
 // Hijack hijacks a 'kubectl exec' session and configures for the session
 // contents to be sent to a recorder.
-func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+func (h *spdyHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 	h.log.Infof("recorder addrs: %v, failOpen: %v", h.addrs, h.failOpen)
 	reqConn, brw, err := h.ResponseWriter.(http.Hijacker).Hijack()
 	if err != nil {
@@ -124,29 +69,15 @@ func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 // spdyHijacker.addrs. Returns conn from provided opts, wrapped in recording
 // logic. If connecting to the recorder fails or an error is received during the
 // session and spdyHijacker.failOpen is false, connection will be closed.
-func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, error) {
+func (h *spdyHijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, error) {
 	const (
 		// https://docs.asciinema.org/manual/asciicast/v2/
-		asciicastv2  = 2
-		ttyKey       = "tty"
-		commandKey   = "command"
-		containerKey = "container"
-	)
-	var (
-		wc      io.WriteCloser
-		err     error
-		errChan <-chan error
+		asciicastv2 = 2
 	)
+	var wc io.WriteCloser
 	h.log.Infof("kubectl exec session will be recorded, recorders: %v, fail open policy: %t", h.addrs, h.failOpen)
-	qp := h.req.URL.Query()
-	container := strings.Join(qp[containerKey], "")
-	var recorderAddr net.Addr
-	trace := &httptrace.ClientTrace{
-		GotConn: func(info httptrace.GotConnInfo) {
-			recorderAddr = info.Conn.RemoteAddr()
-		},
-	}
-	wc, _, errChan, err = h.connectToRecorder(httptrace.WithClientTrace(ctx, trace), h.addrs, h.ts.Dial)
+	// TODO (irbekrm): send client a message that session will be recorded.
+	rw, _, errChan, err := h.connectToRecorder(ctx, h.addrs, h.ts.Dial)
 	if err != nil {
 		msg := fmt.Sprintf("error connecting to session recorders: %v", err)
 		if h.failOpen {
@@ -159,24 +90,34 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 			return nil, multierr.New(errors.New(msg), err)
 		}
 		return nil, errors.New(msg)
-	} else {
-		h.log.Infof("exec session to container %q in Pod %q namespace %q will be recorded, the recording will be sent to a tsrecorder instance at %q", container, h.pod, h.ns, recorderAddr)
 	}
 
+	// TODO (irbekrm): log which recorder
+	h.log.Info("successfully connected to a session recorder")
+	wc = rw
 	cl := tstime.DefaultClock{}
-	rec := tsrecorder.New(wc, cl, cl.Now(), h.failOpen, h.log)
-	tty := strings.Join(qp[ttyKey], "")
-	hasTerm := (tty == "true") // session has terminal attached
-	ch := sessionrecording.CastHeader{
+	lc := &spdyRemoteConnRecorder{
+		log:  h.log,
+		Conn: conn,
+		rec: &recorder{
+			start:    cl.Now(),
+			clock:    cl,
+			failOpen: h.failOpen,
+			conn:     wc,
+		},
+	}
+
+	qp := h.req.URL.Query()
+	ch := CastHeader{
 		Version:   asciicastv2,
-		Timestamp: cl.Now().Unix(),
-		Command:   strings.Join(qp[commandKey], " "),
+		Timestamp: lc.rec.start.Unix(),
+		Command:   strings.Join(qp["command"], " "),
 		SrcNode:   strings.TrimSuffix(h.who.Node.Name, "."),
 		SrcNodeID: h.who.Node.StableID,
-		Kubernetes: &sessionrecording.Kubernetes{
+		Kubernetes: &Kubernetes{
 			PodName:   h.pod,
 			Namespace: h.ns,
-			Container: container,
+			Container: strings.Join(qp["container"], " "),
 		},
 	}
 	if !h.who.Node.IsTagged() {
@@ -185,17 +126,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 	} else {
 		ch.SrcNodeTags = h.who.Node.Tags
 	}
-
-	var lc net.Conn
-	switch h.proto {
-	case SPDYProtocol:
-		lc = spdy.New(conn, rec, ch, hasTerm, h.log)
-	case WSProtocol:
-		lc = ws.New(conn, rec, ch, hasTerm, h.log)
-	default:
-		return nil, fmt.Errorf("unknown protocol: %s", h.proto)
-	}
-
+	lc.ch = ch
 	go func() {
 		var err error
 		select {
@@ -216,6 +147,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 		}
 		msg += "; failure mode set to 'fail closed'; closing connection"
 		h.log.Error(msg)
+		lc.failed = true
 		// TODO (irbekrm): write a message to the client
 		if err := lc.Close(); err != nil {
 			h.log.Infof("error closing recorder connections: %v", err)
@@ -225,6 +157,52 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn,
 	return lc, nil
 }
 
+// CastHeader is the asciicast header to be sent to the recorder at the start of
+// the recording of a session.
+// https://docs.asciinema.org/manual/asciicast/v2/#header
+type CastHeader struct {
+	// Version is the asciinema file format version.
+	Version int `json:"version"`
+
+	// Width is the terminal width in characters.
+	Width int `json:"width"`
+
+	// Height is the terminal height in characters.
+	Height int `json:"height"`
+
+	// Timestamp is the unix timestamp of when the recording started.
+	Timestamp int64 `json:"timestamp"`
+
+	// Tailscale-specific fields: SrcNode is the full MagicDNS name of the
+	// tailnet node originating the connection, without the trailing dot.
+	SrcNode string `json:"srcNode"`
+
+	// SrcNodeID is the node ID of the tailnet node originating the connection.
+	SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"`
+
+	// SrcNodeTags is the list of tags on the node originating the connection (if any).
+	SrcNodeTags []string `json:"srcNodeTags,omitempty"`
+
+	// SrcNodeUserID is the user ID of the node originating the connection (if not tagged).
+	SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged
+
+	// SrcNodeUser is the LoginName of the node originating the connection (if not tagged).
+	SrcNodeUser string `json:"srcNodeUser,omitempty"`
+
+	Command string
+
+	// Kubernetes-specific fields:
+	Kubernetes *Kubernetes `json:"kubernetes,omitempty"`
+}
+
+// Kubernetes contains 'kubectl exec' session specific information for
+// tsrecorder.
+type Kubernetes struct {
+	PodName   string
+	Namespace string
+	Container string
+}
+
 func closeConnWithWarning(conn net.Conn, msg string) error {
 	b := io.NopCloser(bytes.NewBuffer([]byte(msg)))
 	resp := http.Response{Status: http.StatusText(http.StatusForbidden), StatusCode: http.StatusForbidden, Body: b}

cmd/k8s-operator/spdy-hijacker_test.go

@@ -3,7 +3,7 @@
 
 //go:build !plan9
 
-package sessionrecording
+package main
 
 import (
 	"context"
@@ -19,13 +19,12 @@ import (
 
 	"go.uber.org/zap"
 	"tailscale.com/client/tailscale/apitype"
-	"tailscale.com/k8s-operator/sessionrecording/fakes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstest"
 )
 
-func Test_Hijacker(t *testing.T) {
+func Test_SPDYHijacker(t *testing.T) {
 	zl, err := zap.NewDevelopment()
 	if err != nil {
 		t.Fatal(err)
@@ -37,51 +36,38 @@ func Test_Hijacker(t *testing.T) {
 		failRecorderConnPostConnect bool // send error down the error channel
 		wantsConnClosed             bool
 		wantsSetupErr               bool
-		proto                       Protocol
 	}{
 		{
-			name:  "setup_succeeds_conn_stays_open",
-			proto: SPDYProtocol,
+			name: "setup succeeds, conn stays open",
 		},
 		{
-			name:  "setup_succeeds_conn_stays_open_ws",
-			proto: WSProtocol,
-		},
-		{
-			name:                "setup_fails_policy_is_to_fail_open_conn_stays_open",
+			name:                "setup fails, policy is to fail open, conn stays open",
 			failOpen:            true,
 			failRecorderConnect: true,
-			proto:               SPDYProtocol,
 		},
 		{
-			name:                "setup_fails_policy_is_to_fail_closed_conn_is_closed",
+			name:                "setup fails, policy is to fail closed, conn is closed",
 			failRecorderConnect: true,
 			wantsSetupErr:       true,
 			wantsConnClosed:     true,
-			proto:               SPDYProtocol,
 		},
 		{
-			name:                        "connection_fails_post-initial_connect_policy_is_to_fail_open_conn_stays_open",
+			name:                        "connection fails post-initial connect, policy is to fail open, conn stays open",
 			failRecorderConnPostConnect: true,
 			failOpen:                    true,
-			proto:                       SPDYProtocol,
 		},
 		{
-			name:                        "connection_fails_post-initial_connect,_policy_is_to_fail_closed_conn_is_closed",
+			name:                        "connection fails post-initial connect, policy is to fail closed, conn is closed",
 			failRecorderConnPostConnect: true,
 			wantsConnClosed:             true,
-			proto:                       SPDYProtocol,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tc := &fakes.TestConn{}
+			tc := &testConn{}
 			ch := make(chan error)
-			h := &Hijacker{
-				connectToRecorder: func(context.Context,
-					[]netip.AddrPort,
-					func(context.Context, string, string) (net.Conn, error),
-				) (wc io.WriteCloser, rec []*tailcfg.SSHRecordingAttempt, _ <-chan error, err error) {
+			h := &spdyHijacker{
+				connectToRecorder: func(context.Context, []netip.AddrPort, func(context.Context, string, string) (net.Conn, error)) (wc io.WriteCloser, rec []*tailcfg.SSHRecordingAttempt, _ <-chan error, err error) {
 					if tt.failRecorderConnect {
 						err = errors.New("test")
 					}
@@ -92,7 +78,6 @@ func Test_Hijacker(t *testing.T) {
 				log:      zl.Sugar(),
 				ts:       &tsnet.Server{},
 				req:      &http.Request{URL: &url.URL{}},
-				proto:    tt.proto,
 			}
 			ctx := context.Background()
 			_, err := h.setUpRecording(ctx, tc)
@@ -113,8 +98,8 @@ func Test_Hijacker(t *testing.T) {
 			// (test that connection remains open over some period
 			// of time).
 			if err := tstest.WaitFor(timeout, func() (err error) {
-				if tt.wantsConnClosed != tc.IsClosed() {
-					return fmt.Errorf("got connection state: %t, wants connection state: %t", tc.IsClosed(), tt.wantsConnClosed)
+				if tt.wantsConnClosed != tc.isClosed() {
+					return fmt.Errorf("got connection state: %t, wants connection state: %t", tc.isClosed(), tt.wantsConnClosed)
 				}
 				return nil
 			}); err != nil {

cmd/k8s-operator/spdy-remote-conn-recorder.go

@@ -0,0 +1,194 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"sync/atomic"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// spdyRemoteConnRecorder is a wrapper around net.Conn. It reads the bytestream
+// for a 'kubectl exec' session, sends session recording data to the configured
+// recorder and forwards the raw bytes to the original destination.
+type spdyRemoteConnRecorder struct {
+	net.Conn
+	// rec knows how to send data written to it to a tsrecorder instance.
+	rec *recorder
+	ch  CastHeader
+
+	stdoutStreamID atomic.Uint32
+	stderrStreamID atomic.Uint32
+	resizeStreamID atomic.Uint32
+
+	wmu    sync.Mutex // sequences writes
+	closed bool
+	failed bool
+
+	rmu                 sync.Mutex // sequences reads
+	writeCastHeaderOnce sync.Once
+
+	zlibReqReader zlibReader
+	// writeBuf is used to store data written to the connection that has not
+	// yet been parsed as SPDY frames.
+	writeBuf bytes.Buffer
+	// readBuf is used to store data read from the connection that has not
+	// yet been parsed as SPDY frames.
+	readBuf bytes.Buffer
+	log     *zap.SugaredLogger
+}
+
+// Read reads bytes from the original connection and parses them as SPDY frames.
+// If the frame is a data frame for resize stream, sends resize message to the
+// recorder. If the frame is a SYN_STREAM control frame that starts stdout,
+// stderr or resize stream, store the stream ID.
+func (c *spdyRemoteConnRecorder) Read(b []byte) (int, error) {
+	c.rmu.Lock()
+	defer c.rmu.Unlock()
+	n, err := c.Conn.Read(b)
+	if err != nil {
+		return n, fmt.Errorf("error reading from connection: %w", err)
+	}
+	c.readBuf.Write(b[:n])
+
+	var sf spdyFrame
+	ok, err := sf.Parse(c.readBuf.Bytes(), c.log)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing data read from connection: %w", err)
+	}
+	if !ok {
+		// The parsed data in the buffer will be processed together with
+		// the new data on the next call to Read.
+		return n, nil
+	}
+	c.readBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame
+
+	if !sf.Ctrl { // data frame
+		switch sf.StreamID {
+		case c.resizeStreamID.Load():
+			var err error
+			var msg spdyResizeMsg
+			if err = json.Unmarshal(sf.Payload, &msg); err != nil {
+				return 0, fmt.Errorf("error umarshalling resize msg: %w", err)
+			}
+			c.ch.Width = msg.Width
+			c.ch.Height = msg.Height
+		}
+		return n, nil
+	}
+	// We always want to parse the headers, even if we don't care about the
+	// frame, as we need to advance the zlib reader otherwise we will get
+	// garbage.
+	header, err := sf.parseHeaders(&c.zlibReqReader, c.log)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing frame headers: %w", err)
+	}
+	if sf.Type == SYN_STREAM {
+		c.storeStreamID(sf, header)
+	}
+	return n, nil
+}
+
+// Write forwards the raw data of the latest parsed SPDY frame to the original
+// destination. If the frame is an SPDY data frame, it also sends the payload to
+// the connected session recorder.
+func (c *spdyRemoteConnRecorder) Write(b []byte) (int, error) {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	c.writeBuf.Write(b)
+
+	var sf spdyFrame
+	ok, err := sf.Parse(c.writeBuf.Bytes(), c.log)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing data: %w", err)
+	}
+	if !ok {
+		// The parsed data in the buffer will be processed together with
+		// the new data on the next call to Write.
+		return len(b), nil
+	}
+	c.writeBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame
+
+	// If this is a stdout or stderr data frame, send its payload to the
+	// session recorder.
+	if !sf.Ctrl {
+		switch sf.StreamID {
+		case c.stdoutStreamID.Load(), c.stderrStreamID.Load():
+			var err error
+			c.writeCastHeaderOnce.Do(func() {
+				var j []byte
+				j, err = json.Marshal(c.ch)
+				if err != nil {
+					return
+				}
+				j = append(j, '\n')
+				err = c.rec.writeCastLine(j)
+				if err != nil {
+					c.log.Errorf("received error from recorder: %v", err)
+				}
+			})
+			if err != nil {
+				return 0, fmt.Errorf("error writing CastHeader: %w", err)
+			}
+			if err := c.rec.Write(sf.Payload); err != nil {
+				return 0, fmt.Errorf("error sending payload to session recorder: %w", err)
+			}
+		}
+	}
+	// Forward the whole frame to the original destination.
+	_, err = c.Conn.Write(sf.Raw) // send to net.Conn
+	return len(b), err
+}
+
+func (c *spdyRemoteConnRecorder) Close() error {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	if c.closed {
+		return nil
+	}
+	if !c.failed && c.writeBuf.Len() > 0 {
+		c.Conn.Write(c.writeBuf.Bytes())
+	}
+	c.writeBuf.Reset()
+	c.closed = true
+	err := c.Conn.Close()
+	c.rec.Close()
+	return err
+}
+
+// parseSynStream parses SYN_STREAM SPDY control frame and updates
+// spdyRemoteConnRecorder to store the newly created stream's ID if it is one of
+// the stream types we care about. Storing stream_id:stream_type mapping allows
+// us to parse received data frames (that have stream IDs) differently depening
+// on which stream they belong to (i.e send data frame payload for stdout stream
+// to session recorder).
+func (c *spdyRemoteConnRecorder) storeStreamID(sf spdyFrame, header http.Header) {
+	const (
+		streamTypeHeaderKey = "Streamtype"
+	)
+	id := binary.BigEndian.Uint32(sf.Payload[0:4])
+	switch header.Get(streamTypeHeaderKey) {
+	case corev1.StreamTypeStdout:
+		c.stdoutStreamID.Store(id)
+	case corev1.StreamTypeStderr:
+		c.stderrStreamID.Store(id)
+	case corev1.StreamTypeResize:
+		c.resizeStreamID.Store(id)
+	}
+}
+
+type spdyResizeMsg struct {
+	Width  int `json:"width"`
+	Height int `json:"height"`
+}

cmd/k8s-operator/spdy-remote-conn-recorder_test.go

@@ -3,19 +3,19 @@
 
 //go:build !plan9
 
-package spdy
+package main
 
 import (
+	"bytes"
 	"encoding/json"
-	"fmt"
+	"net"
 	"reflect"
+	"sync"
 	"testing"
 
 	"go.uber.org/zap"
-	"tailscale.com/k8s-operator/sessionrecording/fakes"
-	"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
-	"tailscale.com/sessionrecording"
 	"tailscale.com/tstest"
+	"tailscale.com/tstime"
 )
 
 // Test_Writes tests that 1 or more Write calls to spdyRemoteConnRecorder
@@ -29,15 +29,13 @@ func Test_Writes(t *testing.T) {
 	}
 	cl := tstest.NewClock(tstest.ClockOpts{})
 	tests := []struct {
-		name              string
-		inputs            [][]byte
-		wantForwarded     []byte
-		wantRecorded      []byte
-		firstWrite        bool
-		width             int
-		height            int
-		sendInitialResize bool
-		hasTerm           bool
+		name          string
+		inputs        [][]byte
+		wantForwarded []byte
+		wantRecorded  []byte
+		firstWrite    bool
+		width         int
+		height        int
 	}{
 		{
 			name:          "single_write_control_frame_with_payload",
@@ -58,13 +56,13 @@ func Test_Writes(t *testing.T) {
 			name:          "single_write_stdout_data_frame_with_payload",
 			inputs:        [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
 			wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
-			wantRecorded:  fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
+			wantRecorded:  castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
 		},
 		{
 			name:          "single_write_stderr_data_frame_with_payload",
 			inputs:        [][]byte{{0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
 			wantForwarded: []byte{0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
-			wantRecorded:  fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
+			wantRecorded:  castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
 		},
 		{
 			name:          "single_data_frame_unknow_stream_with_payload",
@@ -75,24 +73,13 @@ func Test_Writes(t *testing.T) {
 			name:          "control_frame_and_data_frame_split_across_two_writes",
 			inputs:        [][]byte{{0x80, 0x3, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, {0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
 			wantForwarded: []byte{0x80, 0x3, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
-			wantRecorded:  fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
+			wantRecorded:  castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
 		},
 		{
-			name:              "single_first_write_stdout_data_frame_with_payload_sess_has_terminal",
-			inputs:            [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
-			wantForwarded:     []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
-			wantRecorded:      append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
-			width:             10,
-			height:            20,
-			hasTerm:           true,
-			firstWrite:        true,
-			sendInitialResize: true,
-		},
-		{
-			name:          "single_first_write_stdout_data_frame_with_payload_sess_does_not_have_terminal",
+			name:          "single_first_write_stdout_data_frame_with_payload",
 			inputs:        [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
 			wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
-			wantRecorded:  append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
+			wantRecorded:  append(asciinemaResizeMsg(t, 10, 20), castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
 			width:         10,
 			height:        20,
 			firstWrite:    true,
@@ -100,46 +87,44 @@ func Test_Writes(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tc := &fakes.TestConn{}
-			sr := &fakes.TestSessionRecorder{}
-			rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar())
+			tc := &testConn{}
+			sr := &testSessionRecorder{}
+			rec := &recorder{
+				conn:  sr,
+				clock: cl,
+				start: cl.Now(),
+			}
 
-			c := &conn{
+			c := &spdyRemoteConnRecorder{
 				Conn: tc,
 				log:  zl.Sugar(),
 				rec:  rec,
-				ch: sessionrecording.CastHeader{
+				ch: CastHeader{
 					Width:  tt.width,
 					Height: tt.height,
 				},
-				initialTermSizeSet: make(chan struct{}),
-				hasTerm:            tt.hasTerm,
 			}
 			if !tt.firstWrite {
 				// this test case does not intend to test that cast header gets written once
 				c.writeCastHeaderOnce.Do(func() {})
 			}
-			if tt.sendInitialResize {
-				close(c.initialTermSizeSet)
-			}
 
 			c.stdoutStreamID.Store(stdoutStreamID)
 			c.stderrStreamID.Store(stderrStreamID)
 			for i, input := range tt.inputs {
-				c.hasTerm = tt.hasTerm
 				if _, err := c.Write(input); err != nil {
 					t.Errorf("[%d] spdyRemoteConnRecorder.Write() unexpected error %v", i, err)
 				}
 			}
 
 			// Assert that the expected bytes have been forwarded to the original destination.
-			gotForwarded := tc.WriteBufBytes()
+			gotForwarded := tc.writeBuf.Bytes()
 			if !reflect.DeepEqual(gotForwarded, tt.wantForwarded) {
 				t.Errorf("expected bytes not forwarded, wants\n%v\ngot\n%v", tt.wantForwarded, gotForwarded)
 			}
 
 			// Assert that the expected bytes have been forwarded to the session recorder.
-			gotRecorded := sr.Bytes()
+			gotRecorded := sr.buf.Bytes()
 			if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) {
 				t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", tt.wantRecorded, gotRecorded)
 			}
@@ -212,21 +197,25 @@ func Test_Reads(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tc := &fakes.TestConn{}
-			sr := &fakes.TestSessionRecorder{}
-			rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar())
-			c := &conn{
-				Conn:               tc,
-				log:                zl.Sugar(),
-				rec:                rec,
-				initialTermSizeSet: make(chan struct{}),
+			tc := &testConn{}
+			sr := &testSessionRecorder{}
+			rec := &recorder{
+				conn:  sr,
+				clock: cl,
+				start: cl.Now(),
+			}
+			c := &spdyRemoteConnRecorder{
+				Conn: tc,
+				log:  zl.Sugar(),
+				rec:  rec,
 			}
 			c.resizeStreamID.Store(tt.resizeStreamIDBeforeRead)
 
 			for i, input := range tt.inputs {
 				c.zlibReqReader = reader
-				tc.ResetReadBuf()
-				if err := tc.WriteReadBufBytes(input); err != nil {
+				tc.readBuf.Reset()
+				_, err := tc.readBuf.Write(input)
+				if err != nil {
 					t.Fatalf("writing bytes to test conn: %v", err)
 				}
 				_, err = c.Read(make([]byte, len(input)))
@@ -255,55 +244,17 @@ func Test_Reads(t *testing.T) {
 	}
 }
 
-// Test_conn_ReadRand tests reading arbitrarily generated byte slices from conn to
-// test that we don't panic when parsing input from a broken or malicious
-// client.
-func Test_conn_ReadRand(t *testing.T) {
-	zl, err := zap.NewDevelopment()
+func castLine(t *testing.T, p []byte, clock tstime.Clock) []byte {
+	t.Helper()
+	j, err := json.Marshal([]any{
+		clock.Now().Sub(clock.Now()).Seconds(),
+		"o",
+		string(p),
+	})
 	if err != nil {
-		t.Fatalf("error creating a test logger: %v", err)
-	}
-	for i := range 1000 {
-		tc := &fakes.TestConn{}
-		tc.ResetReadBuf()
-		c := &conn{
-			Conn: tc,
-			log:  zl.Sugar(),
-		}
-		bb := fakes.RandomBytes(t)
-		for j, input := range bb {
-			if err := tc.WriteReadBufBytes(input); err != nil {
-				t.Fatalf("[%d] writing bytes to test conn: %v", i, err)
-			}
-			f := func() {
-				c.Read(make([]byte, len(input)))
-			}
-			testPanic(t, f, fmt.Sprintf("[%d %d] Read panic parsing input of length %d", i, j, len(input)))
-		}
-	}
-}
-
-// Test_conn_WriteRand calls conn.Write with an arbitrary input to validate that
-// it does not panic.
-func Test_conn_WriteRand(t *testing.T) {
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatalf("error creating a test logger: %v", err)
-	}
-	for i := range 100 {
-		tc := &fakes.TestConn{}
-		c := &conn{
-			Conn: tc,
-			log:  zl.Sugar(),
-		}
-		bb := fakes.RandomBytes(t)
-		for j, input := range bb {
-			f := func() {
-				c.Write(input)
-			}
-			testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d", i, j, len(input)))
-		}
+		t.Fatalf("error marshalling cast line: %v", err)
 	}
+	return append(j, '\n')
 }
 
 func resizeMsgBytes(t *testing.T, width, height int) []byte {
@@ -314,3 +265,62 @@ func resizeMsgBytes(t *testing.T, width, height int) []byte {
 	}
 	return bs
 }
+
+func asciinemaResizeMsg(t *testing.T, width, height int) []byte {
+	t.Helper()
+	ch := CastHeader{
+		Width:  width,
+		Height: height,
+	}
+	bs, err := json.Marshal(ch)
+	if err != nil {
+		t.Fatalf("error marshalling CastHeader: %v", err)
+	}
+	return append(bs, '\n')
+}
+
+type testConn struct {
+	net.Conn
+	// writeBuf contains whatever was send to the conn via Write.
+	writeBuf bytes.Buffer
+	// readBuf contains whatever was sent to the conn via Read.
+	readBuf      bytes.Buffer
+	sync.RWMutex // protects the following
+	closed       bool
+}
+
+var _ net.Conn = &testConn{}
+
+func (tc *testConn) Read(b []byte) (int, error) {
+	return tc.readBuf.Read(b)
+}
+
+func (tc *testConn) Write(b []byte) (int, error) {
+	return tc.writeBuf.Write(b)
+}
+
+func (tc *testConn) Close() error {
+	tc.Lock()
+	defer tc.Unlock()
+	tc.closed = true
+	return nil
+}
+func (tc *testConn) isClosed() bool {
+	tc.Lock()
+	defer tc.Unlock()
+	return tc.closed
+}
+
+type testSessionRecorder struct {
+	// buf holds data that was sent to the session recorder.
+	buf bytes.Buffer
+}
+
+func (t *testSessionRecorder) Write(b []byte) (int, error) {
+	return t.buf.Write(b)
+}
+
+func (t *testSessionRecorder) Close() error {
+	t.buf.Reset()
+	return nil
+}

cmd/k8s-operator/sts.go

@@ -31,7 +31,6 @@ import (
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
@@ -47,11 +46,11 @@ const (
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
-	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
-	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
-	// the Ingress or Service.
+	// LabelProxyClass can be set by users on Connectors, tailscale
+	// Ingresses and Services that define cluster ingress or cluster egress,
+	// to specify that configuration in this ProxyClass should be applied to
+	// resources created for the Connector, Ingress or Service.
 	LabelProxyClass = "tailscale.com/proxy-class"
 
 	FinalizerName = "tailscale.com/finalizer"
@@ -65,8 +64,6 @@ const (
 	//MagicDNS name of tailnet node.
 	AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
 
-	AnnotationProxyGroup = "tailscale.com/proxy-group"
-
 	// Annotations settable by users on ingresses.
 	AnnotationFunnel = "tailscale.com/funnel"
 
@@ -304,7 +301,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaledConfigs, _ error) {
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaleConfigs, _ error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			// Hardcode a -0 suffix so that in future, if we support
@@ -345,7 +342,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		if len(tags) == 0 {
 			tags = a.defaultTags
 		}
-		authKey, err = newAuthKey(ctx, a.tsClient, tags)
+		authKey, err = a.newAuthKey(ctx, tags)
 		if err != nil {
 			return "", "", nil, err
 		}
@@ -362,7 +359,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 	latest := tailcfg.CapabilityVersion(-1)
 	var latestConfig ipn.ConfigVAlpha
 	for key, val := range configs {
-		fn := tsoperator.TailscaledConfigFileName(key)
+		fn := tsoperator.TailscaledConfigFileNameForCap(key)
 		b, err := json.Marshal(val)
 		if err != nil {
 			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
@@ -421,11 +418,6 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	if sec == nil {
 		return "", "", nil, nil
 	}
-
-	return deviceInfo(sec)
-}
-
-func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	id = tailcfg.StableNodeID(sec.Data["device_id"])
 	if id == "" {
 		return "", "", nil, nil
@@ -449,7 +441,7 @@ func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, i
 	return id, hostname, ips, nil
 }
 
-func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) {
+func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {
 	caps := tailscale.KeyCapabilities{
 		Devices: tailscale.KeyDeviceCapabilities{
 			Create: tailscale.KeyDeviceCreateCapabilities{
@@ -460,7 +452,7 @@ func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string,
 		},
 	}
 
-	key, _, err := tsClient.CreateKey(ctx, caps)
+	key, _, err := a.tsClient.CreateKey(ctx, caps)
 	if err != nil {
 		return "", err
 	}
@@ -606,18 +598,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	}
-	app, err := appInfoForProxy(sts)
-	if err != nil {
-		// No need to error out if now or in future we end up in a
-		// situation where app info cannot be determined for one of the
-		// many proxy configurations that the operator can produce.
-		logger.Error("[unexpected] unable to determine proxy type")
-	} else {
-		container.Env = append(container.Env, corev1.EnvVar{
-			Name:  "TS_INTERNAL_APP",
-			Value: app,
-		})
-	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	if sts.ProxyClassName != "" {
 		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
@@ -631,22 +611,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS)
 }
 
-func appInfoForProxy(cfg *tailscaleSTSConfig) (string, error) {
-	if cfg.ClusterTargetDNSName != "" || cfg.ClusterTargetIP != "" {
-		return kubetypes.AppIngressProxy, nil
-	}
-	if cfg.TailnetTargetFQDN != "" || cfg.TailnetTargetIP != "" {
-		return kubetypes.AppEgressProxy, nil
-	}
-	if cfg.ServeConfig != nil {
-		return kubetypes.AppIngressResource, nil
-	}
-	if cfg.Connector != nil {
-		return kubetypes.AppConnector, nil
-	}
-	return "", errors.New("unable to determine proxy type")
-}
-
 // mergeStatefulSetLabelsOrAnnots returns a map that contains all keys/values
 // present in 'custom' map as well as those keys/values from the current map
 // whose keys are present in the 'managed' map. The reason why this merge is
@@ -672,7 +636,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	if pc == nil || ss == nil {
 		return ss
 	}
-	if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
+	if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
 		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
 			enableMetrics(ss, pc)
 		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
@@ -718,7 +682,6 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
 	ss.Spec.Template.Spec.Affinity = wantsPod.Affinity
 	ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
-	ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints
 
 	// Update containers.
 	updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container {
@@ -795,7 +758,7 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 // TODO (irbekrm): remove the legacy config once we no longer need to support
 // versions older than cap94,
 // https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
-func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
+func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaleConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
 		AcceptDNS:           "false",
@@ -824,12 +787,33 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 
 	if newAuthkey != "" {
 		conf.AuthKey = &newAuthkey
-	} else if shouldRetainAuthKey(oldSecret) {
-		key, err := authKeyFromSecret(oldSecret)
-		if err != nil {
-			return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
+	} else if oldSecret != nil {
+		var err error
+		latest := tailcfg.CapabilityVersion(-1)
+		latestStr := ""
+		for k, data := range oldSecret.Data {
+			// write to StringData, read from Data as StringData is write-only
+			if len(data) == 0 {
+				continue
+			}
+			v, err := tsoperator.CapVerFromFileName(k)
+			if err != nil {
+				continue
+			}
+			if v > latest {
+				latestStr = k
+				latest = v
+			}
+		}
+		// Allow for configs that don't contain an auth key. Perhaps
+		// users have some mechanisms to delete them. Auth key is
+		// normally not needed after the initial login.
+		if latestStr != "" {
+			conf.AuthKey, err = readAuthKey(oldSecret, latestStr)
+			if err != nil {
+				return nil, err
+			}
 		}
-		conf.AuthKey = key
 	}
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
 	capVerConfigs[95] = *conf
@@ -839,41 +823,6 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 	return capVerConfigs, nil
 }
 
-func authKeyFromSecret(s *corev1.Secret) (key *string, err error) {
-	latest := tailcfg.CapabilityVersion(-1)
-	latestStr := ""
-	for k, data := range s.Data {
-		// write to StringData, read from Data as StringData is write-only
-		if len(data) == 0 {
-			continue
-		}
-		v, err := tsoperator.CapVerFromFileName(k)
-		if err != nil {
-			continue
-		}
-		if v > latest {
-			latestStr = k
-			latest = v
-		}
-	}
-	// Allow for configs that don't contain an auth key. Perhaps
-	// users have some mechanisms to delete them. Auth key is
-	// normally not needed after the initial login.
-	if latestStr != "" {
-		return readAuthKey(s, latestStr)
-	}
-	return key, nil
-}
-
-// shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be
-// retained (because the proxy has not yet successfully authenticated).
-func shouldRetainAuthKey(s *corev1.Secret) bool {
-	if s == nil {
-		return false // nothing to retain here
-	}
-	return len(s.Data["device_id"]) == 0 // proxy has not authed yet
-}
-
 func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
 	return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
 }
@@ -885,7 +834,7 @@ type ptrObject[T any] interface {
 	*T
 }
 
-type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
+type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
 
 // hashBytes produces a hash for the provided tailscaled config that is the same across
 // different invocations of this code. We do not use the
@@ -896,7 +845,7 @@ type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
 // thing that changed is operator version (the hash is also exposed to users via
 // an annotation and might be confusing if it changes without the config having
 // changed).
-func tailscaledConfigHash(c tailscaledConfigs) (string, error) {
+func tailscaledConfigHash(c tailscaleConfigs) (string, error) {
 	b, err := json.Marshal(c)
 	if err != nil {
 		return "", fmt.Errorf("error marshalling tailscaled configs: %w", err)

cmd/k8s-operator/sts_test.go

@@ -18,7 +18,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/types/ptr"
@@ -74,16 +73,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 					NodeSelector:     map[string]string{"beta.kubernetes.io/os": "linux"},
 					Affinity:         &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}},
 					Tolerations:      []corev1.Toleration{{Key: "", Operator: "Exists"}},
-					TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
-						{
-							WhenUnsatisfiable: "DoNotSchedule",
-							TopologyKey:       "kubernetes.io/hostname",
-							MaxSkew:           3,
-							LabelSelector: &metav1.LabelSelector{
-								MatchLabels: map[string]string{"foo": "bar"},
-							},
-						},
-					},
 					TailscaleContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
 							Privileged: ptr.To(true),
@@ -170,7 +159,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
-	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
@@ -213,7 +201,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
-	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)

cmd/k8s-operator/svc.go

@@ -25,7 +25,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
@@ -63,17 +62,15 @@ type ServiceReconciler struct {
 	tsNamespace string
 
 	clock tstime.Clock
-
-	defaultProxyClass string
 }
 
 var (
 	// gaugeEgressProxies tracks the number of egress proxies that we're
 	// currently managing.
-	gaugeEgressProxies = clientmetric.NewGauge(kubetypes.MetricEgressProxyCount)
+	gaugeEgressProxies = clientmetric.NewGauge("k8s_egress_proxies")
 	// gaugeIngressProxies tracks the number of ingress proxies that we're
 	// currently managing.
-	gaugeIngressProxies = clientmetric.NewGauge(kubetypes.MetricIngressProxyCount)
+	gaugeIngressProxies = clientmetric.NewGauge("k8s_ingress_proxies")
 )
 
 func childResourceLabels(name, ns, typ string) map[string]string {
@@ -112,10 +109,6 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
 
-	if _, ok := svc.Annotations[AnnotationProxyGroup]; ok {
-		return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup
-	}
-
 	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
@@ -215,7 +208,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc, a.defaultProxyClass)
+	proxyClass := proxyClassForObject(svc)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -332,7 +325,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	if err != nil {
 		msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
 		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
-		return errors.New(msg)
+		return fmt.Errorf(msg)
 	}
 	for _, ip := range tsIPs {
 		addr, err := netip.ParseAddr(ip)
@@ -358,15 +351,6 @@ func validateService(svc *corev1.Service) []string {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
-	if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" {
-		ip, err := netip.ParseAddr(ipStr)
-		if err != nil {
-			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err))
-		} else if !ip.IsValid() {
-			violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr))
-		}
-	}
-
 	svcName := nameForService(svc)
 	if err := dnsname.ValidLabel(svcName); err != nil {
 		if _, ok := svc.Annotations[AnnotationHostname]; ok {
@@ -420,14 +404,8 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-// proxyClassForObject returns the proxy class for the given object. If the
-// object does not have a proxy class label, it returns the default proxy class
-func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
-	proxyClass, exists := o.GetLabels()[LabelProxyClass]
-	if !exists {
-		proxyClass = proxyDefaultClass
-	}
-	return proxyClass
+func proxyClassForObject(o client.Object) string {
+	return o.GetLabels()[LabelProxyClass]
 }
 
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {

cmd/k8s-operator/testutils_test.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"encoding/json"
 	"net/netip"
-	"reflect"
 	"strings"
 	"sync"
 	"testing"
@@ -52,9 +51,6 @@ type configOpts struct {
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
 	proxyClass                                     string // configuration from the named ProxyClass should be applied to proxy resources
-	app                                            string
-	shouldRemoveAuthKey                            bool
-	secretExtraData                                map[string][]byte
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -146,10 +142,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
 		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
 	}
-	tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
-		Name:  "TS_INTERNAL_APP",
-		Value: opts.app,
-	})
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -232,7 +224,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
-			{Name: "TS_INTERNAL_APP", Value: opts.app},
 		},
 		ImagePullPolicy: "Always",
 		VolumeMounts: []corev1.VolumeMount{
@@ -367,9 +358,6 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 			conf.AcceptRoutes = "true"
 		}
 	}
-	if opts.shouldRemoveAuthKey {
-		conf.AuthKey = nil
-	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -410,9 +398,6 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		labels["tailscale.com/parent-resource-ns"] = "" // Connector is cluster scoped
 	}
 	s.Labels = labels
-	for key, val := range opts.secretExtraData {
-		mak.Set(&s.Data, key, val)
-	}
 	return s
 }
 
@@ -496,7 +481,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
-		t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
+		t.Fatalf("unexpected object (-got +want):\n%s", diff)
 	}
 }
 
@@ -507,7 +492,7 @@ func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns
 		Name:      name,
 		Namespace: ns,
 	}, obj); !apierrors.IsNotFound(err) {
-		t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
+		t.Fatalf("object %s/%s unexpectedly present, wanted missing", ns, name)
 	}
 }
 
@@ -601,17 +586,6 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili
 	return "secret-authkey", k, nil
 }
 
-func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) {
-	return &tailscale.Device{
-		DeviceID: deviceID,
-		Hostname: "hostname-" + deviceID,
-		Addresses: []string{
-			"1.2.3.4",
-			"::1",
-		},
-	}, nil
-}
-
 func (c *fakeTSClient) DeleteDevice(ctx context.Context, deviceID string) error {
 	c.Lock()
 	defer c.Unlock()
@@ -639,14 +613,6 @@ func removeHashAnnotation(sts *appsv1.StatefulSet) {
 	delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
 }
 
-func removeTargetPortsFromSvc(svc *corev1.Service) {
-	newPorts := make([]corev1.ServicePort, 0)
-	for _, p := range svc.Spec.Ports {
-		newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port})
-	}
-	svc.Spec.Ports = newPorts
-}
-
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()

cmd/k8s-operator/tsrecorder.go

@@ -1,397 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"slices"
-	"sync"
-
-	"github.com/pkg/errors"
-	"go.uber.org/zap"
-	xslices "golang.org/x/exp/slices"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"tailscale.com/client/tailscale"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
-	"tailscale.com/tailcfg"
-	"tailscale.com/tstime"
-	"tailscale.com/util/clientmetric"
-	"tailscale.com/util/set"
-)
-
-const (
-	reasonRecorderCreationFailed = "RecorderCreationFailed"
-	reasonRecorderCreated        = "RecorderCreated"
-	reasonRecorderInvalid        = "RecorderInvalid"
-
-	currentProfileKey = "_current-profile"
-)
-
-var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount)
-
-// RecorderReconciler syncs Recorder statefulsets with their definition in
-// Recorder CRs.
-type RecorderReconciler struct {
-	client.Client
-	l           *zap.SugaredLogger
-	recorder    record.EventRecorder
-	clock       tstime.Clock
-	tsNamespace string
-	tsClient    tsClient
-
-	mu        sync.Mutex           // protects following
-	recorders set.Slice[types.UID] // for recorders gauge
-}
-
-func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger {
-	return r.l.With("Recorder", name)
-}
-
-func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := r.logger(req.Name)
-	logger.Debugf("starting reconcile")
-	defer logger.Debugf("reconcile finished")
-
-	tsr := new(tsapi.Recorder)
-	err = r.Get(ctx, req.NamespacedName, tsr)
-	if apierrors.IsNotFound(err) {
-		logger.Debugf("Recorder not found, assuming it was deleted")
-		return reconcile.Result{}, nil
-	} else if err != nil {
-		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err)
-	}
-	if markedForDeletion(tsr) {
-		logger.Debugf("Recorder is being deleted, cleaning up resources")
-		ix := xslices.Index(tsr.Finalizers, FinalizerName)
-		if ix < 0 {
-			logger.Debugf("no finalizer, nothing to do")
-			return reconcile.Result{}, nil
-		}
-
-		if done, err := r.maybeCleanup(ctx, tsr); err != nil {
-			return reconcile.Result{}, err
-		} else if !done {
-			logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
-			return reconcile.Result{RequeueAfter: shortRequeue}, nil
-		}
-
-		tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
-		if err := r.Update(ctx, tsr); err != nil {
-			return reconcile.Result{}, err
-		}
-		return reconcile.Result{}, nil
-	}
-
-	oldTSRStatus := tsr.Status.DeepCopy()
-	setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
-		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldTSRStatus, tsr.Status) {
-			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
-		}
-		return reconcile.Result{}, err
-	}
-
-	if !slices.Contains(tsr.Finalizers, FinalizerName) {
-		// This log line is printed exactly once during initial provisioning,
-		// because once the finalizer is in place this block gets skipped. So,
-		// this is a nice place to log that the high level, multi-reconcile
-		// operation is underway.
-		logger.Infof("ensuring Recorder is set up")
-		tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
-		if err := r.Update(ctx, tsr); err != nil {
-			logger.Errorf("error adding finalizer: %w", err)
-			return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
-		}
-	}
-
-	if err := r.validate(tsr); err != nil {
-		logger.Errorf("error validating Recorder spec: %w", err)
-		message := fmt.Sprintf("Recorder is invalid: %s", err)
-		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message)
-		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
-	}
-
-	if err = r.maybeProvision(ctx, tsr); err != nil {
-		logger.Errorf("error creating Recorder resources: %w", err)
-		message := fmt.Sprintf("failed creating Recorder: %s", err)
-		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
-		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, message)
-	}
-
-	logger.Info("Recorder resources synced")
-	return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated)
-}
-
-func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error {
-	logger := r.logger(tsr.Name)
-
-	r.mu.Lock()
-	r.recorders.Add(tsr.UID)
-	gaugeRecorderResources.Set(int64(r.recorders.Len()))
-	r.mu.Unlock()
-
-	if err := r.ensureAuthSecretCreated(ctx, tsr); err != nil {
-		return fmt.Errorf("error creating secrets: %w", err)
-	}
-	// State secret is precreated so we can use the Recorder CR as its owner ref.
-	sec := tsrStateSecret(tsr, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
-		s.ObjectMeta.Labels = sec.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
-	}); err != nil {
-		return fmt.Errorf("error creating state Secret: %w", err)
-	}
-	sa := tsrServiceAccount(tsr, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
-		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
-	}); err != nil {
-		return fmt.Errorf("error creating ServiceAccount: %w", err)
-	}
-	role := tsrRole(tsr, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
-		r.ObjectMeta.Labels = role.ObjectMeta.Labels
-		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
-		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
-		r.Rules = role.Rules
-	}); err != nil {
-		return fmt.Errorf("error creating Role: %w", err)
-	}
-	roleBinding := tsrRoleBinding(tsr, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
-		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
-		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
-		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
-		r.RoleRef = roleBinding.RoleRef
-		r.Subjects = roleBinding.Subjects
-	}); err != nil {
-		return fmt.Errorf("error creating RoleBinding: %w", err)
-	}
-	ss := tsrStatefulSet(tsr, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
-		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
-		s.Spec = ss.Spec
-	}); err != nil {
-		return fmt.Errorf("error creating StatefulSet: %w", err)
-	}
-
-	var devices []tsapi.RecorderTailnetDevice
-
-	device, ok, err := r.getDeviceInfo(ctx, tsr.Name)
-	if err != nil {
-		return fmt.Errorf("failed to get device info: %w", err)
-	}
-	if !ok {
-		logger.Debugf("no Tailscale hostname known yet, waiting for Recorder pod to finish auth")
-		return nil
-	}
-
-	devices = append(devices, device)
-
-	tsr.Status.Devices = devices
-
-	return nil
-}
-
-// maybeCleanup just deletes the device from the tailnet. All the kubernetes
-// resources linked to a Recorder will get cleaned up via owner references
-// (which we can use because they are all in the same namespace).
-func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
-	logger := r.logger(tsr.Name)
-
-	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
-	if err != nil {
-		return false, err
-	}
-	if !ok {
-		logger.Debugf("state Secret %s-0 not found or does not contain node ID, continuing cleanup", tsr.Name)
-		r.mu.Lock()
-		r.recorders.Remove(tsr.UID)
-		gaugeRecorderResources.Set(int64(r.recorders.Len()))
-		r.mu.Unlock()
-		return true, nil
-	}
-
-	logger.Debugf("deleting device %s from control", string(id))
-	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
-		errResp := &tailscale.ErrResponse{}
-		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
-			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
-		} else {
-			return false, fmt.Errorf("error deleting device: %w", err)
-		}
-	} else {
-		logger.Debugf("device %s deleted from control", string(id))
-	}
-
-	// Unlike most log entries in the reconcile loop, this will get printed
-	// exactly once at the very end of cleanup, because the final step of
-	// cleanup removes the tailscale finalizer, which will make all future
-	// reconciles exit early.
-	logger.Infof("cleaned up Recorder resources")
-	r.mu.Lock()
-	r.recorders.Remove(tsr.UID)
-	gaugeRecorderResources.Set(int64(r.recorders.Len()))
-	r.mu.Unlock()
-	return true, nil
-}
-
-func (r *RecorderReconciler) ensureAuthSecretCreated(ctx context.Context, tsr *tsapi.Recorder) error {
-	logger := r.logger(tsr.Name)
-	key := types.NamespacedName{
-		Namespace: r.tsNamespace,
-		Name:      tsr.Name,
-	}
-	if err := r.Get(ctx, key, &corev1.Secret{}); err == nil {
-		// No updates, already created the auth key.
-		logger.Debugf("auth Secret %s already exists", key.Name)
-		return nil
-	} else if !apierrors.IsNotFound(err) {
-		return err
-	}
-
-	// Create the auth key Secret which is going to be used by the StatefulSet
-	// to authenticate with Tailscale.
-	logger.Debugf("creating authkey for new Recorder")
-	tags := tsr.Spec.Tags
-	if len(tags) == 0 {
-		tags = tsapi.Tags{"tag:k8s"}
-	}
-	authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify())
-	if err != nil {
-		return err
-	}
-
-	logger.Debug("creating a new Secret for the Recorder")
-	if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *RecorderReconciler) validate(tsr *tsapi.Recorder) error {
-	if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil {
-		return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible")
-	}
-
-	return nil
-}
-
-func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string) (*corev1.Secret, error) {
-	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: r.tsNamespace,
-			Name:      fmt.Sprintf("%s-0", tsrName),
-		},
-	}
-	if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
-		if apierrors.IsNotFound(err) {
-			return nil, nil
-		}
-
-		return nil, fmt.Errorf("error getting state Secret: %w", err)
-	}
-
-	return secret, nil
-}
-
-func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
-	secret, err := r.getStateSecret(ctx, tsrName)
-	if err != nil || secret == nil {
-		return "", "", false, err
-	}
-
-	return getNodeMetadata(ctx, secret)
-}
-
-// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
-// is expected to always be non-empty if the node ID is, but not required.
-func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
-	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
-	currentProfile, ok := secret.Data[currentProfileKey]
-	if !ok {
-		return "", "", false, nil
-	}
-	profileBytes, ok := secret.Data[string(currentProfile)]
-	if !ok {
-		return "", "", false, nil
-	}
-	var profile profile
-	if err := json.Unmarshal(profileBytes, &profile); err != nil {
-		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
-	}
-
-	ok = profile.Config.NodeID != ""
-	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
-}
-
-func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	secret, err := r.getStateSecret(ctx, tsrName)
-	if err != nil || secret == nil {
-		return tsapi.RecorderTailnetDevice{}, false, err
-	}
-
-	return getDeviceInfo(ctx, r.tsClient, secret)
-}
-
-func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	nodeID, dnsName, ok, err := getNodeMetadata(ctx, secret)
-	if !ok || err != nil {
-		return tsapi.RecorderTailnetDevice{}, false, err
-	}
-
-	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
-	// need the API. Should we instead update the profile to include addresses?
-	device, err := tsClient.Device(ctx, string(nodeID), nil)
-	if err != nil {
-		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
-	}
-
-	d = tsapi.RecorderTailnetDevice{
-		Hostname:   device.Hostname,
-		TailnetIPs: device.Addresses,
-	}
-	if dnsName != "" {
-		d.URL = fmt.Sprintf("https://%s", dnsName)
-	}
-
-	return d, true, nil
-}
-
-type profile struct {
-	Config struct {
-		NodeID      string `json:"NodeID"`
-		UserProfile struct {
-			LoginName string `json:"LoginName"`
-		} `json:"UserProfile"`
-	} `json:"Config"`
-}
-
-func markedForDeletion(obj metav1.Object) bool {
-	return !obj.GetDeletionTimestamp().IsZero()
-}