remove clusterPeers and use peers and tags to find peers

.github/workflows/checklocks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Build checklocks
         run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks

.github/workflows/codeql-analysis.yml

@@ -45,17 +45,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      uses: actions/setup-go@v4
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-file-build.yml

@@ -10,6 +10,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/checkout@v4
     - name: "Build Docker image"
       run: docker build .

.github/workflows/flakehub-publish-tagged.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: "write"
       contents: "read"
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: "actions/checkout@v4"
         with:
           ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
       - uses: "DeterminateSystems/nix-installer-action@main"

.github/workflows/golangci-lint.yml

@@ -23,9 +23,9 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: false

.github/workflows/govulncheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Install govulncheck
         run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -24,7 +24,7 @@ jobs:
 
       - name: Post to slack
         if: failure() && github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@v1.24.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
         with:

.github/workflows/installer.yml

@@ -98,7 +98,7 @@ jobs:
       # We cannot use v4, as it requires a newer glibc version than some of the
       # tested images provide. See
       # https://github.com/actions/checkout/issues/1487
-      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      uses: actions/checkout@v3
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/kubemanifests.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
     - name: Check out code
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Build and lint Helm chart
       run: |
         eval `./tool/go run ./cmd/mkversion`

.github/workflows/ssh-integrationtest.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
       - name: Run SSH integration tests
         run: |
           make sshintegrationtest

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           - shard: '4/4'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: integration tests as root
@@ -78,9 +78,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -150,16 +150,16 @@ jobs:
     runs-on: windows-2022
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
 
     - name: Install Go
-      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      uses: actions/setup-go@v4
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -190,7 +190,7 @@ jobs:
       options: --privileged
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
@@ -202,7 +202,7 @@ jobs:
     if: github.repository == 'tailscale/tailscale'
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Run VM tests
       run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004
       env:
@@ -214,7 +214,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: build all
       run: ./tool/go install -race ./cmd/...
     - name: build tests
@@ -258,9 +258,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -295,7 +295,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: build some
       run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient
       env:
@@ -317,9 +317,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -350,7 +350,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
       # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed
       # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch
       # some Android breakages early.
@@ -365,9 +365,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: Restore Cache
-      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+      uses: actions/cache@v3
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -399,7 +399,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: test tailscale_go
       run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/...
 
@@ -456,22 +456,18 @@ jobs:
         fuzz-seconds: 300
         dry-run: false
         language: go
-    - name: Set artifacts_path in env (workaround for actions/upload-artifact#176)
-      if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
-      run: |
-        echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      uses: actions/upload-artifact@v3
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts
-        path: ${{ env.artifacts_path }}/out/artifacts
+        path: ./out/artifacts
 
   depaware:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check depaware
       run: |
         export PATH=$(./tool/go env GOROOT)/bin:$PATH
@@ -481,7 +477,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check that 'go generate' is clean
       run: |
         pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
@@ -494,7 +490,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check that 'go mod tidy' is clean
       run: |
         ./tool/go mod tidy
@@ -506,7 +502,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: check licenses
       run: ./scripts/check_license_headers.sh .
 
@@ -522,7 +518,7 @@ jobs:
             goarch: "386"
     steps:
     - name: checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@v4
     - name: install staticcheck
       run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck
     - name: run staticcheck
@@ -563,7 +559,7 @@ jobs:
       # By having the job always run, but skipping its only step as needed, we
       # let the CI output collapse nicely in PRs.
       if: failure() && github.event_name == 'push'
-      uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+      uses: ruby/action-slack@v3.2.1
       with:
         payload: |
           {
@@ -578,7 +574,6 @@ jobs:
           }
       env:
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   check_mergeability:
     if: always()
@@ -601,6 +596,6 @@ jobs:
     steps:
     - name: Decide if change is okay to merge
       if: github.event_name != 'push'
-      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
+      uses: re-actors/alls-green@release/v1
       with:
         jobs: ${{ toJSON(needs) }}

.github/workflows/update-flake.yml

@@ -21,22 +21,21 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Run update-flakes
         run: ./update-flake.sh
 
       - name: Get access token
-        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
         id: generate-token
         with:
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_retrieval_mode: "id"
-          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 #v7.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply+flakes-updater@tailscale.com>

.github/workflows/update-webclient-prebuilt.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
 
       - name: Run go get
         run: |
@@ -23,19 +23,18 @@ jobs:
           ./tool/go mod tidy
 
       - name: Get access token
-        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
         id: generate-token
         with:
           # TODO(will): this should use the code updater app rather than licensing.
           # It has the same permissions, so not a big deal, but still.
           app_id: ${{ secrets.LICENSING_APP_ID }}
-          installation_retrieval_mode: "id"
-          installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
+          installation_id: ${{ secrets.LICENSING_APP_INSTALLATION_ID }}
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
         id: pull-request
-        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 #v7.0.1
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: OSS Updater <noreply+oss-updater@tailscale.com>

.github/workflows/webclient.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@v4
       - name: Install deps
         run: ./tool/yarn --cwd client/web
       - name: Run lint

VERSION.txt

@@ -1 +1 @@
-1.77.0
+1.75.0

assert_ts_toolchain_match.go

@@ -1,27 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build tailscale_go
-
-package tailscaleroot
-
-import (
-	"fmt"
-	"os"
-	"strings"
-)
-
-func init() {
-	tsRev, ok := tailscaleToolchainRev()
-	if !ok {
-		panic("binary built with tailscale_go build tag but failed to read build info or find tailscale.toolchain.rev in build info")
-	}
-	want := strings.TrimSpace(GoToolchainRev)
-	if tsRev != want {
-		if os.Getenv("TS_PERMIT_TOOLCHAIN_MISMATCH") == "1" {
-			fmt.Fprintf(os.Stderr, "tailscale.toolchain.rev = %q, want %q; but ignoring due to TS_PERMIT_TOOLCHAIN_MISMATCH=1\n", tsRev, want)
-			return
-		}
-		panic(fmt.Sprintf("binary built with tailscale_go build tag but Go toolchain %q doesn't match github.com/tailscale/tailscale expected value %q; override this failure with TS_PERMIT_TOOLCHAIN_MISMATCH=1", tsRev, want))
-	}
-}

build_docker.sh

@@ -17,20 +17,12 @@ eval "$(./build_dist.sh shellvars)"
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
 DEFAULT_BASE="tailscale/alpine-base:3.18"
-# Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
-# Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
-# annotations defined here will be overriden by release scripts that call this script.
-# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
-DEFAULT_ANNOTATIONS="org.opencontainers.image.source=https://github.com/tailscale/tailscale/blob/main/build_docker.sh,org.opencontainers.image.vendor=Tailscale"
 
 PUSH="${PUSH:-false}"
 TARGET="${TARGET:-${DEFAULT_TARGET}}"
 TAGS="${TAGS:-${DEFAULT_TAGS}}"
 BASE="${BASE:-${DEFAULT_BASE}}"
 PLATFORM="${PLATFORM:-}" # default to all platforms
-# OCI annotations that will be added to the image.
-# https://github.com/opencontainers/image-spec/blob/main/annotations.md
-ANNOTATIONS="${ANNOTATIONS:-${DEFAULT_ANNOTATIONS}}"
 
 case "$TARGET" in
   client)
@@ -51,7 +43,6 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
-      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/containerboot
     ;;
   operator)
@@ -65,11 +56,9 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
-      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
-      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/operator
     ;;
   k8s-nameserver)
@@ -83,11 +72,9 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
-      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
-      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/k8s-nameserver
     ;;
   *)

client/tailscale/apitype/apitype.go

@@ -4,10 +4,7 @@
 // Package apitype contains types for the Tailscale LocalAPI and control plane API.
 package apitype
 
-import (
-	"tailscale.com/tailcfg"
-	"tailscale.com/types/dnstype"
-)
+import "tailscale.com/tailcfg"
 
 // LocalAPIHost is the Host header value used by the LocalAPI.
 const LocalAPIHost = "local-tailscaled.sock"
@@ -68,11 +65,3 @@ type DNSOSConfig struct {
 	SearchDomains []string
 	MatchDomains  []string
 }
-
-// DNSQueryResponse is the response to a DNS query request sent via LocalAPI.
-type DNSQueryResponse struct {
-	// Bytes is the raw DNS response bytes.
-	Bytes []byte
-	// Resolvers is the list of resolvers that the forwarder deemed able to resolve the query.
-	Resolvers []*dnstype.Resolver
-}

client/tailscale/localclient.go

@@ -37,10 +37,8 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
-	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
-	"tailscale.com/util/syspolicy/setting"
 )
 
 // defaultLocalClient is the default LocalClient when using the legacy
@@ -815,35 +813,6 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
-// GetEffectivePolicy returns the effective policy for the specified scope.
-func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
-	scopeID, err := scope.MarshalText()
-	if err != nil {
-		return nil, err
-	}
-	body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID))
-	if err != nil {
-		return nil, err
-	}
-	return decodeJSON[*setting.Snapshot](body)
-}
-
-// ReloadEffectivePolicy reloads the effective policy for the specified scope
-// by reading and merging policy settings from all applicable policy sources.
-func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
-	scopeID, err := scope.MarshalText()
-	if err != nil {
-		return nil, err
-	}
-	body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody)
-	if err != nil {
-		return nil, err
-	}
-	return decodeJSON[*setting.Snapshot](body)
-}
-
-// GetDNSOSConfig returns the system DNS configuration for the current device.
-// That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
 func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig")
 	if err != nil {
@@ -856,21 +825,6 @@ func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig
 	return &osCfg, nil
 }
 
-// QueryDNS executes a DNS query for a name (`google.com.`) and query type (`CNAME`).
-// It returns the raw DNS response bytes and the resolvers that were used to answer the query
-// (often just one, but can be more if we raced multiple resolvers).
-func (lc *LocalClient) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) {
-	body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType))
-	if err != nil {
-		return nil, nil, err
-	}
-	var res apitype.DNSQueryResponse
-	if err := json.Unmarshal(body, &res); err != nil {
-		return nil, nil, fmt.Errorf("invalid query response: %w", err)
-	}
-	return res.Bytes, res.Resolvers, nil
-}
-
 // StartLoginInteractive starts an interactive login.
 func (lc *LocalClient) StartLoginInteractive(ctx context.Context) error {
 	_, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil)
@@ -1327,17 +1281,6 @@ func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConf
 	return nil
 }
 
-// DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be
-// run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over
-// to another replica whilst there is still some grace period for the existing connections to terminate.
-func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
-	_, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil)
-	if err != nil {
-		return fmt.Errorf("error disconnecting control: %w", err)
-	}
-	return nil
-}
-
 // NetworkLockDisable shuts down network-lock across the tailnet.
 func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) error {
 	if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil {

client/tailscale/tailscale.go

@@ -51,9 +51,6 @@ type Client struct {
 	// HTTPClient optionally specifies an alternate HTTP client to use.
 	// If nil, http.DefaultClient is used.
 	HTTPClient *http.Client
-
-	// UserAgent optionally specifies an alternate User-Agent header
-	UserAgent string
 }
 
 func (c *Client) httpClient() *http.Client {
@@ -100,9 +97,8 @@ func (c *Client) setAuth(r *http.Request) {
 // and can be changed manually by the user.
 func NewClient(tailnet string, auth AuthMethod) *Client {
 	return &Client{
-		tailnet:   tailnet,
-		auth:      auth,
-		UserAgent: "tailscale-client-oss",
+		tailnet: tailnet,
+		auth:    auth,
 	}
 }
 
@@ -114,16 +110,17 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 		return nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
 	}
 	c.setAuth(req)
-	if c.UserAgent != "" {
-		req.Header.Set("User-Agent", c.UserAgent)
-	}
 	return c.httpClient().Do(req)
 }
 
 // sendRequest add the authentication key to the request and sends it. It
 // receives the response and reads up to 10MB of it.
 func (c *Client) sendRequest(req *http.Request) ([]byte, *http.Response, error) {
-	resp, err := c.Do(req)
+	if !I_Acknowledge_This_API_Is_Unstable {
+		return nil, nil, errors.New("use of Client without setting I_Acknowledge_This_API_Is_Unstable")
+	}
+	c.setAuth(req)
+	resp, err := c.httpClient().Do(req)
 	if err != nil {
 		return nil, resp, err
 	}

client/web/web.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -26,7 +27,6 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
-	"tailscale.com/envknob/featureknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -35,7 +35,6 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/views"
 	"tailscale.com/util/httpm"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -114,6 +113,11 @@ const (
 	ManageServerMode ServerMode = "manage"
 )
 
+var (
+	exitNodeRouteV4 = netip.MustParsePrefix("0.0.0.0/0")
+	exitNodeRouteV6 = netip.MustParsePrefix("::/0")
+)
+
 // ServerOpts contains options for constructing a new Server.
 type ServerOpts struct {
 	// Mode specifies the mode of web client being constructed.
@@ -923,10 +927,10 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 			return p == route
 		})
 	}
-	data.AdvertisingExitNodeApproved = routeApproved(tsaddr.AllIPv4()) || routeApproved(tsaddr.AllIPv6())
+	data.AdvertisingExitNodeApproved = routeApproved(exitNodeRouteV4) || routeApproved(exitNodeRouteV6)
 
 	for _, r := range prefs.AdvertiseRoutes {
-		if tsaddr.IsExitRoute(r) {
+		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
 			data.AdvertisingExitNode = true
 		} else {
 			data.AdvertisedRoutes = append(data.AdvertisedRoutes, subnetRoute{
@@ -961,16 +965,37 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 }
 
 func availableFeatures() map[string]bool {
+	env := hostinfo.GetEnvType()
 	features := map[string]bool{
 		"advertise-exit-node": true, // available on all platforms
 		"advertise-routes":    true, // available on all platforms
-		"use-exit-node":       featureknob.CanUseExitNode() == nil,
-		"ssh":                 featureknob.CanRunTailscaleSSH() == nil,
+		"use-exit-node":       canUseExitNode(env) == nil,
+		"ssh":                 envknob.CanRunTailscaleSSH() == nil,
 		"auto-update":         version.IsUnstableBuild() && clientupdate.CanAutoUpdate(),
 	}
+	if env == hostinfo.HomeAssistantAddOn {
+		// Setting SSH on Home Assistant causes trouble on startup
+		// (since the flag is not being passed to `tailscale up`).
+		// Although Tailscale SSH does work here,
+		// it's not terribly useful since it's running in a separate container.
+		features["ssh"] = false
+	}
 	return features
 }
 
+func canUseExitNode(env hostinfo.EnvType) error {
+	switch dist := distro.Get(); dist {
+	case distro.Synology, // see https://github.com/tailscale/tailscale/issues/1995
+		distro.QNAP,
+		distro.Unraid:
+		return fmt.Errorf("Tailscale exit nodes cannot be used on %s.", dist)
+	}
+	if env == hostinfo.HomeAssistantAddOn {
+		return errors.New("Tailscale exit nodes cannot be used on Home Assistant.")
+	}
+	return nil
+}
+
 // aclsAllowAccess returns whether tailnet ACLs (as expressed in the provided filter rules)
 // permit any devices to access the local web client.
 // This does not currently check whether a specific device can connect, just any device.
@@ -1046,7 +1071,7 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 	var currNonExitRoutes []string
 	var currAdvertisingExitNode bool
 	for _, r := range prefs.AdvertiseRoutes {
-		if tsaddr.IsExitRoute(r) {
+		if r == exitNodeRouteV4 || r == exitNodeRouteV6 {
 			currAdvertisingExitNode = true
 			continue
 		}
@@ -1067,7 +1092,12 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er
 		return err
 	}
 
-	if !data.UseExitNode.IsZero() && tsaddr.ContainsExitRoutes(views.SliceOf(routes)) {
+	hasExitNodeRoute := func(all []netip.Prefix) bool {
+		return slices.Contains(all, exitNodeRouteV4) ||
+			slices.Contains(all, exitNodeRouteV6)
+	}
+
+	if !data.UseExitNode.IsZero() && hasExitNodeRoute(routes) {
 		return errors.New("cannot use and advertise exit node at same time")
 	}
 

clientupdate/clientupdate.go

@@ -27,8 +27,11 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
+	"tailscale.com/clientupdate/distsign"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
+	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
@@ -753,6 +756,164 @@ func (up *Updater) updateMacAppStore() error {
 	return nil
 }
 
+const (
+	// winMSIEnv is the environment variable that, if set, is the MSI file for
+	// the update command to install. It's passed like this so we can stop the
+	// tailscale.exe process from running before the msiexec process runs and
+	// tries to overwrite ourselves.
+	winMSIEnv = "TS_UPDATE_WIN_MSI"
+	// winExePathEnv is the environment variable that is set along with
+	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
+	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
+	// install is complete.
+	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
+)
+
+var (
+	verifyAuthenticode func(string) error // set non-nil only on Windows
+	markTempFileFunc   func(string) error // set non-nil only on Windows
+)
+
+func (up *Updater) updateWindows() error {
+	if msi := os.Getenv(winMSIEnv); msi != "" {
+		// stdout/stderr from this part of the install could be lost since the
+		// parent tailscaled is replaced. Create a temp log file to have some
+		// output to debug with in case update fails.
+		close, err := up.switchOutputToFile()
+		if err != nil {
+			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
+		} else {
+			defer close.Close()
+		}
+
+		up.Logf("installing %v ...", msi)
+		if err := up.installMSI(msi); err != nil {
+			up.Logf("MSI install failed: %v", err)
+			return err
+		}
+
+		up.Logf("success.")
+		return nil
+	}
+
+	if !winutil.IsCurrentProcessElevated() {
+		return errors.New(`update must be run as Administrator
+
+you can run the command prompt as Administrator one of these ways:
+* right-click cmd.exe, select 'Run as administrator'
+* press Windows+x, then press a
+* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
+	}
+	ver, err := requestedTailscaleVersion(up.Version, up.Track)
+	if err != nil {
+		return err
+	}
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	if !up.confirm(ver) {
+		return nil
+	}
+
+	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
+	msiDir := filepath.Join(tsDir, "MSICache")
+	if fi, err := os.Stat(tsDir); err != nil {
+		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
+	}
+	if err := os.MkdirAll(msiDir, 0700); err != nil {
+		return err
+	}
+	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
+	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
+	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
+		return err
+	}
+
+	up.Logf("verifying MSI authenticode...")
+	if err := verifyAuthenticode(msiTarget); err != nil {
+		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
+	}
+	up.Logf("authenticode verification succeeded")
+
+	up.Logf("making tailscale.exe copy to switch to...")
+	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
+	selfOrig, selfCopy, err := makeSelfCopy()
+	if err != nil {
+		return err
+	}
+	defer os.Remove(selfCopy)
+	up.Logf("running tailscale.exe copy for final install...")
+
+	cmd := exec.Command(selfCopy, "update")
+	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
+	cmd.Stdout = up.Stderr
+	cmd.Stderr = up.Stderr
+	cmd.Stdin = os.Stdin
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	// Once it's started, exit ourselves, so the binary is free
+	// to be replaced.
+	os.Exit(0)
+	panic("unreachable")
+}
+
+func (up *Updater) switchOutputToFile() (io.Closer, error) {
+	var logFilePath string
+	exePath, err := os.Executable()
+	if err != nil {
+		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
+	} else {
+		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
+	}
+
+	up.Logf("writing update output to %q", logFilePath)
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	up.Logf = func(m string, args ...any) {
+		fmt.Fprintf(logFile, m+"\n", args...)
+	}
+	up.Stdout = logFile
+	up.Stderr = logFile
+	return logFile, nil
+}
+
+func (up *Updater) installMSI(msi string) error {
+	var err error
+	for tries := 0; tries < 2; tries++ {
+		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
+		cmd.Dir = filepath.Dir(msi)
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		if err == nil {
+			break
+		}
+		up.Logf("Install attempt failed: %v", err)
+		uninstallVersion := up.currentVersion
+		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
+			uninstallVersion = v
+		}
+		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
+		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
+		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
+		cmd.Stdout = up.Stdout
+		cmd.Stderr = up.Stderr
+		cmd.Stdin = os.Stdin
+		err = cmd.Run()
+		up.Logf("msiexec uninstall: %v", err)
+	}
+	return err
+}
+
 // cleanupOldDownloads removes all files matching glob (see filepath.Glob).
 // Only regular files are removed, so the glob must match specific files and
 // not directories.
@@ -777,6 +938,53 @@ func (up *Updater) cleanupOldDownloads(glob string) {
 	}
 }
 
+func msiUUIDForVersion(ver string) string {
+	arch := runtime.GOARCH
+	if arch == "386" {
+		arch = "x86"
+	}
+	track, err := versionToTrack(ver)
+	if err != nil {
+		track = UnstableTrack
+	}
+	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
+	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
+}
+
+func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
+	selfExe, err := os.Executable()
+	if err != nil {
+		return "", "", err
+	}
+	f, err := os.Open(selfExe)
+	if err != nil {
+		return "", "", err
+	}
+	defer f.Close()
+	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
+	if err != nil {
+		return "", "", err
+	}
+	if f := markTempFileFunc; f != nil {
+		if err := f(f2.Name()); err != nil {
+			return "", "", err
+		}
+	}
+	if _, err := io.Copy(f2, f); err != nil {
+		f2.Close()
+		return "", "", err
+	}
+	return selfExe, f2.Name(), f2.Close()
+}
+
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
+	if err != nil {
+		return err
+	}
+	return c.Download(context.Background(), pathSrc, fileDst)
+}
+
 func (up *Updater) updateFreeBSD() (err error) {
 	if up.Version != "" {
 		return errors.New("installing a specific version on FreeBSD is not supported")

clientupdate/clientupdate_downloads.go

@@ -1,20 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build (linux && !android) || windows
-
-package clientupdate
-
-import (
-	"context"
-
-	"tailscale.com/clientupdate/distsign"
-)
-
-func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
-	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
-	if err != nil {
-		return err
-	}
-	return c.Download(context.Background(), pathSrc, fileDst)
-}

clientupdate/clientupdate_not_downloads.go

@@ -1,10 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !((linux && !android) || windows)
-
-package clientupdate
-
-func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
-	panic("unreachable")
-}

clientupdate/clientupdate_notwindows.go

@@ -1,10 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !windows
-
-package clientupdate
-
-func (up *Updater) updateWindows() error {
-	panic("unreachable")
-}

clientupdate/clientupdate_windows.go

@@ -7,57 +7,13 @@
 package clientupdate
 
 import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"runtime"
-	"strings"
-
-	"github.com/google/uuid"
 	"golang.org/x/sys/windows"
-	"tailscale.com/util/winutil"
 	"tailscale.com/util/winutil/authenticode"
 )
 
-const (
-	// winMSIEnv is the environment variable that, if set, is the MSI file for
-	// the update command to install. It's passed like this so we can stop the
-	// tailscale.exe process from running before the msiexec process runs and
-	// tries to overwrite ourselves.
-	winMSIEnv = "TS_UPDATE_WIN_MSI"
-	// winExePathEnv is the environment variable that is set along with
-	// winMSIEnv and carries the full path of the calling tailscale.exe binary.
-	// It is used to re-launch the GUI process (tailscale-ipn.exe) after
-	// install is complete.
-	winExePathEnv = "TS_UPDATE_WIN_EXE_PATH"
-)
-
-func makeSelfCopy() (origPathExe, tmpPathExe string, err error) {
-	selfExe, err := os.Executable()
-	if err != nil {
-		return "", "", err
-	}
-	f, err := os.Open(selfExe)
-	if err != nil {
-		return "", "", err
-	}
-	defer f.Close()
-	f2, err := os.CreateTemp("", "tailscale-updater-*.exe")
-	if err != nil {
-		return "", "", err
-	}
-	if err := markTempFileWindows(f2.Name()); err != nil {
-		return "", "", err
-	}
-	if _, err := io.Copy(f2, f); err != nil {
-		f2.Close()
-		return "", "", err
-	}
-	return selfExe, f2.Name(), f2.Close()
+func init() {
+	markTempFileFunc = markTempFileWindows
+	verifyAuthenticode = verifyTailscale
 }
 
 func markTempFileWindows(name string) error {
@@ -67,159 +23,6 @@ func markTempFileWindows(name string) error {
 
 const certSubjectTailscale = "Tailscale Inc."
 
-func verifyAuthenticode(path string) error {
+func verifyTailscale(path string) error {
 	return authenticode.Verify(path, certSubjectTailscale)
 }
-
-func (up *Updater) updateWindows() error {
-	if msi := os.Getenv(winMSIEnv); msi != "" {
-		// stdout/stderr from this part of the install could be lost since the
-		// parent tailscaled is replaced. Create a temp log file to have some
-		// output to debug with in case update fails.
-		close, err := up.switchOutputToFile()
-		if err != nil {
-			up.Logf("failed to create log file for installation: %v; proceeding with existing outputs", err)
-		} else {
-			defer close.Close()
-		}
-
-		up.Logf("installing %v ...", msi)
-		if err := up.installMSI(msi); err != nil {
-			up.Logf("MSI install failed: %v", err)
-			return err
-		}
-
-		up.Logf("success.")
-		return nil
-	}
-
-	if !winutil.IsCurrentProcessElevated() {
-		return errors.New(`update must be run as Administrator
-
-you can run the command prompt as Administrator one of these ways:
-* right-click cmd.exe, select 'Run as administrator'
-* press Windows+x, then press a
-* press Windows+r, type in "cmd", then press Ctrl+Shift+Enter`)
-	}
-	ver, err := requestedTailscaleVersion(up.Version, up.Track)
-	if err != nil {
-		return err
-	}
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	if !up.confirm(ver) {
-		return nil
-	}
-
-	tsDir := filepath.Join(os.Getenv("ProgramData"), "Tailscale")
-	msiDir := filepath.Join(tsDir, "MSICache")
-	if fi, err := os.Stat(tsDir); err != nil {
-		return fmt.Errorf("expected %s to exist, got stat error: %w", tsDir, err)
-	} else if !fi.IsDir() {
-		return fmt.Errorf("expected %s to be a directory; got %v", tsDir, fi.Mode())
-	}
-	if err := os.MkdirAll(msiDir, 0700); err != nil {
-		return err
-	}
-	up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi"))
-	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch)
-	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
-	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
-		return err
-	}
-
-	up.Logf("verifying MSI authenticode...")
-	if err := verifyAuthenticode(msiTarget); err != nil {
-		return fmt.Errorf("authenticode verification of %s failed: %w", msiTarget, err)
-	}
-	up.Logf("authenticode verification succeeded")
-
-	up.Logf("making tailscale.exe copy to switch to...")
-	up.cleanupOldDownloads(filepath.Join(os.TempDir(), "tailscale-updater-*.exe"))
-	selfOrig, selfCopy, err := makeSelfCopy()
-	if err != nil {
-		return err
-	}
-	defer os.Remove(selfCopy)
-	up.Logf("running tailscale.exe copy for final install...")
-
-	cmd := exec.Command(selfCopy, "update")
-	cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig)
-	cmd.Stdout = up.Stderr
-	cmd.Stderr = up.Stderr
-	cmd.Stdin = os.Stdin
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	// Once it's started, exit ourselves, so the binary is free
-	// to be replaced.
-	os.Exit(0)
-	panic("unreachable")
-}
-
-func (up *Updater) installMSI(msi string) error {
-	var err error
-	for tries := 0; tries < 2; tries++ {
-		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/norestart", "/qn")
-		cmd.Dir = filepath.Dir(msi)
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		if err == nil {
-			break
-		}
-		up.Logf("Install attempt failed: %v", err)
-		uninstallVersion := up.currentVersion
-		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
-			uninstallVersion = v
-		}
-		// Assume it's a downgrade, which msiexec won't permit. Uninstall our current version first.
-		up.Logf("Uninstalling current version %q for downgrade...", uninstallVersion)
-		cmd = exec.Command("msiexec.exe", "/x", msiUUIDForVersion(uninstallVersion), "/norestart", "/qn")
-		cmd.Stdout = up.Stdout
-		cmd.Stderr = up.Stderr
-		cmd.Stdin = os.Stdin
-		err = cmd.Run()
-		up.Logf("msiexec uninstall: %v", err)
-	}
-	return err
-}
-
-func msiUUIDForVersion(ver string) string {
-	arch := runtime.GOARCH
-	if arch == "386" {
-		arch = "x86"
-	}
-	track, err := versionToTrack(ver)
-	if err != nil {
-		track = UnstableTrack
-	}
-	msiURL := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", track, ver, arch)
-	return "{" + strings.ToUpper(uuid.NewSHA1(uuid.NameSpaceURL, []byte(msiURL)).String()) + "}"
-}
-
-func (up *Updater) switchOutputToFile() (io.Closer, error) {
-	var logFilePath string
-	exePath, err := os.Executable()
-	if err != nil {
-		logFilePath = filepath.Join(os.TempDir(), "tailscale-updater.log")
-	} else {
-		logFilePath = strings.TrimSuffix(exePath, ".exe") + ".log"
-	}
-
-	up.Logf("writing update output to %q", logFilePath)
-	logFile, err := os.Create(logFilePath)
-	if err != nil {
-		return nil, err
-	}
-
-	up.Logf = func(m string, args ...any) {
-		fmt.Fprintf(logFile, m+"\n", args...)
-	}
-	up.Stdout = logFile
-	up.Stderr = logFile
-	return logFile, nil
-}

cmd/containerboot/forwarding.go

@@ -1,262 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"net"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"tailscale.com/util/linuxfw"
-)
-
-// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
-	var (
-		v4Forwarding, v6Forwarding bool
-	)
-	if clusterProxyTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid cluster destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	if tailnetTargetIP != "" {
-		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
-		if err != nil {
-			return fmt.Errorf("invalid tailnet destination IP: %v", err)
-		}
-		if proxyIP.Is4() {
-			v4Forwarding = true
-		} else {
-			v6Forwarding = true
-		}
-	}
-	// Currently we only proxy traffic to the IPv4 address of the tailnet
-	// target.
-	if tailnetTargetFQDN != "" {
-		v4Forwarding = true
-	}
-	if routes != nil && *routes != "" {
-		for _, route := range strings.Split(*routes, ",") {
-			cidr, err := netip.ParsePrefix(route)
-			if err != nil {
-				return fmt.Errorf("invalid subnet route: %v", err)
-			}
-			if cidr.Addr().Is4() {
-				v4Forwarding = true
-			} else {
-				v6Forwarding = true
-			}
-		}
-	}
-	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
-}
-
-func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
-	var paths []string
-	if v4Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
-	}
-	if v6Forwarding {
-		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
-	}
-
-	// In some common configurations (e.g. default docker,
-	// kubernetes), the container environment denies write access to
-	// most sysctls, including IP forwarding controls. Check the
-	// sysctl values before trying to change them, so that we
-	// gracefully do nothing if the container's already been set up
-	// properly by e.g. a k8s initContainer.
-	for _, path := range paths {
-		bs, err := os.ReadFile(path)
-		if err != nil {
-			return fmt.Errorf("reading %q: %w", path, err)
-		}
-		if v := strings.TrimSpace(string(bs)); v != "1" {
-			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
-				return fmt.Errorf("enabling %q: %w", path, err)
-			}
-		}
-	}
-	return nil
-}
-
-func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.EnsureSNATForDst(local, dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing egress proxy rules: %w", err)
-	}
-	return nil
-}
-
-// installTSForwardingRuleForDestination accepts a destination address and a
-// list of node's tailnet addresses, sets up rules to forward traffic for
-// destination to the tailnet IP matching the destination IP family.
-// Destination can be Pod IP of this node.
-func installTSForwardingRuleForDestination(_ context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstFilter)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
-	}
-	if err := nfr.AddDNATRule(dst, local); err != nil {
-		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	dst, err := netip.ParseAddr(dstStr)
-	if err != nil {
-		return err
-	}
-	var local netip.Addr
-	proxyHasIPv4Address := false
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
-			continue
-		}
-		if pfx.Addr().Is4() {
-			proxyHasIPv4Address = true
-		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr()
-		break
-	}
-	if proxyHasIPv4Address && dst.Is6() {
-		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
-	}
-	if !local.IsValid() {
-		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
-	}
-	if err := nfr.AddDNATRule(local, dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-		return fmt.Errorf("installing ingress proxy rules: %w", err)
-	}
-	return nil
-}
-
-func installIngressForwardingRuleForDNSTarget(_ context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
-	var (
-		tsv4       netip.Addr
-		tsv6       netip.Addr
-		v4Backends []netip.Addr
-		v6Backends []netip.Addr
-	)
-	for _, pfx := range tsIPs {
-		if pfx.IsSingleIP() && pfx.Addr().Is4() {
-			tsv4 = pfx.Addr()
-			continue
-		}
-		if pfx.IsSingleIP() && pfx.Addr().Is6() {
-			tsv6 = pfx.Addr()
-			continue
-		}
-	}
-	// TODO: log if more than one backend address is found and firewall is
-	// in nftables mode that only the first IP will be used.
-	for _, ip := range backendAddrs {
-		if ip.To4() != nil {
-			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
-		}
-		if ip.To16() != nil {
-			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
-		}
-	}
-
-	// Enable IP forwarding here as opposed to at the start of containerboot
-	// as the IPv4/IPv6 requirements might have changed.
-	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
-	// enabled by an init container, so in practice enabling forwarding here
-	// is only needed if this proxy has been configured by manually setting
-	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
-	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
-		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
-	}
-
-	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
-		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
-			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
-		}
-		// The backend might advertize MSS higher than that of the
-		// tailscale interfaces. Clamp MSS of packets going out via
-		// tailscale0 interface to its MTU to prevent broken connections
-		// in environments where path MTU discovery is not working.
-		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
-			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
-		}
-		return nil
-	}
-
-	if len(v4Backends) != 0 {
-		if !tsv4.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
-		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
-			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
-		}
-	}
-	if len(v6Backends) != 0 && !tsv6.IsValid() {
-		if !tsv6.IsValid() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
-		} else if !nfr.HasIPV6NAT() {
-			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
-		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
-			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
-		}
-	}
-	return nil
-}

cmd/containerboot/healthz.go

@@ -1,51 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"log"
-	"net"
-	"net/http"
-	"sync"
-)
-
-// healthz is a simple health check server, if enabled it returns 200 OK if
-// this tailscale node currently has at least one tailnet IP address else
-// returns 503.
-type healthz struct {
-	sync.Mutex
-	hasAddrs bool
-}
-
-func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	h.Lock()
-	defer h.Unlock()
-	if h.hasAddrs {
-		w.Write([]byte("ok"))
-	} else {
-		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
-	}
-}
-
-// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
-// provided address. A containerized tailscale instance is considered healthy if
-// it has at least one tailnet IP address.
-func runHealthz(addr string, h *healthz) {
-	lis, err := net.Listen("tcp", addr)
-	if err != nil {
-		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
-	}
-	mux := http.NewServeMux()
-	mux.Handle("/healthz", h)
-	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
-	hs := &http.Server{Handler: mux}
-
-	go func() {
-		if err := hs.Serve(lis); err != nil {
-			log.Fatalf("failed running health endpoint: %v", err)
-		}
-	}()
-}

cmd/containerboot/kube.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -61,7 +62,7 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 			Path: "/data/authkey",
 		},
 	}
-	if err := kc.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
+	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
 		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
@@ -74,6 +75,56 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 
 var kc kubeclient.Client
 
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kubeclient.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kubeclient.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
+
 func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
@@ -81,7 +132,7 @@ func initKubeClient(root string) {
 		kubeclient.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kubeclient.New("tailscale-container")
+	kc, err = kubeclient.New()
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/main.go

@@ -92,27 +92,36 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
 	"log"
 	"math"
 	"net"
+	"net/http"
 	"net/netip"
 	"os"
+	"os/exec"
 	"os/signal"
+	"path"
 	"path/filepath"
+	"reflect"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"golang.org/x/sys/unix"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/conffile"
 	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -131,9 +140,35 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) {
 func main() {
 	log.SetPrefix("boot: ")
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
+	cfg := &settings{
+		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
+		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
+		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
+		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
+		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
+		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
+		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
+		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
+		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
+		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
+		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
+		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
+		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
+		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
+		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
+		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
+		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
+		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
+		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
+		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
+		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
+		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
+		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
+		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
+	}
 
-	cfg, err := configFromEnv()
-	if err != nil {
+	if err := cfg.validate(); err != nil {
 		log.Fatalf("invalid configuration: %v", err)
 	}
 
@@ -248,8 +283,10 @@ authLoop:
 			switch *n.State {
 			case ipn.NeedsLogin:
 				if isOneStepConfig(cfg) {
-					// This could happen if this is the first time tailscaled was run for this
-					// device and the auth key was not passed via the configfile.
+					// This could happen if this is the
+					// first time tailscaled was run for
+					// this device and the auth key was not
+					// passed via the configfile.
 					log.Fatalf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.")
 				}
 				if err := authTailscale(); err != nil {
@@ -347,9 +384,6 @@ authLoop:
 			}
 		})
 	)
-	// egressSvcsErrorChan will get an error sent to it if this containerboot instance is configured to expose 1+
-	// egress services in HA mode and errored.
-	var egressSvcsErrorChan = make(chan error)
 	defer t.Stop()
 	// resetTimer resets timer for when to next attempt to resolve the DNS
 	// name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The
@@ -375,7 +409,6 @@ authLoop:
 		failedResolveAttempts++
 	}
 
-	var egressSvcsNotify chan ipn.Notify
 	notifyChan := make(chan ipn.Notify)
 	errChan := make(chan error)
 	go func() {
@@ -453,11 +486,7 @@ runLoop:
 					egressAddrs = node.Addresses().AsSlice()
 					newCurentEgressIPs = deephash.Hash(&egressAddrs)
 					egressIPsHaveChanged = newCurentEgressIPs != currentEgressIPs
-					// The firewall rules get (re-)installed:
-					// - on startup
-					// - when the tailnet IPs of the tailnet target have changed
-					// - when the tailnet IPs of this node have changed
-					if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 {
+					if egressIPsHaveChanged && len(egressAddrs) != 0 {
 						var rulesInstalled bool
 						for _, egressAddr := range egressAddrs {
 							ea := egressAddr.Addr()
@@ -554,50 +583,31 @@ runLoop:
 					h.Unlock()
 					healthzRunner()
 				}
-				if egressSvcsNotify != nil {
-					egressSvcsNotify <- n
-				}
 			}
 			if !startupTasksDone {
-				// For containerboot instances that act as TCP proxies (proxying traffic to an endpoint
-				// passed via one of the env vars that containerboot reads) and store state in a
-				// Kubernetes Secret, we consider startup tasks done at the point when device info has
-				// been successfully stored to state Secret. For all other containerboot instances, if
-				// we just get to this point the startup tasks can be considered done.
+				// For containerboot instances that act as TCP
+				// proxies (proxying traffic to an endpoint
+				// passed via one of the env vars that
+				// containerbot reads) and store state in a
+				// Kubernetes Secret, we consider startup tasks
+				// done at the point when device info has been
+				// successfully stored to state Secret.
+				// For all other containerboot instances, if we
+				// just get to this point the startup tasks can
+				// be considered done.
 				if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) {
 					// This log message is used in tests to detect when all
 					// post-auth configuration is done.
 					log.Println("Startup complete, waiting for shutdown signal")
 					startupTasksDone = true
 
-					// Configure egress proxy. Egress proxy will set up firewall rules to proxy
-					// traffic to tailnet targets configured in the provided configuration file. It
-					// will then continuously monitor the config file and netmap updates and
-					// reconfigure the firewall rules as needed. If any of its operations fail, it
-					// will crash this node.
-					if cfg.EgressSvcsCfgPath != "" {
-						log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressSvcsCfgPath)
-						egressSvcsNotify = make(chan ipn.Notify)
-						ep := egressProxy{
-							cfgPath:      cfg.EgressSvcsCfgPath,
-							nfr:          nfr,
-							kc:           kc,
-							stateSecret:  cfg.KubeSecret,
-							netmapChan:   egressSvcsNotify,
-							podIPv4:      cfg.PodIPv4,
-							tailnetAddrs: addrs,
-						}
-						go func() {
-							if err := ep.run(ctx, n); err != nil {
-								egressSvcsErrorChan <- err
-							}
-						}()
-					}
-
-					// Wait on tailscaled process. It won't be cleaned up by default when the
-					// container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the
-					// reaper by a running cmd.Wait in a goroutine immediately after starting
-					// tailscaled?
+					// Wait on tailscaled process. It won't
+					// be cleaned up by default when the
+					// container exits as it is not PID1.
+					// TODO (irbekrm): perhaps we can
+					// replace the reaper by a running
+					// cmd.Wait in a goroutine immediately
+					// after starting tailscaled?
 					reaper := func() {
 						defer wg.Done()
 						for {
@@ -635,13 +645,226 @@ runLoop:
 			}
 			backendAddrs = newBackendAddrs
 			resetTimer(false)
-		case e := <-egressSvcsErrorChan:
-			log.Fatalf("egress proxy failed: %v", e)
 		}
 	}
 	wg.Wait()
 }
 
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+	if certDomainAtomic == nil {
+		panic("cd must not be nil")
+	}
+	var tickChan <-chan time.Time
+	var eventChan <-chan fsnotify.Event
+	if w, err := fsnotify.NewWatcher(); err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(filepath.Dir(path)); err != nil {
+			log.Fatalf("failed to add fsnotify watch: %v", err)
+		}
+		eventChan = w.Events
+	}
+
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-eventChan:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		if certDomain == "" {
+			continue
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("failed to read serve config: %v", err)
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		log.Printf("Applying serve config")
+		if err := lc.SetServeConfig(ctx, sc); err != nil {
+			log.Fatalf("failed to set serve config: %v", err)
+		}
+		prevServeConfig = sc
+	}
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}
+
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
+	args := tailscaledArgs(cfg)
+	// tailscaled runs without context, since it needs to persist
+	// beyond the startup timeout in ctx.
+	cmd := exec.Command("tailscaled", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	log.Printf("Starting tailscaled")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
+	}
+
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
+	log.Printf("Waiting for tailscaled socket")
+	for {
+		if ctx.Err() != nil {
+			log.Fatalf("Timed out waiting for tailscaled socket")
+		}
+		_, err := os.Stat(cfg.Socket)
+		if errors.Is(err, fs.ErrNotExist) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		} else if err != nil {
+			log.Fatalf("Waiting for tailscaled socket: %v", err)
+		}
+		break
+	}
+
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process, nil
+}
+
+// tailscaledArgs uses cfg to construct the argv for tailscaled.
+func tailscaledArgs(cfg *settings) []string {
+	args := []string{"--socket=" + cfg.Socket}
+	switch {
+	case cfg.InKubernetes && cfg.KubeSecret != "":
+		args = append(args, "--state=kube:"+cfg.KubeSecret)
+		if cfg.StateDir == "" {
+			cfg.StateDir = "/tmp"
+		}
+		fallthrough
+	case cfg.StateDir != "":
+		args = append(args, "--statedir="+cfg.StateDir)
+	default:
+		args = append(args, "--state=mem:", "--statedir=/tmp")
+	}
+
+	if cfg.UserspaceMode {
+		args = append(args, "--tun=userspace-networking")
+	} else if err := ensureTunFile(cfg.Root); err != nil {
+		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
+	}
+
+	if cfg.SOCKSProxyAddr != "" {
+		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
+	}
+	if cfg.HTTPProxyAddr != "" {
+		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
+	}
+	if cfg.TailscaledConfigFilePath != "" {
+		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
+	}
+	if cfg.DaemonExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
+	}
+	return args
+}
+
+// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
+// if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleUp(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "up"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale up'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale up failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state and only if TS_AUTH_ONCE is set.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
+	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
+		args = append(args, "--accept-dns=true")
+	} else {
+		args = append(args, "--accept-dns=false")
+	}
+	// --advertise-routes can be passed an empty string to configure a
+	// device (that might have previously advertised subnet routes) to not
+	// advertise any routes. Respect an empty string passed by a user and
+	// use it to explicitly unset the routes.
+	if cfg.Routes != nil {
+		args = append(args, "--advertise-routes="+*cfg.Routes)
+	}
+	if cfg.Hostname != "" {
+		args = append(args, "--hostname="+cfg.Hostname)
+	}
+	log.Printf("Running 'tailscale set'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale set failed: %v", err)
+	}
+	return nil
+}
+
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
 func ensureTunFile(root string) error {
@@ -661,6 +884,350 @@ func ensureTunFile(root string) error {
 	return nil
 }
 
+// ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
+func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTargetFQDN string, routes *string) error {
+	var (
+		v4Forwarding, v6Forwarding bool
+	)
+	if clusterProxyTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetIP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetIP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	// Currently we only proxy traffic to the IPv4 address of the tailnet
+	// target.
+	if tailnetTargetFQDN != "" {
+		v4Forwarding = true
+	}
+	if routes != nil && *routes != "" {
+		for _, route := range strings.Split(*routes, ",") {
+			cidr, err := netip.ParsePrefix(route)
+			if err != nil {
+				return fmt.Errorf("invalid subnet route: %v", err)
+			}
+			if cidr.Addr().Is4() {
+				v4Forwarding = true
+			} else {
+				v6Forwarding = true
+			}
+		}
+	}
+	return enableIPForwarding(v4Forwarding, v6Forwarding, root)
+}
+
+func enableIPForwarding(v4Forwarding, v6Forwarding bool, root string) error {
+	var paths []string
+	if v4Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
+	}
+	if v6Forwarding {
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
+	}
+
+	// In some common configurations (e.g. default docker,
+	// kubernetes), the container environment denies write access to
+	// most sysctls, including IP forwarding controls. Check the
+	// sysctl values before trying to change them, so that we
+	// gracefully do nothing if the container's already been set up
+	// properly by e.g. a k8s initContainer.
+	for _, path := range paths {
+		bs, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %q: %w", path, err)
+		}
+		if v := strings.TrimSpace(string(bs)); v != "1" {
+			if err := os.WriteFile(path, []byte("1"), 0644); err != nil {
+				return fmt.Errorf("enabling %q: %w", path, err)
+			}
+		}
+	}
+	return nil
+}
+
+func installEgressForwardingRule(_ context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.DNATNonTailscaleTraffic("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.AddSNATRuleForDst(local, dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing egress proxy rules: %w", err)
+	}
+	return nil
+}
+
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	proxyHasIPv4Address := false
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	if err := nfr.AddDNATRule(local, dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+		return fmt.Errorf("installing ingress proxy rules: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRuleForDNSTarget(ctx context.Context, backendAddrs []net.IP, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	var (
+		tsv4       netip.Addr
+		tsv6       netip.Addr
+		v4Backends []netip.Addr
+		v6Backends []netip.Addr
+	)
+	for _, pfx := range tsIPs {
+		if pfx.IsSingleIP() && pfx.Addr().Is4() {
+			tsv4 = pfx.Addr()
+			continue
+		}
+		if pfx.IsSingleIP() && pfx.Addr().Is6() {
+			tsv6 = pfx.Addr()
+			continue
+		}
+	}
+	// TODO: log if more than one backend address is found and firewall is
+	// in nftables mode that only the first IP will be used.
+	for _, ip := range backendAddrs {
+		if ip.To4() != nil {
+			v4Backends = append(v4Backends, netip.AddrFrom4([4]byte(ip.To4())))
+		}
+		if ip.To16() != nil {
+			v6Backends = append(v6Backends, netip.AddrFrom16([16]byte(ip.To16())))
+		}
+	}
+
+	// Enable IP forwarding here as opposed to at the start of containerboot
+	// as the IPv4/IPv6 requirements might have changed.
+	// For Kubernetes operator proxies, forwarding for both IPv4 and IPv6 is
+	// enabled by an init container, so in practice enabling forwarding here
+	// is only needed if this proxy has been configured by manually setting
+	// TS_EXPERIMENTAL_DEST_DNS_NAME env var for a containerboot instance.
+	if err := enableIPForwarding(len(v4Backends) != 0, len(v6Backends) != 0, ""); err != nil {
+		log.Printf("[unexpected] failed to ensure IP forwarding: %v", err)
+	}
+
+	updateFirewall := func(dst netip.Addr, backendTargets []netip.Addr) error {
+		if err := nfr.DNATWithLoadBalancer(dst, backendTargets); err != nil {
+			return fmt.Errorf("installing DNAT rules for ingress backends %+#v: %w", backendTargets, err)
+		}
+		// The backend might advertize MSS higher than that of the
+		// tailscale interfaces. Clamp MSS of packets going out via
+		// tailscale0 interface to its MTU to prevent broken connections
+		// in environments where path MTU discovery is not working.
+		if err := nfr.ClampMSSToPMTU("tailscale0", dst); err != nil {
+			return fmt.Errorf("adding rule to clamp traffic via tailscale0: %v", err)
+		}
+		return nil
+	}
+
+	if len(v4Backends) != 0 {
+		if !tsv4.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv4 address, but this node's Tailscale IPs do not contain a valid IPv4 address: %v", backendAddrs, tsIPs)
+		} else if err := updateFirewall(tsv4, v4Backends); err != nil {
+			return fmt.Errorf("Installing IPv4 firewall rules: %w", err)
+		}
+	}
+	if len(v6Backends) != 0 && !tsv6.IsValid() {
+		if !tsv6.IsValid() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but this node's Tailscale IPs do not contain a valid IPv6 address: %v", backendAddrs, tsIPs)
+		} else if !nfr.HasIPV6NAT() {
+			log.Printf("backend targets %v contain at least one IPv6 address, but the chosen firewall mode does not support IPv6 NAT", backendAddrs)
+		} else if err := updateFirewall(tsv6, v6Backends); err != nil {
+			return fmt.Errorf("Installing IPv6 firewall rules: %w", err)
+		}
+	}
+	return nil
+}
+
+// settings is all the configuration for containerboot.
+type settings struct {
+	AuthKey  string
+	Hostname string
+	Routes   *string
+	// ProxyTargetIP is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTargetIP string
+	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
+	// incoming Tailscale traffic should be proxied.
+	ProxyTargetDNSName string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. This is typically a
+	// Tailscale IP.
+	TailnetTargetIP string
+	// TailnetTargetFQDN is an MagicDNS name to which all incoming
+	// non-Tailscale traffic should be proxied. This must be a full Tailnet
+	// node FQDN.
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	PodIP               string
+	HealthCheckAddrPort string
+}
+
+func (s *settings) validate() error {
+	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
+		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
+		}
+	}
+	if s.ProxyTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
+	}
+	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
+		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
+	}
+	if s.TailnetTargetIP != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
+		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
+	}
+	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
+		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
+	}
+	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
+	if s.HealthCheckAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+		}
+	}
+	return nil
+}
+
 func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	// TODO (irbekrm): look at using recursive.Resolver instead to resolve
 	// the DNS names as well as retrieve TTLs. It looks though that this
@@ -683,6 +1250,57 @@ func resolveDNS(ctx context.Context, name string) ([]net.IP, error) {
 	return append(ip4s, ip6s...), nil
 }
 
+// defaultEnv returns the value of the given envvar name, or defVal if
+// unset.
+func defaultEnv(name, defVal string) string {
+	if v, ok := os.LookupEnv(name); ok {
+		return v
+	}
+	return defVal
+}
+
+// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being set to empty string vs unset.
+func defaultEnvStringPointer(name string) *string {
+	if v, ok := os.LookupEnv(name); ok {
+		return &v
+	}
+	return nil
+}
+
+// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
+// returns nil. This is useful in cases where we need to distinguish between a
+// variable being explicitly set to false vs unset.
+func defaultEnvBoolPointer(name string) *bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return nil
+	}
+	return &ret
+}
+
+func defaultEnvs(names []string, defVal string) string {
+	for _, name := range names {
+		if v, ok := os.LookupEnv(name); ok {
+			return v
+		}
+	}
+	return defVal
+}
+
+// defaultBool returns the boolean value of the given envvar name, or
+// defVal if unset or not a bool.
+func defaultBool(name string, defVal bool) bool {
+	v := os.Getenv(name)
+	ret, err := strconv.ParseBool(v)
+	if err != nil {
+		return defVal
+	}
+	return ret
+}
+
 // contextWithExitSignalWatch watches for SIGTERM/SIGINT signals. It returns a
 // context that gets cancelled when a signal is received and a cancel function
 // that can be called to free the resources when the watch should be stopped.
@@ -705,6 +1323,43 @@ func contextWithExitSignalWatch() (context.Context, func()) {
 	return ctx, f
 }
 
+// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
+// in two steps and login should only happen once.
+// Step 1: run 'tailscaled'
+// Step 2):
+// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
+// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
+func isTwoStepConfigAuthOnce(cfg *settings) bool {
+	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
+// in two steps and we should log in every time it starts.
+// Step 1: run 'tailscaled'
+// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
+func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
+	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
+}
+
+// isOneStepConfig returns true if the Tailscale node should always be ran and
+// configured in a single step by running 'tailscaled <config opts>'
+func isOneStepConfig(cfg *settings) bool {
+	return cfg.TailscaledConfigFilePath != ""
+}
+
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
 // tailscaledConfigFilePath returns the path to the tailscaled config file that
 // should be used for the current capability version. It is determined by the
 // TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a
@@ -730,6 +1385,7 @@ func tailscaledConfigFilePath() string {
 		}
 		cv, err := kubeutils.CapVerFromFileName(e.Name())
 		if err != nil {
+			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
 			continue
 		}
 		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
@@ -737,9 +1393,46 @@ func tailscaledConfigFilePath() string {
 		}
 	}
 	if maxCompatVer == -1 {
-		log.Fatalf("no tailscaled config file found in %q for current capability version %d", dir, tailcfg.CurrentCapabilityVersion)
+		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
 	}
-	filePath := filepath.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
-	log.Printf("Using tailscaled config file %q to match current capability version %d", filePath, tailcfg.CurrentCapabilityVersion)
-	return filePath
+	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
+	return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer))
+}
+
+// healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type healthz struct {
+	sync.Mutex
+	hasAddrs bool
+}
+
+func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+	if h.hasAddrs {
+		w.Write([]byte("ok"))
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
+	}
+}
+
+// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
+// provided address. A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func runHealthz(addr string, h *healthz) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", h)
+	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
+	hs := &http.Server{Handler: mux}
+
+	go func() {
+		if err := hs.Serve(lis); err != nil {
+			log.Fatalf("failed running health endpoint: %v", err)
+		}
+	}()
 }

cmd/containerboot/serve.go

@@ -1,96 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"log"
-	"os"
-	"path/filepath"
-	"reflect"
-	"sync/atomic"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"tailscale.com/client/tailscale"
-	"tailscale.com/ipn"
-)
-
-// watchServeConfigChanges watches path for changes, and when it sees one, reads
-// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
-// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
-// is written to when the certDomain changes, causing the serve config to be
-// re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
-	if certDomainAtomic == nil {
-		panic("cd must not be nil")
-	}
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
-			log.Fatalf("failed to add fsnotify watch: %v", err)
-		}
-		eventChan = w.Events
-	}
-
-	var certDomain string
-	var prevServeConfig *ipn.ServeConfig
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-cdChanged:
-			certDomain = *certDomainAtomic.Load()
-		case <-tickChan:
-		case <-eventChan:
-			// We can't do any reasonable filtering on the event because of how
-			// k8s handles these mounts. So just re-read the file and apply it
-			// if it's changed.
-		}
-		if certDomain == "" {
-			continue
-		}
-		sc, err := readServeConfig(path, certDomain)
-		if err != nil {
-			log.Fatalf("failed to read serve config: %v", err)
-		}
-		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
-			continue
-		}
-		log.Printf("Applying serve config")
-		if err := lc.SetServeConfig(ctx, sc); err != nil {
-			log.Fatalf("failed to set serve config: %v", err)
-		}
-		prevServeConfig = sc
-	}
-}
-
-// readServeConfig reads the ipn.ServeConfig from path, replacing
-// ${TS_CERT_DOMAIN} with certDomain.
-func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
-	if path == "" {
-		return nil, nil
-	}
-	j, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
-	var sc ipn.ServeConfig
-	if err := json.Unmarshal(j, &sc); err != nil {
-		return nil, err
-	}
-	return &sc, nil
-}

cmd/containerboot/services.go

@@ -1,571 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"tailscale.com/ipn"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/kube/kubeclient"
-	"tailscale.com/tailcfg"
-	"tailscale.com/util/linuxfw"
-	"tailscale.com/util/mak"
-)
-
-const tailscaleTunInterface = "tailscale0"
-
-// This file contains functionality to run containerboot as a proxy that can
-// route cluster traffic to one or more tailnet targets, based on portmapping
-// rules read from a configfile. Currently (9/2024) this is only used for the
-// Kubernetes operator egress proxies.
-
-// egressProxy knows how to configure firewall rules to route cluster traffic to
-// one or more tailnet services.
-type egressProxy struct {
-	cfgPath string // path to egress service config file
-
-	nfr linuxfw.NetfilterRunner // never nil
-
-	kc          kubeclient.Client // never nil
-	stateSecret string            // name of the kube state Secret
-
-	netmapChan chan ipn.Notify // chan to receive netmap updates on
-
-	podIPv4 string // never empty string, currently only IPv4 is supported
-
-	// tailnetFQDNs is the egress service FQDN to tailnet IP mappings that
-	// were last used to configure firewall rules for this proxy.
-	// TODO(irbekrm): target addresses are also stored in the state Secret.
-	// Evaluate whether we should retrieve them from there and not store in
-	// memory at all.
-	targetFQDNs map[string][]netip.Prefix
-
-	// used to configure firewall rules.
-	tailnetAddrs []netip.Prefix
-}
-
-// run configures egress proxy firewall rules and ensures that the firewall rules are reconfigured when:
-// - the mounted egress config has changed
-// - the proxy's tailnet IP addresses have changed
-// - tailnet IPs have changed for any backend targets specified by tailnet FQDN
-func (ep *egressProxy) run(ctx context.Context, n ipn.Notify) error {
-	var tickChan <-chan time.Time
-	var eventChan <-chan fsnotify.Event
-	// TODO (irbekrm): take a look if this can be pulled into a single func
-	// shared with serve config loader.
-	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		tickChan = ticker.C
-	} else {
-		defer w.Close()
-		if err := w.Add(filepath.Dir(ep.cfgPath)); err != nil {
-			return fmt.Errorf("failed to add fsnotify watch: %w", err)
-		}
-		eventChan = w.Events
-	}
-
-	if err := ep.sync(ctx, n); err != nil {
-		return err
-	}
-	for {
-		var err error
-		select {
-		case <-ctx.Done():
-			return nil
-		case <-tickChan:
-			err = ep.sync(ctx, n)
-		case <-eventChan:
-			log.Printf("config file change detected, ensuring firewall config is up to date...")
-			err = ep.sync(ctx, n)
-		case n = <-ep.netmapChan:
-			shouldResync := ep.shouldResync(n)
-			if shouldResync {
-				log.Printf("netmap change detected, ensuring firewall config is up to date...")
-				err = ep.sync(ctx, n)
-			}
-		}
-		if err != nil {
-			return fmt.Errorf("error syncing egress service config: %w", err)
-		}
-	}
-}
-
-// sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if
-// any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current
-// firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such
-// as failed firewall update
-func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error {
-	cfgs, err := ep.getConfigs()
-	if err != nil {
-		return fmt.Errorf("error retrieving egress service configs: %w", err)
-	}
-	status, err := ep.getStatus(ctx)
-	if err != nil {
-		return fmt.Errorf("error retrieving current egress proxy status: %w", err)
-	}
-	newStatus, err := ep.syncEgressConfigs(cfgs, status, n)
-	if err != nil {
-		return fmt.Errorf("error syncing egress service configs: %w", err)
-	}
-	if !servicesStatusIsEqual(newStatus, status) {
-		if err := ep.setStatus(ctx, newStatus, n); err != nil {
-			return fmt.Errorf("error setting egress proxy status: %w", err)
-		}
-	}
-	return nil
-}
-
-// addrsHaveChanged returns true if the provided netmap update contains tailnet address change for this proxy node.
-// Netmap must not be nil.
-func (ep *egressProxy) addrsHaveChanged(n ipn.Notify) bool {
-	return !reflect.DeepEqual(ep.tailnetAddrs, n.NetMap.SelfNode.Addresses())
-}
-
-// syncEgressConfigs adds and deletes firewall rules to match the desired
-// configuration. It uses the provided status to determine what is currently
-// applied and updates the status after a successful sync.
-func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *egressservices.Status, n ipn.Notify) (*egressservices.Status, error) {
-	if !(wantsServicesConfigured(cfgs) || hasServicesConfigured(status)) {
-		return nil, nil
-	}
-
-	// Delete unnecessary services.
-	if err := ep.deleteUnnecessaryServices(cfgs, status); err != nil {
-		return nil, fmt.Errorf("error deleting services: %w", err)
-
-	}
-	newStatus := &egressservices.Status{}
-	if !wantsServicesConfigured(cfgs) {
-		return newStatus, nil
-	}
-
-	// Add new services, update rules for any that have changed.
-	rulesPerSvcToAdd := make(map[string][]rule, 0)
-	rulesPerSvcToDelete := make(map[string][]rule, 0)
-	for svcName, cfg := range *cfgs {
-		tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, n)
-		if err != nil {
-			return nil, fmt.Errorf("error determining tailnet target IPs: %w", err)
-		}
-		rulesToAdd, rulesToDelete, err := updatesForCfg(svcName, cfg, status, tailnetTargetIPs)
-		if err != nil {
-			return nil, fmt.Errorf("error validating service changes: %v", err)
-		}
-		log.Printf("syncegressservices: looking at svc %s rulesToAdd %d rulesToDelete %d", svcName, len(rulesToAdd), len(rulesToDelete))
-		if len(rulesToAdd) != 0 {
-			mak.Set(&rulesPerSvcToAdd, svcName, rulesToAdd)
-		}
-		if len(rulesToDelete) != 0 {
-			mak.Set(&rulesPerSvcToDelete, svcName, rulesToDelete)
-		}
-		if len(rulesToAdd) != 0 || ep.addrsHaveChanged(n) {
-			// For each tailnet target, set up SNAT from the local tailnet device address of the matching
-			// family.
-			for _, t := range tailnetTargetIPs {
-				var local netip.Addr
-				for _, pfx := range n.NetMap.SelfNode.Addresses().All() {
-					if !pfx.IsSingleIP() {
-						continue
-					}
-					if pfx.Addr().Is4() != t.Is4() {
-						continue
-					}
-					local = pfx.Addr()
-					break
-				}
-				if !local.IsValid() {
-					return nil, fmt.Errorf("no valid local IP: %v", local)
-				}
-				if err := ep.nfr.EnsureSNATForDst(local, t); err != nil {
-					return nil, fmt.Errorf("error setting up SNAT rule: %w", err)
-				}
-			}
-		}
-		// Update the status. Status will be written back to the state Secret by the caller.
-		mak.Set(&newStatus.Services, svcName, &egressservices.ServiceStatus{TailnetTargetIPs: tailnetTargetIPs, TailnetTarget: cfg.TailnetTarget, Ports: cfg.Ports})
-	}
-
-	// Actually apply the firewall rules.
-	if err := ensureRulesAdded(rulesPerSvcToAdd, ep.nfr); err != nil {
-		return nil, fmt.Errorf("error adding rules: %w", err)
-	}
-	if err := ensureRulesDeleted(rulesPerSvcToDelete, ep.nfr); err != nil {
-		return nil, fmt.Errorf("error deleting rules: %w", err)
-	}
-
-	return newStatus, nil
-}
-
-// updatesForCfg calculates any rules that need to be added or deleted for an individucal egress service config.
-func updatesForCfg(svcName string, cfg egressservices.Config, status *egressservices.Status, tailnetTargetIPs []netip.Addr) ([]rule, []rule, error) {
-	rulesToAdd := make([]rule, 0)
-	rulesToDelete := make([]rule, 0)
-	currentConfig, ok := lookupCurrentConfig(svcName, status)
-
-	// If no rules for service are present yet, add them all.
-	if !ok {
-		for _, t := range tailnetTargetIPs {
-			for ports := range cfg.Ports {
-				log.Printf("syncegressservices: svc %s adding port %v", svcName, ports)
-				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: t})
-			}
-		}
-		return rulesToAdd, rulesToDelete, nil
-	}
-
-	// If there are no backend targets available, delete any currently configured rules.
-	if len(tailnetTargetIPs) == 0 {
-		log.Printf("tailnet target for egress service %s does not have any backend addresses, deleting all rules", svcName)
-		for _, ip := range currentConfig.TailnetTargetIPs {
-			for ports := range currentConfig.Ports {
-				rulesToDelete = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
-			}
-		}
-		return rulesToAdd, rulesToDelete, nil
-	}
-
-	// If there are rules present for backend targets that no longer match, delete them.
-	for _, ip := range currentConfig.TailnetTargetIPs {
-		var found bool
-		for _, wantsIP := range tailnetTargetIPs {
-			if reflect.DeepEqual(ip, wantsIP) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			for ports := range currentConfig.Ports {
-				rulesToDelete = append(rulesToDelete, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
-			}
-		}
-	}
-
-	// Sync rules for the currently wanted backend targets.
-	for _, ip := range tailnetTargetIPs {
-
-		// If the backend target is not yet present in status, add all rules.
-		var found bool
-		for _, gotIP := range currentConfig.TailnetTargetIPs {
-			if reflect.DeepEqual(ip, gotIP) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			for ports := range cfg.Ports {
-				rulesToAdd = append(rulesToAdd, rule{tailnetPort: ports.TargetPort, containerPort: ports.MatchPort, protocol: ports.Protocol, tailnetIP: ip})
-			}
-			continue
-		}
-
-		// If the backend target is present in status, check that the
-		// currently applied rules are up to date.
-
-		// Delete any current portmappings that are no longer present in config.
-		for port := range currentConfig.Ports {
-			if _, ok := cfg.Ports[port]; ok {
-				continue
-			}
-			rulesToDelete = append(rulesToDelete, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
-		}
-
-		// Add any new portmappings.
-		for port := range cfg.Ports {
-			if _, ok := currentConfig.Ports[port]; ok {
-				continue
-			}
-			rulesToAdd = append(rulesToAdd, rule{tailnetPort: port.TargetPort, containerPort: port.MatchPort, protocol: port.Protocol, tailnetIP: ip})
-		}
-	}
-	return rulesToAdd, rulesToDelete, nil
-}
-
-// deleteUnneccessaryServices ensure that any services found on status, but not
-// present in config are deleted.
-func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, status *egressservices.Status) error {
-	if !hasServicesConfigured(status) {
-		return nil
-	}
-	if !wantsServicesConfigured(cfgs) {
-		for svcName, svc := range status.Services {
-			log.Printf("service %s is no longer required, deleting", svcName)
-			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
-				return fmt.Errorf("error deleting service %s: %w", svcName, err)
-			}
-		}
-		return nil
-	}
-
-	for svcName, svc := range status.Services {
-		if _, ok := (*cfgs)[svcName]; !ok {
-			log.Printf("service %s is no longer required, deleting", svcName)
-			if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil {
-				return fmt.Errorf("error deleting service %s: %w", svcName, err)
-			}
-			// TODO (irbekrm): also delete the SNAT rule here
-		}
-	}
-	return nil
-}
-
-// getConfigs gets the mounted egress service configuration.
-func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) {
-	j, err := os.ReadFile(ep.cfgPath)
-	if os.IsNotExist(err) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	if len(j) == 0 || string(j) == "" {
-		return nil, nil
-	}
-	cfg := &egressservices.Configs{}
-	if err := json.Unmarshal(j, &cfg); err != nil {
-		return nil, err
-	}
-	return cfg, nil
-}
-
-// getStatus gets the current status of the configured firewall. The current
-// status is stored in state Secret. Returns nil status if no status that
-// applies to the current proxy Pod was found. Uses the Pod IP to determine if a
-// status found in the state Secret applies to this proxy Pod.
-func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, error) {
-	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
-	if err != nil {
-		return nil, fmt.Errorf("error retrieving state secret: %w", err)
-	}
-	status := &egressservices.Status{}
-	raw, ok := secret.Data[egressservices.KeyEgressServices]
-	if !ok {
-		return nil, nil
-	}
-	if err := json.Unmarshal([]byte(raw), status); err != nil {
-		return nil, fmt.Errorf("error unmarshalling previous config: %w", err)
-	}
-	if reflect.DeepEqual(status.PodIPv4, ep.podIPv4) {
-		return status, nil
-	}
-	return nil, nil
-}
-
-// setStatus writes egress proxy's currently configured firewall to the state
-// Secret and updates proxy's tailnet addresses.
-func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, n ipn.Notify) error {
-	// Pod IP is used to determine if a stored status applies to THIS proxy Pod.
-	if status == nil {
-		status = &egressservices.Status{}
-	}
-	status.PodIPv4 = ep.podIPv4
-	secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
-	if err != nil {
-		return fmt.Errorf("error retrieving state Secret: %w", err)
-	}
-	bs, err := json.Marshal(status)
-	if err != nil {
-		return fmt.Errorf("error marshalling service config: %w", err)
-	}
-	secret.Data[egressservices.KeyEgressServices] = bs
-	patch := kubeclient.JSONPatch{
-		Op:    "replace",
-		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
-		Value: bs,
-	}
-	if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil {
-		return fmt.Errorf("error patching state Secret: %w", err)
-	}
-	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
-	return nil
-}
-
-// tailnetTargetIPsForSvc returns the tailnet IPs to which traffic for this
-// egress service should be proxied. The egress service can be configured by IP
-// or by FQDN. If it's configured by IP, just return that. If it's configured by
-// FQDN, resolve the FQDN and return the resolved IPs. It checks if the
-// netfilter runner supports IPv6 NAT and skips any IPv6 addresses if it
-// doesn't.
-func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.Notify) (addrs []netip.Addr, err error) {
-	if svc.TailnetTarget.IP != "" {
-		addr, err := netip.ParseAddr(svc.TailnetTarget.IP)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing tailnet target IP: %w", err)
-		}
-		if addr.Is6() && !ep.nfr.HasIPV6NAT() {
-			log.Printf("tailnet target is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode. This will probably not work.")
-			return addrs, nil
-		}
-		return []netip.Addr{addr}, nil
-	}
-
-	if svc.TailnetTarget.FQDN == "" {
-		return nil, errors.New("unexpected egress service config- neither tailnet target IP nor FQDN is set")
-	}
-	if n.NetMap == nil {
-		log.Printf("netmap is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN)
-		return addrs, nil
-	}
-	var (
-		node      tailcfg.NodeView
-		nodeFound bool
-	)
-	for _, nn := range n.NetMap.Peers {
-		if equalFQDNs(nn.Name(), svc.TailnetTarget.FQDN) {
-			node = nn
-			nodeFound = true
-			break
-		}
-	}
-	if nodeFound {
-		for _, addr := range node.Addresses().AsSlice() {
-			if addr.Addr().Is6() && !ep.nfr.HasIPV6NAT() {
-				log.Printf("tailnet target %v is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode, skipping.", addr.Addr().String())
-				continue
-			}
-			addrs = append(addrs, addr.Addr())
-		}
-		// Egress target endpoints configured via FQDN are stored, so
-		// that we can determine if a netmap update should trigger a
-		// resync.
-		mak.Set(&ep.targetFQDNs, svc.TailnetTarget.FQDN, node.Addresses().AsSlice())
-	}
-	return addrs, nil
-}
-
-// shouldResync parses netmap update and returns true if the update contains
-// changes for which the egress proxy's firewall should be reconfigured.
-func (ep *egressProxy) shouldResync(n ipn.Notify) bool {
-	if n.NetMap == nil {
-		return false
-	}
-
-	// If proxy's tailnet addresses have changed, resync.
-	if !reflect.DeepEqual(n.NetMap.SelfNode.Addresses().AsSlice(), ep.tailnetAddrs) {
-		log.Printf("node addresses have changed, trigger egress config resync")
-		ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()
-		return true
-	}
-
-	// If the IPs for any of the egress services configured via FQDN have
-	// changed, resync.
-	for fqdn, ips := range ep.targetFQDNs {
-		for _, nn := range n.NetMap.Peers {
-			if equalFQDNs(nn.Name(), fqdn) {
-				if !reflect.DeepEqual(ips, nn.Addresses().AsSlice()) {
-					log.Printf("backend addresses for egress target %q have changed old IPs %v, new IPs %v trigger egress config resync", nn.Name(), ips, nn.Addresses().AsSlice())
-				}
-				return true
-			}
-		}
-	}
-	return false
-}
-
-// ensureServiceDeleted ensures that any rules for an egress service are removed
-// from the firewall configuration.
-func ensureServiceDeleted(svcName string, svc *egressservices.ServiceStatus, nfr linuxfw.NetfilterRunner) error {
-
-	// Note that the portmap is needed for iptables based firewall only.
-	// Nftables group rules for a service in a chain, so there is no need to
-	// specify individual portmapping based rules.
-	pms := make([]linuxfw.PortMap, 0)
-	for pm := range svc.Ports {
-		pms = append(pms, linuxfw.PortMap{MatchPort: pm.MatchPort, TargetPort: pm.TargetPort, Protocol: pm.Protocol})
-	}
-
-	if err := nfr.DeleteSvc(svcName, tailscaleTunInterface, svc.TailnetTargetIPs, pms); err != nil {
-		return fmt.Errorf("error deleting service %s: %w", svcName, err)
-	}
-	return nil
-}
-
-// ensureRulesAdded ensures that all portmapping rules are added to the firewall
-// configuration. For any rules that already exist, calling this function is a
-// no-op. In case of nftables, a service consists of one or two (one per IP
-// family) chains that conain the portmapping rules for the service and the
-// chains as needed when this function is called.
-func ensureRulesAdded(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
-	for svc, rules := range rulesPerSvc {
-		for _, rule := range rules {
-			log.Printf("ensureRulesAdded svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
-			if err := nfr.EnsurePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
-				return fmt.Errorf("error ensuring rule: %w", err)
-			}
-		}
-	}
-	return nil
-}
-
-// ensureRulesDeleted ensures that the given rules are deleted from the firewall
-// configuration. For any rules that do not exist, calling this funcion is a
-// no-op.
-func ensureRulesDeleted(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error {
-	for svc, rules := range rulesPerSvc {
-		for _, rule := range rules {
-			log.Printf("ensureRulesDeleted svc %s tailnetTarget %s container port %d tailnet port %d protocol %s", svc, rule.tailnetIP, rule.containerPort, rule.tailnetPort, rule.protocol)
-			if err := nfr.DeletePortMapRuleForSvc(svc, tailscaleTunInterface, rule.tailnetIP, linuxfw.PortMap{MatchPort: rule.containerPort, TargetPort: rule.tailnetPort, Protocol: rule.protocol}); err != nil {
-				return fmt.Errorf("error deleting rule: %w", err)
-			}
-		}
-	}
-	return nil
-}
-
-func lookupCurrentConfig(svcName string, status *egressservices.Status) (*egressservices.ServiceStatus, bool) {
-	if status == nil || len(status.Services) == 0 {
-		return nil, false
-	}
-	c, ok := status.Services[svcName]
-	return c, ok
-}
-
-func equalFQDNs(s, s1 string) bool {
-	s, _ = strings.CutSuffix(s, ".")
-	s1, _ = strings.CutSuffix(s1, ".")
-	return strings.EqualFold(s, s1)
-}
-
-// rule contains configuration for an egress proxy firewall rule.
-type rule struct {
-	containerPort uint16     // port to match incoming traffic
-	tailnetPort   uint16     // tailnet service port
-	tailnetIP     netip.Addr // tailnet service IP
-	protocol      string
-}
-
-func wantsServicesConfigured(cfgs *egressservices.Configs) bool {
-	return cfgs != nil && len(*cfgs) != 0
-}
-
-func hasServicesConfigured(status *egressservices.Status) bool {
-	return status != nil && len(status.Services) != 0
-}
-
-func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
-	if st == nil && st1 == nil {
-		return true
-	}
-	if st == nil || st1 == nil {
-		return false
-	}
-	st.PodIPv4 = ""
-	st1.PodIPv4 = ""
-	return reflect.DeepEqual(*st, *st1)
-}

cmd/containerboot/services_test.go

@@ -1,175 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"net/netip"
-	"reflect"
-	"testing"
-
-	"tailscale.com/kube/egressservices"
-)
-
-func Test_updatesForSvc(t *testing.T) {
-	tailnetIPv4, tailnetIPv6 := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
-	tailnetIPv4_1, tailnetIPv6_1 := netip.MustParseAddr("100.88.88.88"), netip.MustParseAddr("fd7a:115c:a1e0::4101:512f")
-	ports := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {}}
-	ports1 := map[egressservices.PortMap]struct{}{{Protocol: "udp", MatchPort: 4004, TargetPort: 53}: {}}
-	ports2 := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {},
-		{Protocol: "tcp", MatchPort: 4005, TargetPort: 443}: {}}
-	fqdnSpec := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
-		Ports:         ports,
-	}
-	fqdnSpec1 := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
-		Ports:         ports1,
-	}
-	fqdnSpec2 := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-		Ports:         ports,
-	}
-	fqdnSpec3 := egressservices.Config{
-		TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-		Ports:         ports2,
-	}
-	r := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4}
-	r1 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6}
-	r2 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv4}
-	r3 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv6}
-	r4 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4_1}
-	r5 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6_1}
-	r6 := rule{containerPort: 4005, tailnetPort: 443, protocol: "tcp", tailnetIP: tailnetIPv4}
-
-	tests := []struct {
-		name              string
-		svcName           string
-		tailnetTargetIPs  []netip.Addr
-		podIP             string
-		spec              egressservices.Config
-		status            *egressservices.Status
-		wantRulesToAdd    []rule
-		wantRulesToDelete []rule
-	}{
-		{
-			name:              "add_fqdn_svc_that_does_not_yet_exist",
-			svcName:           "test",
-			tailnetTargetIPs:  []netip.Addr{tailnetIPv4, tailnetIPv6},
-			spec:              fqdnSpec,
-			status:            &egressservices.Status{},
-			wantRulesToAdd:    []rule{r, r1},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "fqdn_svc_already_exists",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-			spec:             fqdnSpec,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "fqdn_svc_already_exists_add_port_remove_port",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-			spec:             fqdnSpec1,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{r2, r3},
-			wantRulesToDelete: []rule{r, r1},
-		},
-		{
-			name:             "fqdn_svc_already_exists_change_fqdn_backend_ips",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4_1, tailnetIPv6_1},
-			spec:             fqdnSpec,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
-					TailnetTarget:    egressservices.TailnetTarget{FQDN: "test"},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{r4, r5},
-			wantRulesToDelete: []rule{r, r1},
-		},
-		{
-			name:              "add_ip_service",
-			svcName:           "test",
-			tailnetTargetIPs:  []netip.Addr{tailnetIPv4},
-			spec:              fqdnSpec2,
-			status:            &egressservices.Status{},
-			wantRulesToAdd:    []rule{r},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "add_ip_service_already_exists",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
-			spec:             fqdnSpec2,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
-					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "ip_service_add_port",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
-			spec:             fqdnSpec3,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
-					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-					Ports:            ports,
-				}}},
-			wantRulesToAdd:    []rule{r6},
-			wantRulesToDelete: []rule{},
-		},
-		{
-			name:             "ip_service_delete_port",
-			svcName:          "test",
-			tailnetTargetIPs: []netip.Addr{tailnetIPv4},
-			spec:             fqdnSpec,
-			status: &egressservices.Status{
-				Services: map[string]*egressservices.ServiceStatus{"test": {
-					TailnetTargetIPs: []netip.Addr{tailnetIPv4},
-					TailnetTarget:    egressservices.TailnetTarget{IP: tailnetIPv4.String()},
-					Ports:            ports2,
-				}}},
-			wantRulesToAdd:    []rule{},
-			wantRulesToDelete: []rule{r6},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			gotRulesToAdd, gotRulesToDelete, err := updatesForCfg(tt.svcName, tt.spec, tt.status, tt.tailnetTargetIPs)
-			if err != nil {
-				t.Errorf("updatesForSvc() unexpected error %v", err)
-				return
-			}
-			if !reflect.DeepEqual(gotRulesToAdd, tt.wantRulesToAdd) {
-				t.Errorf("updatesForSvc() got rulesToAdd = \n%v\n want rulesToAdd \n%v", gotRulesToAdd, tt.wantRulesToAdd)
-			}
-			if !reflect.DeepEqual(gotRulesToDelete, tt.wantRulesToDelete) {
-				t.Errorf("updatesForSvc() got rulesToDelete = \n%v\n want rulesToDelete \n%v", gotRulesToDelete, tt.wantRulesToDelete)
-			}
-		})
-	}
-}

cmd/containerboot/settings.go

@@ -1,324 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log"
-	"net/netip"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	"tailscale.com/ipn/conffile"
-	"tailscale.com/kube/kubeclient"
-)
-
-// settings is all the configuration for containerboot.
-type settings struct {
-	AuthKey  string
-	Hostname string
-	Routes   *string
-	// ProxyTargetIP is the destination IP to which all incoming
-	// Tailscale traffic should be proxied. If empty, no proxying
-	// is done. This is typically a locally reachable IP.
-	ProxyTargetIP string
-	// ProxyTargetDNSName is a DNS name to whose backing IP addresses all
-	// incoming Tailscale traffic should be proxied.
-	ProxyTargetDNSName string
-	// TailnetTargetIP is the destination IP to which all incoming
-	// non-Tailscale traffic should be proxied. This is typically a
-	// Tailscale IP.
-	TailnetTargetIP string
-	// TailnetTargetFQDN is an MagicDNS name to which all incoming
-	// non-Tailscale traffic should be proxied. This must be a full Tailnet
-	// node FQDN.
-	TailnetTargetFQDN             string
-	ServeConfigPath               string
-	DaemonExtraArgs               string
-	ExtraArgs                     string
-	InKubernetes                  bool
-	UserspaceMode                 bool
-	StateDir                      string
-	AcceptDNS                     *bool
-	KubeSecret                    string
-	SOCKSProxyAddr                string
-	HTTPProxyAddr                 string
-	Socket                        string
-	AuthOnce                      bool
-	Root                          string
-	KubernetesCanPatch            bool
-	TailscaledConfigFilePath      string
-	EnableForwardingOptimizations bool
-	// If set to true and, if this containerboot instance is a Kubernetes
-	// ingress proxy, set up rules to forward incoming cluster traffic to be
-	// forwarded to the ingress target in cluster.
-	AllowProxyingClusterTrafficViaIngress bool
-	// PodIP is the IP of the Pod if running in Kubernetes. This is used
-	// when setting up rules to proxy cluster traffic to cluster ingress
-	// target.
-	// Deprecated: use PodIPv4, PodIPv6 instead to support dual stack clusters
-	PodIP               string
-	PodIPv4             string
-	PodIPv6             string
-	HealthCheckAddrPort string
-	EgressSvcsCfgPath   string
-}
-
-func configFromEnv() (*settings, error) {
-	cfg := &settings{
-		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
-		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
-		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
-		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
-		ProxyTargetIP:                         defaultEnv("TS_DEST_IP", ""),
-		ProxyTargetDNSName:                    defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""),
-		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
-		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
-		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
-		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
-		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
-		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
-		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
-		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
-		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
-		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
-		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
-		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
-		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
-		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
-		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
-		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
-		PodIP:                                 defaultEnv("POD_IP", ""),
-		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
-		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
-		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
-	}
-	podIPs, ok := os.LookupEnv("POD_IPS")
-	if ok {
-		ips := strings.Split(podIPs, ",")
-		if len(ips) > 2 {
-			return nil, fmt.Errorf("POD_IPs can contain at most 2 IPs, got %d (%v)", len(ips), ips)
-		}
-		for _, ip := range ips {
-			parsed, err := netip.ParseAddr(ip)
-			if err != nil {
-				return nil, fmt.Errorf("error parsing IP address %s: %w", ip, err)
-			}
-			if parsed.Is4() {
-				cfg.PodIPv4 = parsed.String()
-				continue
-			}
-			cfg.PodIPv6 = parsed.String()
-		}
-	}
-	if err := cfg.validate(); err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v", err)
-	}
-	return cfg, nil
-}
-
-func (s *settings) validate() error {
-	if s.TailscaledConfigFilePath != "" {
-		dir, file := path.Split(s.TailscaledConfigFilePath)
-		if _, err := os.Stat(dir); err != nil {
-			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
-		}
-		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
-		}
-		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
-			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
-		}
-	}
-	if s.ProxyTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_DEST_IP is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME is not supported with TS_USERSPACE")
-	}
-	if s.ProxyTargetDNSName != "" && s.ProxyTargetIP != "" {
-		return errors.New("TS_EXPERIMENTAL_DEST_DNS_NAME and TS_DEST_IP cannot both be set")
-	}
-	if s.TailnetTargetIP != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.UserspaceMode {
-		return errors.New("TS_TAILNET_TARGET_FQDN is not supported with TS_USERSPACE")
-	}
-	if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" {
-		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
-	}
-	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
-	}
-	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
-		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
-	}
-	if s.EnableForwardingOptimizations && s.UserspaceMode {
-		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
-	}
-	if s.HealthCheckAddrPort != "" {
-		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
-			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
-		}
-	}
-	return nil
-}
-
-// setupKube is responsible for doing any necessary configuration and checks to
-// ensure that tailscale state storage and authentication mechanism will work on
-// Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
-	if cfg.KubeSecret == "" {
-		return nil
-	}
-	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
-	if err != nil {
-		return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-	}
-	cfg.KubernetesCanPatch = canPatch
-
-	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
-	if err != nil {
-		if !kubeclient.IsNotFoundErr(err) {
-			return fmt.Errorf("getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
-		}
-
-		if !canCreate {
-			return fmt.Errorf("tailscale state Secret %s does not exist and we don't have permissions to create it. "+
-				"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
-				"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
-				"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
-		}
-	}
-
-	// Return early if we already have an auth key.
-	if cfg.AuthKey != "" || isOneStepConfig(cfg) {
-		return nil
-	}
-
-	if s == nil {
-		log.Print("TS_AUTHKEY not provided and state Secret does not exist, login will be interactive if needed.")
-		return nil
-	}
-
-	keyBytes, _ := s.Data["authkey"]
-	key := string(keyBytes)
-
-	if key != "" {
-		// Enforce that we must be able to patch out the authkey after
-		// authenticating if you want to use this feature. This avoids
-		// us having to deal with the case where we might leave behind
-		// an unnecessary reusable authkey in a secret, like a rake in
-		// the grass.
-		if !cfg.KubernetesCanPatch {
-			return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the Secret to manage the authkey.")
-		}
-		cfg.AuthKey = key
-	}
-
-	log.Print("No authkey found in state Secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-
-	return nil
-}
-
-// isTwoStepConfigAuthOnce returns true if the Tailscale node should be configured
-// in two steps and login should only happen once.
-// Step 1: run 'tailscaled'
-// Step 2):
-// A) if this is the first time starting this node run 'tailscale up --authkey <authkey> <config opts>'
-// B) if this is not the first time starting this node run 'tailscale set <config opts>'.
-func isTwoStepConfigAuthOnce(cfg *settings) bool {
-	return cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isTwoStepConfigAlwaysAuth returns true if the Tailscale node should be configured
-// in two steps and we should log in every time it starts.
-// Step 1: run 'tailscaled'
-// Step 2): run 'tailscale up --authkey <authkey> <config opts>'
-func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
-	return !cfg.AuthOnce && cfg.TailscaledConfigFilePath == ""
-}
-
-// isOneStepConfig returns true if the Tailscale node should always be ran and
-// configured in a single step by running 'tailscaled <config opts>'
-func isOneStepConfig(cfg *settings) bool {
-	return cfg.TailscaledConfigFilePath != ""
-}
-
-// isL3Proxy returns true if the Tailscale node needs to be configured to act
-// as an L3 proxy, proxying to an endpoint provided via one of the config env
-// vars.
-func isL3Proxy(cfg *settings) bool {
-	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress || cfg.EgressSvcsCfgPath != ""
-}
-
-// hasKubeStateStore returns true if the state must be stored in a Kubernetes
-// Secret.
-func hasKubeStateStore(cfg *settings) bool {
-	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
-}
-
-// defaultEnv returns the value of the given envvar name, or defVal if
-// unset.
-func defaultEnv(name, defVal string) string {
-	if v, ok := os.LookupEnv(name); ok {
-		return v
-	}
-	return defVal
-}
-
-// defaultEnvStringPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being set to empty string vs unset.
-func defaultEnvStringPointer(name string) *string {
-	if v, ok := os.LookupEnv(name); ok {
-		return &v
-	}
-	return nil
-}
-
-// defaultEnvBoolPointer returns a pointer to the given envvar value if set, else
-// returns nil. This is useful in cases where we need to distinguish between a
-// variable being explicitly set to false vs unset.
-func defaultEnvBoolPointer(name string) *bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return nil
-	}
-	return &ret
-}
-
-func defaultEnvs(names []string, defVal string) string {
-	for _, name := range names {
-		if v, ok := os.LookupEnv(name); ok {
-			return v
-		}
-	}
-	return defVal
-}
-
-// defaultBool returns the boolean value of the given envvar name, or
-// defVal if unset or not a bool.
-func defaultBool(name string, defVal bool) bool {
-	v := os.Getenv(name)
-	ret, err := strconv.ParseBool(v)
-	if err != nil {
-		return defVal
-	}
-	return ret
-}

cmd/containerboot/tailscaled.go

@@ -1,162 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build linux
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log"
-	"os"
-	"os/exec"
-	"strings"
-	"syscall"
-	"time"
-
-	"tailscale.com/client/tailscale"
-)
-
-func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, *os.Process, error) {
-	args := tailscaledArgs(cfg)
-	// tailscaled runs without context, since it needs to persist
-	// beyond the startup timeout in ctx.
-	cmd := exec.Command("tailscaled", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-	log.Printf("Starting tailscaled")
-	if err := cmd.Start(); err != nil {
-		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)
-	}
-
-	// Wait for the socket file to appear, otherwise API ops will racily fail.
-	log.Printf("Waiting for tailscaled socket")
-	for {
-		if ctx.Err() != nil {
-			log.Fatalf("Timed out waiting for tailscaled socket")
-		}
-		_, err := os.Stat(cfg.Socket)
-		if errors.Is(err, fs.ErrNotExist) {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		} else if err != nil {
-			log.Fatalf("Waiting for tailscaled socket: %v", err)
-		}
-		break
-	}
-
-	tsClient := &tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
-	return tsClient, cmd.Process, nil
-}
-
-// tailscaledArgs uses cfg to construct the argv for tailscaled.
-func tailscaledArgs(cfg *settings) []string {
-	args := []string{"--socket=" + cfg.Socket}
-	switch {
-	case cfg.InKubernetes && cfg.KubeSecret != "":
-		args = append(args, "--state=kube:"+cfg.KubeSecret)
-		if cfg.StateDir == "" {
-			cfg.StateDir = "/tmp"
-		}
-		fallthrough
-	case cfg.StateDir != "":
-		args = append(args, "--statedir="+cfg.StateDir)
-	default:
-		args = append(args, "--state=mem:", "--statedir=/tmp")
-	}
-
-	if cfg.UserspaceMode {
-		args = append(args, "--tun=userspace-networking")
-	} else if err := ensureTunFile(cfg.Root); err != nil {
-		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
-	}
-
-	if cfg.SOCKSProxyAddr != "" {
-		args = append(args, "--socks5-server="+cfg.SOCKSProxyAddr)
-	}
-	if cfg.HTTPProxyAddr != "" {
-		args = append(args, "--outbound-http-proxy-listen="+cfg.HTTPProxyAddr)
-	}
-	if cfg.TailscaledConfigFilePath != "" {
-		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
-	}
-	if cfg.DaemonExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
-	}
-	return args
-}
-
-// tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
-// if TS_AUTH_ONCE is set, only the first time containerboot starts.
-func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	if cfg.AuthKey != "" {
-		args = append(args, "--authkey="+cfg.AuthKey)
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
-	}
-	return nil
-}
-
-// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
-// options that are passed in via environment variables. This is run after the
-// node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS != nil && *cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}

cmd/derper/depaware.txt

@@ -113,10 +113,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
-        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
-        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper
+        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
         tailscale.com/paths                                          from tailscale.com/client/tailscale
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
@@ -129,7 +128,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -140,7 +139,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/ipn
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
-        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/ipn+
@@ -155,7 +153,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
-        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
@@ -164,17 +162,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
-        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
-        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
-        tailscale.com/util/testenv                                   from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
@@ -195,7 +187,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
-        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -256,7 +248,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper+
+        flag                                                         from tailscale.com/cmd/derper
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -264,7 +256,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
-        html/template                                                from tailscale.com/cmd/derper
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
@@ -291,7 +282,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
-   W    os/user                                                      from tailscale.com/util/winutil+
+   W    os/user                                                      from tailscale.com/util/winutil
         path                                                         from github.com/prometheus/client_golang/prometheus/internal+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
@@ -309,8 +300,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
         text/tabwriter                                               from runtime/pprof
-        text/template                                                from html/template
-        text/template/parse                                          from html/template+
         time                                                         from compress/gzip+
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+

cmd/derper/derper.go

@@ -19,7 +19,6 @@ import (
 	"expvar"
 	"flag"
 	"fmt"
-	"html/template"
 	"io"
 	"log"
 	"math"
@@ -213,16 +212,25 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
 		w.WriteHeader(200)
-		err := homePageTemplate.Execute(w, templateData{
-			ShowAbuseInfo: validProdHostname.MatchString(*hostname),
-			Disabled:      !*runDERP,
-			AllowDebug:    tsweb.AllowDebugAccess(r),
-		})
-		if err != nil {
-			if r.Context().Err() == nil {
-				log.Printf("homePageTemplate.Execute: %v", err)
-			}
-			return
+		io.WriteString(w, `<html><body>
+<h1>DERP</h1>
+<p>
+  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
+</p>
+<p>
+  Documentation:
+</p>
+<ul>
+  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
+  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
+  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
+</ul>
+`)
+		if !*runDERP {
+			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
+		}
+		if tsweb.AllowDebugAccess(r) {
+			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
 		}
 	}))
 	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -460,52 +468,3 @@ func init() {
 		return 0
 	}))
 }
-
-type templateData struct {
-	ShowAbuseInfo bool
-	Disabled      bool
-	AllowDebug    bool
-}
-
-// homePageTemplate renders the home page using [templateData].
-var homePageTemplate = template.Must(template.New("home").Parse(`<html><body>
-<h1>DERP</h1>
-<p>
-  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
-</p>
-
-<p>
-  It provides STUN, interactive connectivity establishment, and relaying of end-to-end encrypted traffic
-  for Tailscale clients.
-</p>
-
-{{if .ShowAbuseInfo }}
-<p>
-  If you suspect abuse, please contact <a href="mailto:security@tailscale.com">security@tailscale.com</a>.
-</p>
-{{end}}
-
-<p>
-  Documentation:
-</p>
-
-<ul>
-{{if .ShowAbuseInfo }}
-  <li><a href="https://tailscale.com/security-policies">Tailscale Security Policies</a></li>
-  <li><a href="https://tailscale.com/tailscale-aup">Tailscale Acceptable Use Policies</a></li>
-{{end}}
-  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
-  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
-  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
-</ul>
-
-{{if .Disabled}}
-<p>Status: <b>disabled</b></p>
-{{end}}
-
-{{if .AllowDebug}}
-<p>Debug info at <a href='/debug/'>/debug/</a>.</p>
-{{end}}
-</body>
-</html>
-`))

cmd/derper/derper_test.go

@@ -4,9 +4,7 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -112,30 +110,3 @@ func TestDeps(t *testing.T) {
 		},
 	}.Check(t)
 }
-
-func TestTemplate(t *testing.T) {
-	buf := &bytes.Buffer{}
-	err := homePageTemplate.Execute(buf, templateData{
-		ShowAbuseInfo: true,
-		Disabled:      true,
-		AllowDebug:    true,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	str := buf.String()
-	if !strings.Contains(str, "If you suspect abuse") {
-		t.Error("Output is missing abuse mailto")
-	}
-	if !strings.Contains(str, "Tailscale Security Policies") {
-		t.Error("Output is missing Tailscale Security Policies link")
-	}
-	if !strings.Contains(str, "Status:") {
-		t.Error("Output is missing disabled status")
-	}
-	if !strings.Contains(str, "Debug info") {
-		t.Error("Output is missing debug info")
-	}
-	fmt.Println(buf.String())
-}

cmd/derpprobe/derpprobe.go

@@ -29,7 +29,6 @@ var (
 	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
 	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
 	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
-	regionCode   = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -48,9 +47,6 @@ func main() {
 	if *bwInterval > 0 {
 		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
 	}
-	if *regionCode != "" {
-		opts = append(opts, prober.WithRegion(*regionCode))
-	}
 	dp, err := prober.DERP(p, *derpMapURL, opts...)
 	if err != nil {
 		log.Fatal(err)
@@ -79,11 +75,6 @@ func main() {
 		prober.WithPageLink("Prober metrics", "/debug/varz"),
 		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
 	), tsweb.HandlerOptions{Logf: log.Printf}))
-	mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "text/plain")
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ok\n"))
-	}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }

cmd/get-authkey/main.go

@@ -51,7 +51,6 @@ func main() {
 
 	ctx := context.Background()
 	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-get-authkey"
 	tsClient.HTTPClient = credentials.Client(ctx)
 	tsClient.BaseURL = baseURL
 

cmd/k8s-operator/connector.go

@@ -13,8 +13,7 @@ import (
 	"sync"
 	"time"
 
-	"errors"
-
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
@@ -59,7 +58,6 @@ type ConnectorReconciler struct {
 
 	subnetRouters set.Slice[types.UID] // for subnet routers gauge
 	exitNodes     set.Slice[types.UID] // for exit nodes gauge
-	appConnectors set.Slice[types.UID] // for app connectors gauge
 }
 
 var (
@@ -69,8 +67,6 @@ var (
 	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
 	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
-	// gaugeConnectorAppConnectorResources tracks the number of Connectors currently managed by this operator instance that are app connectors.
-	gaugeConnectorAppConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithAppConnectorCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -112,12 +108,13 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	oldCnStatus := cn.Status.DeepCopy()
 	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
-		var updateErr error
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
-			updateErr = a.Client.Status().Update(ctx, cn)
+			if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
 		}
-		return res, errors.Join(err, updateErr)
+		return res, err
 	}
 
 	if !slices.Contains(cn.Finalizers, FinalizerName) {
@@ -153,9 +150,6 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 	}
-	if cn.Spec.AppConnector != nil {
-		cn.Status.IsAppConnector = true
-	}
 	cn.Status.SubnetRoutes = ""
 	return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 }
@@ -195,37 +189,23 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 	}
 
-	if cn.Spec.AppConnector != nil {
-		sts.Connector.isAppConnector = true
-		if len(cn.Spec.AppConnector.Routes) != 0 {
-			sts.Connector.routes = cn.Spec.AppConnector.Routes.Stringify()
-		}
-	}
-
 	a.mu.Lock()
-	if cn.Spec.ExitNode {
+	if sts.Connector.isExitNode {
 		a.exitNodes.Add(cn.UID)
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
-	if cn.Spec.SubnetRouter != nil {
+	if sts.Connector.routes != "" {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
-	if cn.Spec.AppConnector != nil {
-		a.appConnectors.Add(cn.GetUID())
-	} else {
-		a.appConnectors.Remove(cn.GetUID())
-	}
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
-	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
-	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
@@ -268,15 +248,12 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
 	a.mu.Lock()
 	a.subnetRouters.Remove(cn.UID)
 	a.exitNodes.Remove(cn.UID)
-	a.appConnectors.Remove(cn.UID)
 	a.mu.Unlock()
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
-	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
-	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 	return true, nil
 }
@@ -285,14 +262,8 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 	// Connector fields are already validated at apply time with CEL validation
 	// on custom resource fields. The checks here are a backup in case the
 	// CEL validation breaks without us noticing.
-	if cn.Spec.SubnetRouter == nil && !cn.Spec.ExitNode && cn.Spec.AppConnector == nil {
-		return errors.New("invalid spec: a Connector must be configured as at least one of subnet router, exit node or app connector")
-	}
-	if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
-		return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
-	}
-	if cn.Spec.AppConnector != nil {
-		return validateAppConnector(cn.Spec.AppConnector)
+	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
+		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
 	}
 	if cn.Spec.SubnetRouter == nil {
 		return nil
@@ -301,27 +272,19 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 }
 
 func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
-	if len(sb.AdvertiseRoutes) == 0 {
+	if len(sb.AdvertiseRoutes) < 1 {
 		return errors.New("invalid subnet router spec: no routes defined")
 	}
-	return validateRoutes(sb.AdvertiseRoutes)
-}
-
-func validateAppConnector(ac *tsapi.AppConnector) error {
-	return validateRoutes(ac.Routes)
-}
-
-func validateRoutes(routes tsapi.Routes) error {
-	var errs []error
-	for _, route := range routes {
+	var err error
+	for _, route := range sb.AdvertiseRoutes {
 		pfx, e := netip.ParsePrefix(string(route))
 		if e != nil {
-			errs = append(errs, fmt.Errorf("route %v is invalid: %v", route, e))
+			err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err))
 			continue
 		}
 		if pfx.Masked() != pfx {
-			errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
+			err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
 		}
 	}
-	return errors.Join(errs...)
+	return err
 }

cmd/k8s-operator/connector_test.go

@@ -8,14 +8,12 @@ package main
 import (
 	"context"
 	"testing"
-	"time"
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -280,7 +278,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassReady),
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
@@ -298,100 +296,3 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	expectReconciled(t, cr, "", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 }
-
-func TestConnectorWithAppConnector(t *testing.T) {
-	// Setup
-	cn := &tsapi.Connector{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "test",
-			UID:  types.UID("1234-UID"),
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind:       tsapi.ConnectorKind,
-			APIVersion: "tailscale.io/v1alpha1",
-		},
-		Spec: tsapi.ConnectorSpec{
-			AppConnector: &tsapi.AppConnector{},
-		},
-	}
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(cn).
-		WithStatusSubresource(cn).
-		Build()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	cl := tstest.NewClock(tstest.ClockOpts{})
-	fr := record.NewFakeRecorder(1)
-	cr := &ConnectorReconciler{
-		Client: fc,
-		clock:  cl,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:   zl.Sugar(),
-		recorder: fr,
-	}
-
-	// 1. Connector with app connnector is created and becomes ready
-	expectReconciled(t, cr, "", "test")
-	fullName, shortName := findGenName(t, fc, "", "test", "connector")
-	opts := configOpts{
-		stsName:        shortName,
-		secretName:     fullName,
-		parentType:     "connector",
-		hostname:       "test-connector",
-		app:            kubetypes.AppConnector,
-		isAppConnector: true,
-	}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
-	// Connector's ready condition should be set to true
-
-	cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
-	cn.Status.IsAppConnector = true
-	cn.Status.Conditions = []metav1.Condition{{
-		Type:               string(tsapi.ConnectorReady),
-		Status:             metav1.ConditionTrue,
-		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
-		Reason:             reasonConnectorCreated,
-		Message:            reasonConnectorCreated,
-	}}
-	expectEqual(t, fc, cn, nil)
-
-	// 2. Connector with invalid app connector routes has status set to invalid
-	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
-		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
-	})
-	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
-	expectReconciled(t, cr, "", "test")
-	cn.Status.Conditions = []metav1.Condition{{
-		Type:               string(tsapi.ConnectorReady),
-		Status:             metav1.ConditionFalse,
-		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
-		Reason:             reasonConnectorInvalid,
-		Message:            "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
-	}}
-	expectEqual(t, fc, cn, nil)
-
-	// 3. Connector with valid app connnector routes becomes ready
-	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
-		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
-	})
-	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
-	cn.Status.Conditions = []metav1.Condition{{
-		Type:               string(tsapi.ConnectorReady),
-		Status:             metav1.ConditionTrue,
-		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
-		Reason:             reasonConnectorCreated,
-		Message:            reasonConnectorCreated,
-	}}
-	expectReconciled(t, cr, "", "test")
-}

cmd/k8s-operator/depaware.txt

@@ -80,6 +80,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -306,7 +310,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
@@ -650,11 +654,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
-  LW    tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
-        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/ipn/localapi+
@@ -665,7 +668,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -688,7 +690,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
         tailscale.com/k8s-operator/sessionrecording/tsrecorder       from tailscale.com/k8s-operator/sessionrecording+
         tailscale.com/k8s-operator/sessionrecording/ws               from tailscale.com/k8s-operator/sessionrecording
-        tailscale.com/kube/egressservices                            from tailscale.com/cmd/k8s-operator
         tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
         tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
         tailscale.com/kube/kubetypes                                 from tailscale.com/cmd/k8s-operator+
@@ -732,11 +733,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
-        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/tsd+
+        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -771,7 +772,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
-        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/appc+
@@ -789,7 +789,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/appc+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlclient+
@@ -808,12 +808,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
-        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
-        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
@@ -823,7 +819,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -22,7 +22,7 @@ rules:
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["tailscale.com"]
-  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status", "proxygroups", "proxygroups/status"]
+  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
   verbs: ["get", "list", "watch", "update"]
 - apiGroups: ["tailscale.com"]
   resources: ["dnsconfigs", "dnsconfigs/status"]
@@ -53,15 +53,12 @@ rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get","list","watch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
-  verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
+  verbs: ["get", "list", "watch"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
   verbs: ["get", "create", "patch", "update", "list", "watch"]

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -16,9 +16,6 @@ rules:
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create", "patch", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -57,12 +57,12 @@ operatorConfig:
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.
-# https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
-# https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
+# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
+# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
 # Note that this section contains only a few global configuration options and
 # will not be updated with more configuration options in the future.
 # If you need more configuration options, take a look at ProxyClass:
-# https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
+# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
@@ -79,13 +79,12 @@ proxyConfig:
   defaultTags: "tag:k8s"
   firewallMode: auto
   # If defined, this proxy class will be used as the default proxy class for
-  # service and ingress resources that do not have a proxy class defined. It
-  # does not apply to Connector resources.
+  # service and ingress resources that do not have a proxy class defined.
   defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.
-# https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
+# https://tailscale.com/kb/1236/kubernetes-operator/#accessing-the-kubernetes-control-plane-using-an-api-server-proxy
 apiServerProxyConfig:
   mode: "false" # "true", "false", "noauth"
 

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -24,10 +24,6 @@ spec:
           jsonPath: .status.isExitNode
           name: IsExitNode
           type: string
-        - description: Whether this Connector instance is an app connector.
-          jsonPath: .status.isAppConnector
-          name: IsAppConnector
-          type: string
         - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
@@ -41,7 +37,7 @@ spec:
             exit node.
             Connector is a cluster-scoped resource.
             More info:
-            https://tailscale.com/kb/1441/kubernetes-operator-connector
+            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
           type: object
           required:
             - spec
@@ -70,40 +66,10 @@ spec:
                 https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
-                appConnector:
-                  description: |-
-                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
-                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
-                    Connector does not act as an app connector.
-                    Note that you will need to manually configure the permissions and the domains for the app connector via the
-                    Admin panel.
-                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
-                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
-                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
-                    tested or optimised for.
-                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
-                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
-                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
-                    device with a static IP address.
-                    https://tailscale.com/kb/1281/app-connectors
-                  type: object
-                  properties:
-                    routes:
-                      description: |-
-                        Routes are optional preconfigured routes for the domains routed via the app connector.
-                        If not set, routes for the domains will be discovered dynamically.
-                        If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
-                        also dynamically discover other routes.
-                        https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
-                      type: array
-                      minItems: 1
-                      items:
-                        type: string
-                        format: cidr
                 exitNode:
                   description: |-
-                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
-                    This field is mutually exclusive with the appConnector field.
+                    ExitNode defines whether the Connector node should act as a
+                    Tailscale exit node. Defaults to false.
                     https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
@@ -124,11 +90,9 @@ spec:
                   type: string
                 subnetRouter:
                   description: |-
-                    SubnetRouter defines subnet routes that the Connector device should
-                    expose to tailnet as a Tailscale subnet router.
+                    SubnetRouter defines subnet routes that the Connector node should
+                    expose to tailnet. If unset, none are exposed.
                     https://tailscale.com/kb/1019/subnets/
-                    If this field is unset, the device does not get configured as a Tailscale subnet router.
-                    This field is mutually exclusive with the appConnector field.
                   type: object
                   required:
                     - advertiseRoutes
@@ -151,7 +115,7 @@ spec:
                     To autoapprove the subnet routes or exit node defined by a Connector,
                     you can configure Tailscale ACLs to give these tags the necessary
                     permissions.
-                    See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
+                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
                     If you specify custom tags here, you must also make the operator an owner of these tags.
                     See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
                     Tags cannot be changed once a Connector node has been created.
@@ -161,10 +125,8 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
               x-kubernetes-validations:
-                - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
-                  message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
-                - rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
-                  message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
+                - rule: has(self.subnetRouter) || self.exitNode == true
+                  message: A Connector needs to be either an exit node or a subnet router, or both.
             status:
               description: |-
                 ConnectorStatus describes the status of the Connector. This is set
@@ -238,9 +200,6 @@ spec:
                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                     node.
                   type: string
-                isAppConnector:
-                  description: IsAppConnector is set to true if the Connector acts as an app connector.
-                  type: boolean
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -30,7 +30,7 @@ spec:
             connector.spec.proxyClass field.
             ProxyClass is a cluster scoped resource.
             More info:
-            https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
+            https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource.
           type: object
           required:
             - spec
@@ -1896,182 +1896,6 @@ spec:
                                   Value is the taint value the toleration matches to.
                                   If the operator is Exists, the value should be empty, otherwise just a regular string.
                                 type: string
-                        topologySpreadConstraints:
-                          description: |-
-                            Proxy Pod's topology spread constraints.
-                            By default Tailscale Kubernetes operator does not apply any topology spread constraints.
-                            https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
-                          type: array
-                          items:
-                            description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
-                            type: object
-                            required:
-                              - maxSkew
-                              - topologyKey
-                              - whenUnsatisfiable
-                            properties:
-                              labelSelector:
-                                description: |-
-                                  LabelSelector is used to find matching pods.
-                                  Pods that match this label selector are counted to determine the number of pods
-                                  in their corresponding topology domain.
-                                type: object
-                                properties:
-                                  matchExpressions:
-                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
-                                    type: array
-                                    items:
-                                      description: |-
-                                        A label selector requirement is a selector that contains values, a key, and an operator that
-                                        relates the key and values.
-                                      type: object
-                                      required:
-                                        - key
-                                        - operator
-                                      properties:
-                                        key:
-                                          description: key is the label key that the selector applies to.
-                                          type: string
-                                        operator:
-                                          description: |-
-                                            operator represents a key's relationship to a set of values.
-                                            Valid operators are In, NotIn, Exists and DoesNotExist.
-                                          type: string
-                                        values:
-                                          description: |-
-                                            values is an array of string values. If the operator is In or NotIn,
-                                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                            the values array must be empty. This array is replaced during a strategic
-                                            merge patch.
-                                          type: array
-                                          items:
-                                            type: string
-                                          x-kubernetes-list-type: atomic
-                                    x-kubernetes-list-type: atomic
-                                  matchLabels:
-                                    description: |-
-                                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                                    type: object
-                                    additionalProperties:
-                                      type: string
-                                x-kubernetes-map-type: atomic
-                              matchLabelKeys:
-                                description: |-
-                                  MatchLabelKeys is a set of pod label keys to select the pods over which
-                                  spreading will be calculated. The keys are used to lookup values from the
-                                  incoming pod labels, those key-value labels are ANDed with labelSelector
-                                  to select the group of existing pods over which spreading will be calculated
-                                  for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
-                                  MatchLabelKeys cannot be set when LabelSelector isn't set.
-                                  Keys that don't exist in the incoming pod labels will
-                                  be ignored. A null or empty list means only match against labelSelector.
-
-                                  This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
-                                type: array
-                                items:
-                                  type: string
-                                x-kubernetes-list-type: atomic
-                              maxSkew:
-                                description: |-
-                                  MaxSkew describes the degree to which pods may be unevenly distributed.
-                                  When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
-                                  between the number of matching pods in the target topology and the global minimum.
-                                  The global minimum is the minimum number of matching pods in an eligible domain
-                                  or zero if the number of eligible domains is less than MinDomains.
-                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
-                                  labelSelector spread as 2/2/1:
-                                  In this case, the global minimum is 1.
-                                  | zone1 | zone2 | zone3 |
-                                  |  P P  |  P P  |   P   |
-                                  - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
-                                  scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
-                                  violate MaxSkew(1).
-                                  - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
-                                  When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
-                                  to topologies that satisfy it.
-                                  It's a required field. Default value is 1 and 0 is not allowed.
-                                type: integer
-                                format: int32
-                              minDomains:
-                                description: |-
-                                  MinDomains indicates a minimum number of eligible domains.
-                                  When the number of eligible domains with matching topology keys is less than minDomains,
-                                  Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
-                                  And when the number of eligible domains with matching topology keys equals or greater than minDomains,
-                                  this value has no effect on scheduling.
-                                  As a result, when the number of eligible domains is less than minDomains,
-                                  scheduler won't schedule more than maxSkew Pods to those domains.
-                                  If value is nil, the constraint behaves as if MinDomains is equal to 1.
-                                  Valid values are integers greater than 0.
-                                  When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
-
-                                  For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
-                                  labelSelector spread as 2/2/2:
-                                  | zone1 | zone2 | zone3 |
-                                  |  P P  |  P P  |  P P  |
-                                  The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
-                                  In this situation, new pod with the same labelSelector cannot be scheduled,
-                                  because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
-                                  it will violate MaxSkew.
-                                type: integer
-                                format: int32
-                              nodeAffinityPolicy:
-                                description: |-
-                                  NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
-                                  when calculating pod topology spread skew. Options are:
-                                  - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
-                                  - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
-
-                                  If this value is nil, the behavior is equivalent to the Honor policy.
-                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
-                                type: string
-                              nodeTaintsPolicy:
-                                description: |-
-                                  NodeTaintsPolicy indicates how we will treat node taints when calculating
-                                  pod topology spread skew. Options are:
-                                  - Honor: nodes without taints, along with tainted nodes for which the incoming pod
-                                  has a toleration, are included.
-                                  - Ignore: node taints are ignored. All nodes are included.
-
-                                  If this value is nil, the behavior is equivalent to the Ignore policy.
-                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
-                                type: string
-                              topologyKey:
-                                description: |-
-                                  TopologyKey is the key of node labels. Nodes that have a label with this key
-                                  and identical values are considered to be in the same topology.
-                                  We consider each <key, value> as a "bucket", and try to put balanced number
-                                  of pods into each bucket.
-                                  We define a domain as a particular instance of a topology.
-                                  Also, we define an eligible domain as a domain whose nodes meet the requirements of
-                                  nodeAffinityPolicy and nodeTaintsPolicy.
-                                  e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
-                                  And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
-                                  It's a required field.
-                                type: string
-                              whenUnsatisfiable:
-                                description: |-
-                                  WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
-                                  the spread constraint.
-                                  - DoNotSchedule (default) tells the scheduler not to schedule it.
-                                  - ScheduleAnyway tells the scheduler to schedule the pod in any location,
-                                    but giving higher precedence to topologies that would help reduce the
-                                    skew.
-                                  A constraint is considered "Unsatisfiable" for an incoming pod
-                                  if and only if every possible node assignment for that pod would violate
-                                  "MaxSkew" on some topology.
-                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
-                                  labelSelector spread as 3/1/1:
-                                  | zone1 | zone2 | zone3 |
-                                  | P P P |   P   |   P   |
-                                  If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
-                                  to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
-                                  MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
-                                  won't make it *more* imbalanced.
-                                  It's a required field.
-                                type: string
                 tailscale:
                   description: |-
                     TailscaleConfig contains options to configure the tailscale-specific
@@ -2084,7 +1908,7 @@ spec:
                         routes advertized by other nodes on the tailnet, such as subnet
                         routes.
                         This is equivalent of passing --accept-routes flag to a tailscale Linux client.
-                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
+                        https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
                         Defaults to false.
                       type: boolean
             status:

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -1,187 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
-  name: proxygroups.tailscale.com
-spec:
-  group: tailscale.com
-  names:
-    kind: ProxyGroup
-    listKind: ProxyGroupList
-    plural: proxygroups
-    shortNames:
-      - pg
-    singular: proxygroup
-  scope: Cluster
-  versions:
-    - additionalPrinterColumns:
-        - description: Status of the deployed ProxyGroup resources.
-          jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
-          name: Status
-          type: string
-      name: v1alpha1
-      schema:
-        openAPIV3Schema:
-          type: object
-          required:
-            - spec
-          properties:
-            apiVersion:
-              description: |-
-                APIVersion defines the versioned schema of this representation of an object.
-                Servers should convert recognized schemas to the latest internal value, and
-                may reject unrecognized values.
-                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
-              type: string
-            kind:
-              description: |-
-                Kind is a string value representing the REST resource this object represents.
-                Servers may infer this from the endpoint the client submits requests to.
-                Cannot be updated.
-                In CamelCase.
-                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
-              type: string
-            metadata:
-              type: object
-            spec:
-              description: Spec describes the desired ProxyGroup instances.
-              type: object
-              required:
-                - type
-              properties:
-                hostnamePrefix:
-                  description: |-
-                    HostnamePrefix is the hostname prefix to use for tailnet devices created
-                    by the ProxyGroup. Each device will have the integer number from its
-                    StatefulSet pod appended to this prefix to form the full hostname.
-                    HostnamePrefix can contain lower case letters, numbers and dashes, it
-                    must not start with a dash and must be between 1 and 62 characters long.
-                  type: string
-                  pattern: ^[a-z0-9][a-z0-9-]{0,61}$
-                proxyClass:
-                  description: |-
-                    ProxyClass is the name of the ProxyClass custom resource that contains
-                    configuration options that should be applied to the resources created
-                    for this ProxyGroup. If unset, and there is no default ProxyClass
-                    configured, the operator will create resources with the default
-                    configuration.
-                  type: string
-                replicas:
-                  description: |-
-                    Replicas specifies how many replicas to create the StatefulSet with.
-                    Defaults to 2.
-                  type: integer
-                  format: int32
-                tags:
-                  description: |-
-                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
-                    If you specify custom tags here, make sure you also make the operator
-                    an owner of these tags.
-                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
-                    Tags cannot be changed once a ProxyGroup device has been created.
-                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
-                  type: array
-                  items:
-                    type: string
-                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
-                type:
-                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                  type: string
-                  enum:
-                    - egress
-            status:
-              description: |-
-                ProxyGroupStatus describes the status of the ProxyGroup resources. This is
-                set and managed by the Tailscale operator.
-              type: object
-              properties:
-                conditions:
-                  description: |-
-                    List of status conditions to indicate the status of the ProxyGroup
-                    resources. Known condition types are `ProxyGroupReady`.
-                  type: array
-                  items:
-                    description: Condition contains details for one aspect of the current state of this API Resource.
-                    type: object
-                    required:
-                      - lastTransitionTime
-                      - message
-                      - reason
-                      - status
-                      - type
-                    properties:
-                      lastTransitionTime:
-                        description: |-
-                          lastTransitionTime is the last time the condition transitioned from one status to another.
-                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
-                        type: string
-                        format: date-time
-                      message:
-                        description: |-
-                          message is a human readable message indicating details about the transition.
-                          This may be an empty string.
-                        type: string
-                        maxLength: 32768
-                      observedGeneration:
-                        description: |-
-                          observedGeneration represents the .metadata.generation that the condition was set based upon.
-                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
-                          with respect to the current state of the instance.
-                        type: integer
-                        format: int64
-                        minimum: 0
-                      reason:
-                        description: |-
-                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
-                          Producers of specific condition types may define expected values and meanings for this field,
-                          and whether the values are considered a guaranteed API.
-                          The value should be a CamelCase string.
-                          This field may not be empty.
-                        type: string
-                        maxLength: 1024
-                        minLength: 1
-                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
-                      status:
-                        description: status of the condition, one of True, False, Unknown.
-                        type: string
-                        enum:
-                          - "True"
-                          - "False"
-                          - Unknown
-                      type:
-                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
-                        type: string
-                        maxLength: 316
-                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                  x-kubernetes-list-map-keys:
-                    - type
-                  x-kubernetes-list-type: map
-                devices:
-                  description: List of tailnet devices associated with the ProxyGroup StatefulSet.
-                  type: array
-                  items:
-                    type: object
-                    required:
-                      - hostname
-                    properties:
-                      hostname:
-                        description: |-
-                          Hostname is the fully qualified domain name of the device.
-                          If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
-                          node.
-                        type: string
-                      tailnetIPs:
-                        description: |-
-                          TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
-                          assigned to the device.
-                        type: array
-                        items:
-                          type: string
-                  x-kubernetes-list-map-keys:
-                    - hostname
-                  x-kubernetes-list-type: map
-      served: true
-      storage: true
-      subresources:
-        status: {}

cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml

@@ -1670,7 +1670,7 @@ spec:
                     - type
                   x-kubernetes-list-type: map
                 devices:
-                  description: List of tailnet devices associated with the Recorder StatefulSet.
+                  description: List of tailnet devices associated with the Recorder statefulset.
                   type: array
                   items:
                     type: object

cmd/k8s-operator/deploy/examples/proxygroup.yaml

@@ -1,7 +0,0 @@
-apiVersion: tailscale.com/v1alpha1
-kind: ProxyGroup
-metadata:
-  name: egress-proxies
-spec:
-  type: egress
-  replicas: 3

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -53,10 +53,6 @@ spec:
               jsonPath: .status.isExitNode
               name: IsExitNode
               type: string
-            - description: Whether this Connector instance is an app connector.
-              jsonPath: .status.isAppConnector
-              name: IsAppConnector
-              type: string
             - description: Status of the deployed Connector resources.
               jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
               name: Status
@@ -70,7 +66,7 @@ spec:
                     exit node.
                     Connector is a cluster-scoped resource.
                     More info:
-                    https://tailscale.com/kb/1441/kubernetes-operator-connector
+                    https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
                 properties:
                     apiVersion:
                         description: |-
@@ -95,40 +91,10 @@ spec:
                             More info:
                             https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
-                            appConnector:
-                                description: |-
-                                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
-                                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
-                                    Connector does not act as an app connector.
-                                    Note that you will need to manually configure the permissions and the domains for the app connector via the
-                                    Admin panel.
-                                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
-                                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
-                                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
-                                    tested or optimised for.
-                                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
-                                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
-                                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
-                                    device with a static IP address.
-                                    https://tailscale.com/kb/1281/app-connectors
-                                properties:
-                                    routes:
-                                        description: |-
-                                            Routes are optional preconfigured routes for the domains routed via the app connector.
-                                            If not set, routes for the domains will be discovered dynamically.
-                                            If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
-                                            also dynamically discover other routes.
-                                            https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
-                                        items:
-                                            format: cidr
-                                            type: string
-                                        minItems: 1
-                                        type: array
-                                type: object
                             exitNode:
                                 description: |-
-                                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
-                                    This field is mutually exclusive with the appConnector field.
+                                    ExitNode defines whether the Connector node should act as a
+                                    Tailscale exit node. Defaults to false.
                                     https://tailscale.com/kb/1103/exit-nodes
                                 type: boolean
                             hostname:
@@ -149,11 +115,9 @@ spec:
                                 type: string
                             subnetRouter:
                                 description: |-
-                                    SubnetRouter defines subnet routes that the Connector device should
-                                    expose to tailnet as a Tailscale subnet router.
+                                    SubnetRouter defines subnet routes that the Connector node should
+                                    expose to tailnet. If unset, none are exposed.
                                     https://tailscale.com/kb/1019/subnets/
-                                    If this field is unset, the device does not get configured as a Tailscale subnet router.
-                                    This field is mutually exclusive with the appConnector field.
                                 properties:
                                     advertiseRoutes:
                                         description: |-
@@ -176,7 +140,7 @@ spec:
                                     To autoapprove the subnet routes or exit node defined by a Connector,
                                     you can configure Tailscale ACLs to give these tags the necessary
                                     permissions.
-                                    See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
+                                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
                                     If you specify custom tags here, you must also make the operator an owner of these tags.
                                     See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
                                     Tags cannot be changed once a Connector node has been created.
@@ -187,10 +151,8 @@ spec:
                                 type: array
                         type: object
                         x-kubernetes-validations:
-                            - message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
-                              rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
-                            - message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
-                              rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
+                            - message: A Connector needs to be either an exit node or a subnet router, or both.
+                              rule: has(self.subnetRouter) || self.exitNode == true
                     status:
                         description: |-
                             ConnectorStatus describes the status of the Connector. This is set
@@ -263,9 +225,6 @@ spec:
                                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                                     node.
                                 type: string
-                            isAppConnector:
-                                description: IsAppConnector is set to true if the Connector acts as an app connector.
-                                type: boolean
                             isExitNode:
                                 description: IsExitNode is set to true if the Connector acts as an exit node.
                                 type: boolean
@@ -504,7 +463,7 @@ spec:
                     connector.spec.proxyClass field.
                     ProxyClass is a cluster scoped resource.
                     More info:
-                    https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource
+                    https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource.
                 properties:
                     apiVersion:
                         description: |-
@@ -2364,182 +2323,6 @@ spec:
                                                             type: string
                                                     type: object
                                                 type: array
-                                            topologySpreadConstraints:
-                                                description: |-
-                                                    Proxy Pod's topology spread constraints.
-                                                    By default Tailscale Kubernetes operator does not apply any topology spread constraints.
-                                                    https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
-                                                items:
-                                                    description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
-                                                    properties:
-                                                        labelSelector:
-                                                            description: |-
-                                                                LabelSelector is used to find matching pods.
-                                                                Pods that match this label selector are counted to determine the number of pods
-                                                                in their corresponding topology domain.
-                                                            properties:
-                                                                matchExpressions:
-                                                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
-                                                                    items:
-                                                                        description: |-
-                                                                            A label selector requirement is a selector that contains values, a key, and an operator that
-                                                                            relates the key and values.
-                                                                        properties:
-                                                                            key:
-                                                                                description: key is the label key that the selector applies to.
-                                                                                type: string
-                                                                            operator:
-                                                                                description: |-
-                                                                                    operator represents a key's relationship to a set of values.
-                                                                                    Valid operators are In, NotIn, Exists and DoesNotExist.
-                                                                                type: string
-                                                                            values:
-                                                                                description: |-
-                                                                                    values is an array of string values. If the operator is In or NotIn,
-                                                                                    the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                                                                    the values array must be empty. This array is replaced during a strategic
-                                                                                    merge patch.
-                                                                                items:
-                                                                                    type: string
-                                                                                type: array
-                                                                                x-kubernetes-list-type: atomic
-                                                                        required:
-                                                                            - key
-                                                                            - operator
-                                                                        type: object
-                                                                    type: array
-                                                                    x-kubernetes-list-type: atomic
-                                                                matchLabels:
-                                                                    additionalProperties:
-                                                                        type: string
-                                                                    description: |-
-                                                                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                                                                        map is equivalent to an element of matchExpressions, whose key field is "key", the
-                                                                        operator is "In", and the values array contains only "value". The requirements are ANDed.
-                                                                    type: object
-                                                            type: object
-                                                            x-kubernetes-map-type: atomic
-                                                        matchLabelKeys:
-                                                            description: |-
-                                                                MatchLabelKeys is a set of pod label keys to select the pods over which
-                                                                spreading will be calculated. The keys are used to lookup values from the
-                                                                incoming pod labels, those key-value labels are ANDed with labelSelector
-                                                                to select the group of existing pods over which spreading will be calculated
-                                                                for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
-                                                                MatchLabelKeys cannot be set when LabelSelector isn't set.
-                                                                Keys that don't exist in the incoming pod labels will
-                                                                be ignored. A null or empty list means only match against labelSelector.
-
-                                                                This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
-                                                            items:
-                                                                type: string
-                                                            type: array
-                                                            x-kubernetes-list-type: atomic
-                                                        maxSkew:
-                                                            description: |-
-                                                                MaxSkew describes the degree to which pods may be unevenly distributed.
-                                                                When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
-                                                                between the number of matching pods in the target topology and the global minimum.
-                                                                The global minimum is the minimum number of matching pods in an eligible domain
-                                                                or zero if the number of eligible domains is less than MinDomains.
-                                                                For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
-                                                                labelSelector spread as 2/2/1:
-                                                                In this case, the global minimum is 1.
-                                                                | zone1 | zone2 | zone3 |
-                                                                |  P P  |  P P  |   P   |
-                                                                - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
-                                                                scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
-                                                                violate MaxSkew(1).
-                                                                - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
-                                                                When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
-                                                                to topologies that satisfy it.
-                                                                It's a required field. Default value is 1 and 0 is not allowed.
-                                                            format: int32
-                                                            type: integer
-                                                        minDomains:
-                                                            description: |-
-                                                                MinDomains indicates a minimum number of eligible domains.
-                                                                When the number of eligible domains with matching topology keys is less than minDomains,
-                                                                Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
-                                                                And when the number of eligible domains with matching topology keys equals or greater than minDomains,
-                                                                this value has no effect on scheduling.
-                                                                As a result, when the number of eligible domains is less than minDomains,
-                                                                scheduler won't schedule more than maxSkew Pods to those domains.
-                                                                If value is nil, the constraint behaves as if MinDomains is equal to 1.
-                                                                Valid values are integers greater than 0.
-                                                                When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
-
-                                                                For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
-                                                                labelSelector spread as 2/2/2:
-                                                                | zone1 | zone2 | zone3 |
-                                                                |  P P  |  P P  |  P P  |
-                                                                The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
-                                                                In this situation, new pod with the same labelSelector cannot be scheduled,
-                                                                because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
-                                                                it will violate MaxSkew.
-                                                            format: int32
-                                                            type: integer
-                                                        nodeAffinityPolicy:
-                                                            description: |-
-                                                                NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
-                                                                when calculating pod topology spread skew. Options are:
-                                                                - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
-                                                                - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
-
-                                                                If this value is nil, the behavior is equivalent to the Honor policy.
-                                                                This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
-                                                            type: string
-                                                        nodeTaintsPolicy:
-                                                            description: |-
-                                                                NodeTaintsPolicy indicates how we will treat node taints when calculating
-                                                                pod topology spread skew. Options are:
-                                                                - Honor: nodes without taints, along with tainted nodes for which the incoming pod
-                                                                has a toleration, are included.
-                                                                - Ignore: node taints are ignored. All nodes are included.
-
-                                                                If this value is nil, the behavior is equivalent to the Ignore policy.
-                                                                This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
-                                                            type: string
-                                                        topologyKey:
-                                                            description: |-
-                                                                TopologyKey is the key of node labels. Nodes that have a label with this key
-                                                                and identical values are considered to be in the same topology.
-                                                                We consider each <key, value> as a "bucket", and try to put balanced number
-                                                                of pods into each bucket.
-                                                                We define a domain as a particular instance of a topology.
-                                                                Also, we define an eligible domain as a domain whose nodes meet the requirements of
-                                                                nodeAffinityPolicy and nodeTaintsPolicy.
-                                                                e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
-                                                                And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
-                                                                It's a required field.
-                                                            type: string
-                                                        whenUnsatisfiable:
-                                                            description: |-
-                                                                WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
-                                                                the spread constraint.
-                                                                - DoNotSchedule (default) tells the scheduler not to schedule it.
-                                                                - ScheduleAnyway tells the scheduler to schedule the pod in any location,
-                                                                  but giving higher precedence to topologies that would help reduce the
-                                                                  skew.
-                                                                A constraint is considered "Unsatisfiable" for an incoming pod
-                                                                if and only if every possible node assignment for that pod would violate
-                                                                "MaxSkew" on some topology.
-                                                                For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
-                                                                labelSelector spread as 3/1/1:
-                                                                | zone1 | zone2 | zone3 |
-                                                                | P P P |   P   |   P   |
-                                                                If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
-                                                                to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
-                                                                MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
-                                                                won't make it *more* imbalanced.
-                                                                It's a required field.
-                                                            type: string
-                                                    required:
-                                                        - maxSkew
-                                                        - topologyKey
-                                                        - whenUnsatisfiable
-                                                    type: object
-                                                type: array
                                         type: object
                                 type: object
                             tailscale:
@@ -2553,7 +2336,7 @@ spec:
                                             routes advertized by other nodes on the tailnet, such as subnet
                                             routes.
                                             This is equivalent of passing --accept-routes flag to a tailscale Linux client.
-                                            https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
+                                            https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
                                             Defaults to false.
                                         type: boolean
                                 type: object
@@ -2635,194 +2418,6 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
-metadata:
-    annotations:
-        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
-    name: proxygroups.tailscale.com
-spec:
-    group: tailscale.com
-    names:
-        kind: ProxyGroup
-        listKind: ProxyGroupList
-        plural: proxygroups
-        shortNames:
-            - pg
-        singular: proxygroup
-    scope: Cluster
-    versions:
-        - additionalPrinterColumns:
-            - description: Status of the deployed ProxyGroup resources.
-              jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
-              name: Status
-              type: string
-          name: v1alpha1
-          schema:
-            openAPIV3Schema:
-                properties:
-                    apiVersion:
-                        description: |-
-                            APIVersion defines the versioned schema of this representation of an object.
-                            Servers should convert recognized schemas to the latest internal value, and
-                            may reject unrecognized values.
-                            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
-                        type: string
-                    kind:
-                        description: |-
-                            Kind is a string value representing the REST resource this object represents.
-                            Servers may infer this from the endpoint the client submits requests to.
-                            Cannot be updated.
-                            In CamelCase.
-                            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
-                        type: string
-                    metadata:
-                        type: object
-                    spec:
-                        description: Spec describes the desired ProxyGroup instances.
-                        properties:
-                            hostnamePrefix:
-                                description: |-
-                                    HostnamePrefix is the hostname prefix to use for tailnet devices created
-                                    by the ProxyGroup. Each device will have the integer number from its
-                                    StatefulSet pod appended to this prefix to form the full hostname.
-                                    HostnamePrefix can contain lower case letters, numbers and dashes, it
-                                    must not start with a dash and must be between 1 and 62 characters long.
-                                pattern: ^[a-z0-9][a-z0-9-]{0,61}$
-                                type: string
-                            proxyClass:
-                                description: |-
-                                    ProxyClass is the name of the ProxyClass custom resource that contains
-                                    configuration options that should be applied to the resources created
-                                    for this ProxyGroup. If unset, and there is no default ProxyClass
-                                    configured, the operator will create resources with the default
-                                    configuration.
-                                type: string
-                            replicas:
-                                description: |-
-                                    Replicas specifies how many replicas to create the StatefulSet with.
-                                    Defaults to 2.
-                                format: int32
-                                type: integer
-                            tags:
-                                description: |-
-                                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
-                                    If you specify custom tags here, make sure you also make the operator
-                                    an owner of these tags.
-                                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
-                                    Tags cannot be changed once a ProxyGroup device has been created.
-                                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
-                                items:
-                                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
-                                    type: string
-                                type: array
-                            type:
-                                description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                                enum:
-                                    - egress
-                                type: string
-                        required:
-                            - type
-                        type: object
-                    status:
-                        description: |-
-                            ProxyGroupStatus describes the status of the ProxyGroup resources. This is
-                            set and managed by the Tailscale operator.
-                        properties:
-                            conditions:
-                                description: |-
-                                    List of status conditions to indicate the status of the ProxyGroup
-                                    resources. Known condition types are `ProxyGroupReady`.
-                                items:
-                                    description: Condition contains details for one aspect of the current state of this API Resource.
-                                    properties:
-                                        lastTransitionTime:
-                                            description: |-
-                                                lastTransitionTime is the last time the condition transitioned from one status to another.
-                                                This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
-                                            format: date-time
-                                            type: string
-                                        message:
-                                            description: |-
-                                                message is a human readable message indicating details about the transition.
-                                                This may be an empty string.
-                                            maxLength: 32768
-                                            type: string
-                                        observedGeneration:
-                                            description: |-
-                                                observedGeneration represents the .metadata.generation that the condition was set based upon.
-                                                For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
-                                                with respect to the current state of the instance.
-                                            format: int64
-                                            minimum: 0
-                                            type: integer
-                                        reason:
-                                            description: |-
-                                                reason contains a programmatic identifier indicating the reason for the condition's last transition.
-                                                Producers of specific condition types may define expected values and meanings for this field,
-                                                and whether the values are considered a guaranteed API.
-                                                The value should be a CamelCase string.
-                                                This field may not be empty.
-                                            maxLength: 1024
-                                            minLength: 1
-                                            pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
-                                            type: string
-                                        status:
-                                            description: status of the condition, one of True, False, Unknown.
-                                            enum:
-                                                - "True"
-                                                - "False"
-                                                - Unknown
-                                            type: string
-                                        type:
-                                            description: type of condition in CamelCase or in foo.example.com/CamelCase.
-                                            maxLength: 316
-                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                                            type: string
-                                    required:
-                                        - lastTransitionTime
-                                        - message
-                                        - reason
-                                        - status
-                                        - type
-                                    type: object
-                                type: array
-                                x-kubernetes-list-map-keys:
-                                    - type
-                                x-kubernetes-list-type: map
-                            devices:
-                                description: List of tailnet devices associated with the ProxyGroup StatefulSet.
-                                items:
-                                    properties:
-                                        hostname:
-                                            description: |-
-                                                Hostname is the fully qualified domain name of the device.
-                                                If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
-                                                node.
-                                            type: string
-                                        tailnetIPs:
-                                            description: |-
-                                                TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
-                                                assigned to the device.
-                                            items:
-                                                type: string
-                                            type: array
-                                    required:
-                                        - hostname
-                                    type: object
-                                type: array
-                                x-kubernetes-list-map-keys:
-                                    - hostname
-                                x-kubernetes-list-type: map
-                        type: object
-                required:
-                    - spec
-                type: object
-          served: true
-          storage: true
-          subresources:
-            status: {}
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
 metadata:
     annotations:
         controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
@@ -4489,7 +4084,7 @@ spec:
                                     - type
                                 x-kubernetes-list-type: map
                             devices:
-                                description: List of tailnet devices associated with the Recorder StatefulSet.
+                                description: List of tailnet devices associated with the Recorder statefulset.
                                 items:
                                     properties:
                                         hostname:
@@ -4576,8 +4171,6 @@ rules:
         - connectors/status
         - proxyclasses
         - proxyclasses/status
-        - proxygroups
-        - proxygroups/status
       verbs:
         - get
         - list
@@ -4638,14 +4231,6 @@ rules:
         - patch
         - update
         - watch
-    - apiGroups:
-        - ""
-      resources:
-        - pods
-      verbs:
-        - get
-        - list
-        - watch
     - apiGroups:
         - apps
       resources:
@@ -4668,9 +4253,6 @@ rules:
         - get
         - list
         - watch
-        - create
-        - update
-        - deletecollection
     - apiGroups:
         - rbac.authorization.k8s.io
       resources:
@@ -4703,14 +4285,6 @@ rules:
         - patch
         - update
         - watch
-    - apiGroups:
-        - ""
-      resources:
-        - events
-      verbs:
-        - create
-        - patch
-        - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -30,14 +30,6 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_UID
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.uid
           securityContext:
             capabilities:
               add:

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -24,11 +24,3 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_UID
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.uid

cmd/k8s-operator/egress-eps.go

@@ -1,213 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/netip"
-	"reflect"
-	"strings"
-
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/types/ptr"
-)
-
-// egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies.
-type egressEpsReconciler struct {
-	client.Client
-	logger      *zap.SugaredLogger
-	tsNamespace string
-}
-
-// Reconcile reconciles an EndpointSlice for a tailnet service. It updates the EndpointSlice with the endpoints of
-// those ProxyGroup Pods that are ready to route traffic to the tailnet service.
-// It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired
-// configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready.
-func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
-	l := er.logger.With("Service", req.NamespacedName)
-	l.Debugf("starting reconcile")
-	defer l.Debugf("reconcile finished")
-
-	eps := new(discoveryv1.EndpointSlice)
-	err = er.Get(ctx, req.NamespacedName, eps)
-	if apierrors.IsNotFound(err) {
-		l.Debugf("EndpointSlice not found")
-		return reconcile.Result{}, nil
-	}
-	if err != nil {
-		return reconcile.Result{}, fmt.Errorf("failed to get EndpointSlice: %w", err)
-	}
-	if !eps.DeletionTimestamp.IsZero() {
-		l.Debugf("EnpointSlice is being deleted")
-		return res, nil
-	}
-
-	// Get the user-created ExternalName Service and use its status conditions to determine whether cluster
-	// resources are set up for this tailnet service.
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      eps.Labels[LabelParentName],
-			Namespace: eps.Labels[LabelParentNamespace],
-		},
-	}
-	err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
-	if apierrors.IsNotFound(err) {
-		l.Infof("ExternalName Service %s/%s not found, perhaps it was deleted", svc.Namespace, svc.Name)
-		return res, nil
-	}
-	if err != nil {
-		return res, fmt.Errorf("error retrieving ExternalName Service: %w", err)
-	}
-	if !tsoperator.EgressServiceIsValidAndConfigured(svc) {
-		l.Infof("Cluster resources for ExternalName Service %s/%s are not yet configured", svc.Namespace, svc.Name)
-		return res, nil
-	}
-
-	// TODO(irbekrm): currently this reconcile loop runs all the checks every time it's triggered, which is
-	// wasteful. Once we have a Ready condition for ExternalName Services for ProxyGroup, use the condition to
-	// determine if a reconcile is needed.
-
-	oldEps := eps.DeepCopy()
-	proxyGroupName := eps.Labels[labelProxyGroup]
-	tailnetSvc := tailnetSvcName(svc)
-	l = l.With("tailnet-service-name", tailnetSvc)
-
-	// Retrieve the desired tailnet service configuration from the ConfigMap.
-	_, cfgs, err := egressSvcsConfigs(ctx, er.Client, proxyGroupName, er.tsNamespace)
-	if err != nil {
-		return res, fmt.Errorf("error retrieving tailnet services configuration: %w", err)
-	}
-	cfg, ok := (*cfgs)[tailnetSvc]
-	if !ok {
-		l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc)
-		return res, nil
-	}
-
-	// Check which Pods in ProxyGroup are ready to route traffic to this
-	// egress service.
-	podList := &corev1.PodList{}
-	if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil {
-		return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
-	}
-	newEndpoints := make([]discoveryv1.Endpoint, 0)
-	for _, pod := range podList.Items {
-		ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, l)
-		if err != nil {
-			return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err)
-		}
-		if !ready {
-			continue // maybe next time
-		}
-		podIP, err := podIPv4(&pod) // we currently only support IPv4
-		if err != nil {
-			return res, fmt.Errorf("error determining IPv4 address for Pod: %w", err)
-		}
-		newEndpoints = append(newEndpoints, discoveryv1.Endpoint{
-			Hostname:  (*string)(&pod.UID),
-			Addresses: []string{podIP},
-			Conditions: discoveryv1.EndpointConditions{
-				Ready:       ptr.To(true),
-				Serving:     ptr.To(true),
-				Terminating: ptr.To(false),
-			},
-		})
-	}
-	// Note that Endpoints are being overwritten with the currently valid endpoints so we don't need to explicitly
-	// run a cleanup for deleted Pods etc.
-	eps.Endpoints = newEndpoints
-	if !reflect.DeepEqual(eps, oldEps) {
-		l.Infof("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods")
-		if err := er.Update(ctx, eps); err != nil {
-			return res, fmt.Errorf("error updating EndpointSlice: %w", err)
-		}
-	}
-	return res, nil
-}
-
-func podIPv4(pod *corev1.Pod) (string, error) {
-	for _, ip := range pod.Status.PodIPs {
-		parsed, err := netip.ParseAddr(ip.IP)
-		if err != nil {
-			return "", fmt.Errorf("error parsing IP address %s: %w", ip, err)
-		}
-		if parsed.Is4() {
-			return parsed.String(), nil
-		}
-	}
-	return "", nil
-}
-
-// podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to
-// route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service
-// status written there to the desired service configuration.
-func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, l *zap.SugaredLogger) (bool, error) {
-	l = l.With("proxy_pod", pod.Name)
-	l.Debugf("checking whether proxy is ready to route to egress service")
-	if !pod.DeletionTimestamp.IsZero() {
-		l.Debugf("proxy Pod is being deleted, ignore")
-		return false, nil
-	}
-	podIP, err := podIPv4(&pod)
-	if err != nil {
-		return false, fmt.Errorf("error determining Pod IP address: %v", err)
-	}
-	if podIP == "" {
-		l.Infof("[unexpected] Pod does not have an IPv4 address, and IPv6 is not currently supported")
-		return false, nil
-	}
-	stateS := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      pod.Name,
-			Namespace: pod.Namespace,
-		},
-	}
-	err = er.Get(ctx, client.ObjectKeyFromObject(stateS), stateS)
-	if apierrors.IsNotFound(err) {
-		l.Debugf("proxy does not have a state Secret, waiting...")
-		return false, nil
-	}
-	if err != nil {
-		return false, fmt.Errorf("error getting state Secret: %w", err)
-	}
-	svcStatusBS := stateS.Data[egressservices.KeyEgressServices]
-	if len(svcStatusBS) == 0 {
-		l.Debugf("proxy's state Secret does not contain egress services status, waiting...")
-		return false, nil
-	}
-	svcStatus := &egressservices.Status{}
-	if err := json.Unmarshal(svcStatusBS, svcStatus); err != nil {
-		return false, fmt.Errorf("error unmarshalling egress service status: %w", err)
-	}
-	if !strings.EqualFold(podIP, svcStatus.PodIPv4) {
-		l.Infof("proxy's egress service status is for Pod IP %s, current proxy's Pod IP %s, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP)
-		return false, nil
-	}
-	st, ok := (*svcStatus).Services[tailnetSvcName]
-	if !ok {
-		l.Infof("proxy's state Secret does not have egress service status, waiting...")
-		return false, nil
-	}
-	if !reflect.DeepEqual(cfg.TailnetTarget, st.TailnetTarget) {
-		l.Infof("proxy has configured egress service for tailnet target %v, current target is %v, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget)
-		return false, nil
-	}
-	if !reflect.DeepEqual(cfg.Ports, st.Ports) {
-		l.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports)
-		return false, nil
-	}
-	l.Debugf("proxy is ready to route traffic to egress service")
-	return true, nil
-}

cmd/k8s-operator/egress-eps_test.go

@@ -1,211 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"math/rand/v2"
-	"testing"
-
-	"github.com/AlekSi/pointer"
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/tstest"
-	"tailscale.com/util/mak"
-)
-
-func TestTailscaleEgressEndpointSlices(t *testing.T) {
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetFQDN: "foo.bar.ts.net",
-				AnnotationProxyGroup:        "foo",
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ExternalName: "placeholder",
-			Type:         corev1.ServiceTypeExternalName,
-			Selector:     nil,
-			Ports: []corev1.ServicePort{
-				{
-					Name:     "http",
-					Protocol: "TCP",
-					Port:     80,
-				},
-			},
-		},
-		Status: corev1.ServiceStatus{
-			Conditions: []metav1.Condition{
-				condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, "", "", clock),
-				condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "", "", clock),
-			},
-		},
-	}
-	port := randomPort()
-	cm := configMapForSvc(t, svc, port)
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(svc, cm).
-		WithStatusSubresource(svc).
-		Build()
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	er := &egressEpsReconciler{
-		Client:      fc,
-		logger:      zl.Sugar(),
-		tsNamespace: "operator-ns",
-	}
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "foo",
-			Namespace: "operator-ns",
-			Labels: map[string]string{
-				LabelParentName:      "test",
-				LabelParentNamespace: "default",
-				labelSvcType:         typeEgress,
-				labelProxyGroup:      "foo"},
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-	}
-	mustCreate(t, fc, eps)
-
-	t.Run("no_proxy_group_resources", func(t *testing.T) {
-		expectReconciled(t, er, "operator-ns", "foo") // should not error
-	})
-
-	t.Run("no_pods_ready_to_route_traffic", func(t *testing.T) {
-		pod, stateS := podAndSecretForProxyGroup("foo")
-		mustCreate(t, fc, pod)
-		mustCreate(t, fc, stateS)
-		expectReconciled(t, er, "operator-ns", "foo") // should not error
-	})
-
-	t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) {
-		pod, stateS := podAndSecretForProxyGroup("foo")
-		stBs := serviceStatusForPodIP(t, svc, pod.Status.PodIPs[0].IP, port)
-		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
-			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
-		})
-		expectReconciled(t, er, "operator-ns", "foo")
-		eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
-			Addresses: []string{"10.0.0.1"},
-			Hostname:  pointer.To("foo"),
-			Conditions: discoveryv1.EndpointConditions{
-				Serving:     pointer.ToBool(true),
-				Ready:       pointer.ToBool(true),
-				Terminating: pointer.ToBool(false),
-			},
-		})
-		expectEqual(t, fc, eps, nil)
-	})
-	t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
-		_, stateS := podAndSecretForProxyGroup("foo")           // replica Pod has IP 10.0.0.1
-		stBs := serviceStatusForPodIP(t, svc, "10.0.0.2", port) // status is for a Pod with IP 10.0.0.2
-		mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
-			mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
-		})
-		expectReconciled(t, er, "operator-ns", "foo")
-		eps.Endpoints = []discoveryv1.Endpoint{}
-		expectEqual(t, fc, eps, nil)
-	})
-}
-
-func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.ConfigMap {
-	t.Helper()
-	ports := make(map[egressservices.PortMap]struct{})
-	for _, port := range svc.Spec.Ports {
-		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
-	}
-	cfg := egressservices.Config{
-		Ports: ports,
-	}
-	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
-		cfg.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
-	}
-	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
-		cfg.TailnetTarget = egressservices.TailnetTarget{IP: ip}
-	}
-	name := tailnetSvcName(svc)
-	cfgs := egressservices.Configs{name: cfg}
-	bs, err := json.Marshal(&cfgs)
-	if err != nil {
-		t.Fatalf("error marshalling config: %v", err)
-	}
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      pgEgressCMName(svc.Annotations[AnnotationProxyGroup]),
-			Namespace: "operator-ns",
-		},
-		BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs},
-	}
-	return cm
-}
-
-func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint16) []byte {
-	t.Helper()
-	ports := make(map[egressservices.PortMap]struct{})
-	for _, port := range svc.Spec.Ports {
-		ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
-	}
-	svcSt := egressservices.ServiceStatus{Ports: ports}
-	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
-		svcSt.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
-	}
-	if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
-		svcSt.TailnetTarget = egressservices.TailnetTarget{IP: ip}
-	}
-	svcName := tailnetSvcName(svc)
-	st := egressservices.Status{
-		PodIPv4:  ip,
-		Services: map[string]*egressservices.ServiceStatus{svcName: &svcSt},
-	}
-	bs, err := json.Marshal(st)
-	if err != nil {
-		t.Fatalf("error marshalling service status: %v", err)
-	}
-	return bs
-}
-
-func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
-	p := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-0", pg),
-			Namespace: "operator-ns",
-			Labels:    pgLabels(pg, nil),
-			UID:       "foo",
-		},
-		Status: corev1.PodStatus{
-			PodIPs: []corev1.PodIP{
-				{IP: "10.0.0.1"},
-			},
-		},
-	}
-	s := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-0", pg),
-			Namespace: "operator-ns",
-			Labels:    pgSecretLabels(pg, "state"),
-		},
-	}
-	return p, s
-}
-
-func randomPort() uint16 {
-	return uint16(rand.Int32N(1000) + 1000)
-}

cmd/k8s-operator/egress-services-readiness.go

@@ -1,179 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-
-	"go.uber.org/zap"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/tstime"
-)
-
-const (
-	reasonReadinessCheckFailed     = "ReadinessCheckFailed"
-	reasonClusterResourcesNotReady = "ClusterResourcesNotReady"
-	reasonNoProxies                = "NoProxiesConfigured"
-	reasonNotReady                 = "NotReadyToRouteTraffic"
-	reasonReady                    = "ReadyToRouteTraffic"
-	reasonPartiallyReady           = "PartiallyReadyToRouteTraffic"
-	msgReadyToRouteTemplate        = "%d out of %d replicas are ready to route traffic"
-)
-
-type egressSvcsReadinessReconciler struct {
-	client.Client
-	logger      *zap.SugaredLogger
-	clock       tstime.Clock
-	tsNamespace string
-}
-
-// Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the
-// EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to
-// route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress
-// service to determine how many replicas are currently able to route traffic.
-func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
-	l := esrr.logger.With("Service", req.NamespacedName)
-	defer l.Info("reconcile finished")
-
-	svc := new(corev1.Service)
-	if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
-		l.Info("Service not found")
-		return res, nil
-	} else if err != nil {
-		return res, fmt.Errorf("failed to get Service: %w", err)
-	}
-	var (
-		reason, msg string
-		st          metav1.ConditionStatus = metav1.ConditionUnknown
-	)
-	oldStatus := svc.Status.DeepCopy()
-	defer func() {
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
-		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
-			err = errors.Join(err, esrr.Status().Update(ctx, svc))
-		}
-	}()
-
-	crl := egressSvcChildResourceLabels(svc)
-	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl)
-	if err != nil {
-		err = fmt.Errorf("error getting EndpointSlice: %w", err)
-		reason = reasonReadinessCheckFailed
-		msg = err.Error()
-		return res, err
-	}
-	if eps == nil {
-		l.Infof("EndpointSlice for Service does not yet exist, waiting...")
-		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: svc.Annotations[AnnotationProxyGroup],
-		},
-	}
-	err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg)
-	if apierrors.IsNotFound(err) {
-		l.Infof("ProxyGroup for Service does not exist, waiting...")
-		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	if err != nil {
-		err = fmt.Errorf("error retrieving ProxyGroup: %w", err)
-		reason = reasonReadinessCheckFailed
-		msg = err.Error()
-		return res, err
-	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup for Service is not ready, waiting...")
-		reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-
-	replicas := pgReplicas(pg)
-	if replicas == 0 {
-		l.Infof("ProxyGroup replicas set to 0")
-		reason, msg = reasonNoProxies, reasonNoProxies
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	podLabels := pgLabels(pg.Name, nil)
-	var readyReplicas int32
-	for i := range replicas {
-		podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
-		pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
-		if err != nil {
-			err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err)
-			reason = reasonReadinessCheckFailed
-			msg = err.Error()
-			return res, err
-		}
-		if pod == nil {
-			l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
-			reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
-			return res, nil
-		}
-		l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
-		ready := false
-		for _, ep := range eps.Endpoints {
-			l.Infof("looking at endpoint with addresses %v", ep.Addresses)
-			if endpointReadyForPod(&ep, pod, l) {
-				l.Infof("endpoint is ready for Pod")
-				ready = true
-				break
-			}
-		}
-		if ready {
-			readyReplicas++
-		}
-	}
-	msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
-	if readyReplicas == 0 {
-		reason = reasonNotReady
-		st = metav1.ConditionFalse
-		return res, nil
-	}
-	st = metav1.ConditionTrue
-	if readyReplicas < replicas {
-		reason = reasonPartiallyReady
-	} else {
-		reason = reasonReady
-	}
-	return res, nil
-}
-
-// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic.
-// Endpoint must not be nil.
-func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
-	podIP, err := podIPv4(pod)
-	if err != nil {
-		l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
-		return false
-	}
-	// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.
-	if len(ep.Addresses) != 1 {
-		return false
-	}
-	return strings.EqualFold(ep.Addresses[0], podIP) &&
-		*ep.Conditions.Ready &&
-		*ep.Conditions.Serving &&
-		!*ep.Conditions.Terminating
-}

cmd/k8s-operator/egress-services-readiness_test.go

@@ -1,169 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/AlekSi/pointer"
-	"go.uber.org/zap"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/tstest"
-	"tailscale.com/tstime"
-)
-
-func TestEgressServiceReadiness(t *testing.T) {
-	// We need to pass a ProxyGroup object to WithStatusSubresource because of some quirks in how the fake client
-	// works. Without this code further down would not be able to update ProxyGroup status.
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithStatusSubresource(&tsapi.ProxyGroup{}).
-		Build()
-	zl, _ := zap.NewDevelopment()
-	cl := tstest.NewClock(tstest.ClockOpts{})
-	rec := &egressSvcsReadinessReconciler{
-		tsNamespace: "operator-ns",
-		Client:      fc,
-		logger:      zl.Sugar(),
-		clock:       cl,
-	}
-	tailnetFQDN := "my-app.tailnetxyz.ts.net"
-	egressSvc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "my-app",
-			Namespace: "dev",
-			Annotations: map[string]string{
-				AnnotationProxyGroup:        "dev",
-				AnnotationTailnetTargetFQDN: tailnetFQDN,
-			},
-		},
-	}
-	fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}}
-	l := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "my-app",
-			Namespace: "operator-ns",
-			Labels:    l,
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-	}
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "dev",
-		},
-	}
-	mustCreate(t, fc, egressSvc)
-	setClusterNotReady(egressSvc, cl, zl.Sugar())
-	t.Run("endpointslice_does_not_exist", func(t *testing.T) {
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // not ready
-	})
-	t.Run("proxy_group_does_not_exist", func(t *testing.T) {
-		mustCreate(t, fc, eps)
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // still not ready
-	})
-	t.Run("proxy_group_not_ready", func(t *testing.T) {
-		mustCreate(t, fc, pg)
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // still not ready
-	})
-	t.Run("no_ready_replicas", func(t *testing.T) {
-		setPGReady(pg, cl, zl.Sugar())
-		mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
-			p.Status = pg.Status
-		})
-		expectEqual(t, fc, pg, nil)
-		for i := range pgReplicas(pg) {
-			p := pod(pg, i)
-			mustCreate(t, fc, p)
-			mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
-				existing.Status.PodIPs = p.Status.PodIPs
-			})
-		}
-		expectReconciled(t, rec, "dev", "my-app")
-		setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
-		expectEqual(t, fc, egressSvc, nil) // still not ready
-	})
-	t.Run("one_ready_replica", func(t *testing.T) {
-		setEndpointForReplica(pg, 0, eps)
-		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
-			e.Endpoints = eps.Endpoints
-		})
-		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // partially ready
-	})
-	t.Run("all_replicas_ready", func(t *testing.T) {
-		for i := range pgReplicas(pg) {
-			setEndpointForReplica(pg, i, eps)
-		}
-		mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
-			e.Endpoints = eps.Endpoints
-		})
-		setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
-		expectReconciled(t, rec, "dev", "my-app")
-		expectEqual(t, fc, egressSvc, nil) // ready
-	})
-}
-
-func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger) {
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, l)
-}
-
-func setNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas int32) {
-	msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas)
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, l)
-}
-
-func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas, readyReplicas int32) {
-	reason := reasonPartiallyReady
-	if readyReplicas == replicas {
-		reason = reasonReady
-	}
-	msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, l)
-}
-
-func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) {
-	tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l)
-}
-
-func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {
-	p := pod(pg, ordinal)
-	eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
-		Addresses: []string{p.Status.PodIPs[0].IP},
-		Conditions: discoveryv1.EndpointConditions{
-			Ready:       pointer.ToBool(true),
-			Serving:     pointer.ToBool(true),
-			Terminating: pointer.ToBool(false),
-		},
-	})
-}
-
-func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
-	l := pgLabels(pg.Name, nil)
-	l[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
-	ip := fmt.Sprintf("10.0.0.%d", ordinal)
-	return &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%d", pg.Name, ordinal),
-			Namespace: "operator-ns",
-			Labels:    l,
-		},
-		Status: corev1.PodStatus{
-			PodIPs: []corev1.PodIP{{IP: ip}},
-		},
-	}
-}

cmd/k8s-operator/egress-services.go

@@ -1,716 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math/rand/v2"
-	"reflect"
-	"slices"
-	"strings"
-	"sync"
-
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apiserver/pkg/storage/names"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/kube/kubetypes"
-	"tailscale.com/tstime"
-	"tailscale.com/util/clientmetric"
-	"tailscale.com/util/mak"
-	"tailscale.com/util/set"
-)
-
-const (
-	reasonEgressSvcInvalid        = "EgressSvcInvalid"
-	reasonEgressSvcValid          = "EgressSvcValid"
-	reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
-	reasonProxyGroupNotReady      = "ProxyGroupNotReady"
-
-	labelProxyGroup = "tailscale.com/proxy-group"
-
-	labelSvcType = "tailscale.com/svc-type" // ingress or egress
-	typeEgress   = "egress"
-	// maxPorts is the maximum number of ports that can be exposed on a
-	// container. In practice this will be ports in range [3000 - 4000). The
-	// high range should make it easier to distinguish container ports from
-	// the tailnet target ports for debugging purposes (i.e when reading
-	// netfilter rules). The limit of 10000 is somewhat arbitrary, the
-	// assumption is that this would not be hit in practice.
-	maxPorts = 10000
-
-	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
-)
-
-var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
-
-// egressSvcsReconciler reconciles user created ExternalName Services that specify a tailnet
-// endpoint that should be exposed to cluster workloads and an egress ProxyGroup
-// on whose proxies it should be exposed.
-type egressSvcsReconciler struct {
-	client.Client
-	logger      *zap.SugaredLogger
-	recorder    record.EventRecorder
-	clock       tstime.Clock
-	tsNamespace string
-
-	mu   sync.Mutex           // protects following
-	svcs set.Slice[types.UID] // UIDs of all currently managed egress Services for ProxyGroup
-}
-
-// Reconcile reconciles an ExternalName Service that specifies a tailnet target and a ProxyGroup on whose proxies should
-// forward cluster traffic to the target.
-// For an ExternalName Service the reconciler:
-//
-// - for each port N defined on the ExternalName Service, allocates a port X in range [3000- 4000), unique for the
-// ProxyGroup proxies. Proxies will forward cluster traffic received on port N to port M on the tailnet target
-//
-// - creates a ClusterIP Service in the operator's namespace with portmappings for all M->N port pairs. This will allow
-// cluster workloads to send traffic on the user-defined tailnet target port and get it transparently mapped to the
-// randomly selected port on proxy Pods.
-//
-// - creates an EndpointSlice in the operator's namespace with kubernetes.io/service-name label pointing to the
-// ClusterIP Service. The endpoints will get dynamically updates to proxy Pod IPs as the Pods become ready to route
-// traffic to the tailnet target. kubernetes.io/service-name label ensures that kube-proxy sets up routing rules to
-// forward cluster traffic received on ClusterIP Service's IP address to the endpoints (Pod IPs).
-//
-// - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the
-// portmappings.
-func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
-	l := esr.logger.With("Service", req.NamespacedName)
-	defer l.Info("reconcile finished")
-
-	svc := new(corev1.Service)
-	if err = esr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
-		l.Info("Service not found")
-		return res, nil
-	} else if err != nil {
-		return res, fmt.Errorf("failed to get Service: %w", err)
-	}
-
-	// Name of the 'egress service', meaning the tailnet target.
-	tailnetSvc := tailnetSvcName(svc)
-	l = l.With("tailnet-service", tailnetSvc)
-
-	// Note that resources for egress Services are only cleaned up when the
-	// Service is actually deleted (and not if, for example, user decides to
-	// remove the Tailscale annotation from it). This should be fine- we
-	// assume that the egress ExternalName Services are always created for
-	// Tailscale operator specifically.
-	if !svc.DeletionTimestamp.IsZero() {
-		l.Info("Service is being deleted, ensuring resource cleanup")
-		return res, esr.maybeCleanup(ctx, svc, l)
-	}
-
-	oldStatus := svc.Status.DeepCopy()
-	defer func() {
-		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
-			err = errors.Join(err, esr.Status().Update(ctx, svc))
-		}
-	}()
-
-	// Validate the user-created ExternalName Service and the associated ProxyGroup.
-	if ok, err := esr.validateClusterResources(ctx, svc, l); err != nil {
-		return res, fmt.Errorf("error validating cluster resources: %w", err)
-	} else if !ok {
-		return res, nil
-	}
-
-	if !slices.Contains(svc.Finalizers, FinalizerName) {
-		l.Infof("configuring tailnet service") // logged exactly once
-		svc.Finalizers = append(svc.Finalizers, FinalizerName)
-		if err := esr.Update(ctx, svc); err != nil {
-			err := fmt.Errorf("failed to add finalizer: %w", err)
-			r := svcConfiguredReason(svc, false, l)
-			tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
-			return res, err
-		}
-		esr.mu.Lock()
-		esr.svcs.Add(svc.UID)
-		gaugeEgressServices.Set(int64(esr.svcs.Len()))
-		esr.mu.Unlock()
-	}
-
-	if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, l); err != nil {
-		err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err)
-		r := svcConfiguredReason(svc, false, l)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
-		return res, err
-	}
-
-	return res, esr.maybeProvision(ctx, svc, l)
-}
-
-func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) {
-	r := svcConfiguredReason(svc, false, l)
-	st := metav1.ConditionFalse
-	defer func() {
-		msg := r
-		if st != metav1.ConditionTrue && err != nil {
-			msg = err.Error()
-		}
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, l)
-	}()
-
-	crl := egressSvcChildResourceLabels(svc)
-	clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl)
-	if err != nil {
-		err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
-		return err
-	}
-	if clusterIPSvc == nil {
-		clusterIPSvc = esr.clusterIPSvcForEgress(crl)
-	}
-	upToDate := svcConfigurationUpToDate(svc, l)
-	provisioned := true
-	if !upToDate {
-		if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, l); err != nil {
-			return err
-		}
-	}
-	if !provisioned {
-		l.Infof("unable to provision cluster resources")
-		return nil
-	}
-
-	// Update ExternalName Service to point at the ClusterIP Service.
-	clusterDomain := retrieveClusterDomain(esr.tsNamespace, l)
-	clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain)
-	if svc.Spec.ExternalName != clusterIPSvcFQDN {
-		l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
-		svc.Spec.ExternalName = clusterIPSvcFQDN
-		if err = esr.Update(ctx, svc); err != nil {
-			err = fmt.Errorf("error updating ExternalName Service: %w", err)
-			return err
-		}
-	}
-	r = svcConfiguredReason(svc, true, l)
-	st = metav1.ConditionTrue
-	return nil
-}
-
-func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, l *zap.SugaredLogger) (*corev1.Service, bool, error) {
-	l.Infof("updating configuration...")
-	usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName)
-	if err != nil {
-		return nil, false, fmt.Errorf("error calculating used ports for ProxyGroup %s: %w", proxyGroupName, err)
-	}
-
-	oldClusterIPSvc := clusterIPSvc.DeepCopy()
-	// loop over ClusterIP Service ports, remove any that are not needed.
-	for i := len(clusterIPSvc.Spec.Ports) - 1; i >= 0; i-- {
-		pm := clusterIPSvc.Spec.Ports[i]
-		found := false
-		for _, wantsPM := range svc.Spec.Ports {
-			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			l.Debugf("portmapping %s:%d -> %s:%d is no longer required, removing", pm.Protocol, pm.TargetPort.IntVal, pm.Protocol, pm.Port)
-			clusterIPSvc.Spec.Ports = slices.Delete(clusterIPSvc.Spec.Ports, i, i+1)
-		}
-	}
-
-	// loop over ExternalName Service ports, for each one not found on
-	// ClusterIP Service produce new target port and add a portmapping to
-	// the ClusterIP Service.
-	for _, wantsPM := range svc.Spec.Ports {
-		found := false
-		for _, gotPM := range clusterIPSvc.Spec.Ports {
-			if wantsPM.Port == gotPM.Port && strings.EqualFold(string(wantsPM.Protocol), string(gotPM.Protocol)) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			// Calculate a free port to expose on container and add
-			// a new PortMap to the ClusterIP Service.
-			if usedPorts.Len() == maxPorts {
-				// TODO(irbekrm): refactor to avoid extra reconciles here. Low priority as in practice,
-				// the limit should not be hit.
-				return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts)
-			}
-			p := unusedPort(usedPorts)
-			l.Debugf("mapping tailnet target port %d to container port %d", wantsPM.Port, p)
-			usedPorts.Insert(p)
-			clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
-				Name:       wantsPM.Name,
-				Protocol:   wantsPM.Protocol,
-				Port:       wantsPM.Port,
-				TargetPort: intstr.FromInt32(p),
-			})
-		}
-	}
-	if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
-		if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
-			svc.Labels = clusterIPSvc.Labels
-			svc.Spec = clusterIPSvc.Spec
-		}); err != nil {
-			return nil, false, fmt.Errorf("error ensuring ClusterIP Service: %v", err)
-		}
-	}
-
-	crl := egressSvcEpsLabels(svc, clusterIPSvc)
-	// TODO(irbekrm): support IPv6, but need to investigate how kube proxy
-	// sets up Service -> Pod routing when IPv6 is involved.
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
-			Namespace: esr.tsNamespace,
-			Labels:    crl,
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-		Ports:       epsPortsFromSvc(clusterIPSvc),
-	}
-	if eps, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
-		e.Labels = eps.Labels
-		e.AddressType = eps.AddressType
-		e.Ports = eps.Ports
-		for _, p := range e.Endpoints {
-			p.Conditions.Ready = nil
-		}
-	}); err != nil {
-		return nil, false, fmt.Errorf("error ensuring EndpointSlice: %w", err)
-	}
-
-	cm, cfgs, err := egressSvcsConfigs(ctx, esr.Client, proxyGroupName, esr.tsNamespace)
-	if err != nil {
-		return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err)
-	}
-	if cm == nil {
-		l.Info("ConfigMap not yet created, waiting..")
-		return nil, false, nil
-	}
-	tailnetSvc := tailnetSvcName(svc)
-	gotCfg := (*cfgs)[tailnetSvc]
-	wantsCfg := egressSvcCfg(svc, clusterIPSvc)
-	if !reflect.DeepEqual(gotCfg, wantsCfg) {
-		l.Debugf("updating egress services ConfigMap %s", cm.Name)
-		mak.Set(cfgs, tailnetSvc, wantsCfg)
-		bs, err := json.Marshal(cfgs)
-		if err != nil {
-			return nil, false, fmt.Errorf("error marshalling egress services configs: %w", err)
-		}
-		mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
-		if err := esr.Update(ctx, cm); err != nil {
-			return nil, false, fmt.Errorf("error updating egress services ConfigMap: %w", err)
-		}
-	}
-	l.Infof("egress service configuration has been updated")
-	return clusterIPSvc, true, nil
-}
-
-func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
-	logger.Info("ensuring that resources created for egress service are deleted")
-
-	// Delete egress service config from the ConfigMap mounted by the proxies.
-	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, logger); err != nil {
-		return fmt.Errorf("error deleting egress service config: %w", err)
-	}
-
-	// Delete the ClusterIP Service and EndpointSlice for the egress
-	// service.
-	types := []client.Object{
-		&corev1.Service{},
-		&discoveryv1.EndpointSlice{},
-	}
-	crl := egressSvcChildResourceLabels(svc)
-	for _, typ := range types {
-		if err := esr.DeleteAllOf(ctx, typ, client.InNamespace(esr.tsNamespace), client.MatchingLabels(crl)); err != nil {
-			return fmt.Errorf("error deleting %s: %w", typ, err)
-		}
-	}
-
-	ix := slices.Index(svc.Finalizers, FinalizerName)
-	if ix != -1 {
-		logger.Debug("Removing Tailscale finalizer from Service")
-		svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
-		if err := esr.Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to remove finalizer: %w", err)
-		}
-	}
-	esr.mu.Lock()
-	esr.svcs.Remove(svc.UID)
-	gaugeEgressServices.Set(int64(esr.svcs.Len()))
-	esr.mu.Unlock()
-	logger.Info("successfully cleaned up resources for egress Service")
-	return nil
-}
-
-func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) error {
-	wantsProxyGroup := svc.Annotations[AnnotationProxyGroup]
-	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
-	if cond == nil {
-		return nil
-	}
-	ss := strings.Split(cond.Reason, ":")
-	if len(ss) < 3 {
-		return nil
-	}
-	if strings.EqualFold(wantsProxyGroup, ss[2]) {
-		return nil
-	}
-	esr.logger.Infof("egress Service configured on ProxyGroup %s, wants ProxyGroup %s, cleaning up...", ss[2], wantsProxyGroup)
-	if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, l); err != nil {
-		return fmt.Errorf("error deleting egress service config: %w", err)
-	}
-	return nil
-}
-
-// usedPortsForPG calculates the currently used match ports for ProxyGroup
-// containers. It does that by looking by retrieving all target ports of all
-// ClusterIP Services created for egress services exposed on this ProxyGroup's
-// proxies.
-// TODO(irbekrm): this is currently good enough because we only have a single worker and
-// because these Services are created by us, so we can always expect to get the
-// latest ClusterIP Services via the controller cache. It will not work as well
-// once we split into multiple workers- at that point we probably want to set
-// used ports on ProxyGroup's status.
-func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) {
-	svcList := &corev1.ServiceList{}
-	if err := esr.List(ctx, svcList, client.InNamespace(esr.tsNamespace), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
-		return nil, fmt.Errorf("error listing Services: %w", err)
-	}
-	usedPorts := sets.New[int32]()
-	for _, s := range svcList.Items {
-		for _, p := range s.Spec.Ports {
-			usedPorts.Insert(p.TargetPort.IntVal)
-		}
-	}
-	return usedPorts, nil
-}
-
-// clusterIPSvcForEgress returns a template for the ClusterIP Service created
-// for an egress service exposed on ProxyGroup proxies. The ClusterIP Service
-// has no selector. Traffic sent to it will be routed to the endpoints defined
-// by an EndpointSlice created for this egress service.
-func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
-	return &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: svcNameBase(crl[LabelParentName]),
-			Namespace:    esr.tsNamespace,
-			Labels:       crl,
-		},
-		Spec: corev1.ServiceSpec{
-			Type: corev1.ServiceTypeClusterIP,
-		},
-	}
-}
-
-func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
-	crl := egressSvcChildResourceLabels(svc)
-	cmName := pgEgressCMName(crl[labelProxyGroup])
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      cmName,
-			Namespace: esr.tsNamespace,
-		},
-	}
-	l := logger.With("ConfigMap", client.ObjectKeyFromObject(cm))
-	l.Debug("ensuring that egress service configuration is removed from proxy config")
-	if err := esr.Get(ctx, client.ObjectKeyFromObject(cm), cm); apierrors.IsNotFound(err) {
-		l.Debugf("ConfigMap not found")
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("error retrieving ConfigMap: %w", err)
-	}
-	bs := cm.BinaryData[egressservices.KeyEgressServices]
-	if len(bs) == 0 {
-		l.Debugf("ConfigMap does not contain egress service configs")
-		return nil
-	}
-	cfgs := &egressservices.Configs{}
-	if err := json.Unmarshal(bs, cfgs); err != nil {
-		return fmt.Errorf("error unmarshalling egress services configs")
-	}
-	tailnetSvc := tailnetSvcName(svc)
-	_, ok := (*cfgs)[tailnetSvc]
-	if !ok {
-		l.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted")
-		return nil
-	}
-	l.Infof("before deleting config %+#v", *cfgs)
-	delete(*cfgs, tailnetSvc)
-	l.Infof("after deleting config %+#v", *cfgs)
-	bs, err := json.Marshal(cfgs)
-	if err != nil {
-		return fmt.Errorf("error marshalling egress services configs: %w", err)
-	}
-	mak.Set(&cm.BinaryData, egressservices.KeyEgressServices, bs)
-	return esr.Update(ctx, cm)
-}
-
-func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (bool, error) {
-	proxyGroupName := svc.Annotations[AnnotationProxyGroup]
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: proxyGroupName,
-		},
-	}
-	if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
-		l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	} else if err != nil {
-		err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, err
-	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-
-	if violations := validateEgressService(svc, pg); len(violations) > 0 {
-		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
-		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
-		l.Info(msg)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-	l.Debugf("egress service is valid")
-	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
-	return true, nil
-}
-
-func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
-	violations := validateService(svc)
-
-	// We check that only one of these two is set in the earlier validateService function.
-	if svc.Annotations[AnnotationTailnetTargetFQDN] == "" && svc.Annotations[AnnotationTailnetTargetIP] == "" {
-		violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", AnnotationTailnetTargetFQDN, AnnotationTailnetTargetIP))
-	}
-	if len(svc.Spec.Ports) == 0 {
-		violations = append(violations, "egress Service for ProxyGroup must have at least one target Port specified")
-	}
-	if svc.Spec.Type != corev1.ServiceTypeExternalName {
-		violations = append(violations, fmt.Sprintf("unexpected egress Service type %s. The only supported type is ExternalName.", svc.Spec.Type))
-	}
-	if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
-		violations = append(violations, fmt.Sprintf("egress Service references ProxyGroup of type %s, must be type %s", pg.Spec.Type, tsapi.ProxyGroupTypeEgress))
-	}
-	return violations
-}
-
-// egressSvcNameBase returns a name base that can be passed to
-// ObjectMeta.GenerateName to generate a name for the ClusterIP Service.
-// The generated name needs to be short enough so that it can later be used to
-// generate a valid Kubernetes resource name for the EndpointSlice in form
-// 'ipv4-|ipv6-<ClusterIP Service name>.
-// A valid Kubernetes resource name must not be longer than 253 chars.
-func svcNameBase(s string) string {
-	// -ipv4 - ipv6
-	const maxClusterIPSvcNameLength = 253 - 5
-	base := fmt.Sprintf("ts-%s-", s)
-	generator := names.SimpleNameGenerator
-	for {
-		generatedName := generator.GenerateName(base)
-		excess := len(generatedName) - maxClusterIPSvcNameLength
-		if excess <= 0 {
-			return base
-		}
-		base = base[:len(base)-1-excess] // cut off the excess chars
-		base = base + "-"                // re-instate the dash
-	}
-}
-
-// unusedPort returns a port in range [3000 - 4000). The caller must ensure that
-// usedPorts does not contain all ports in range [3000 - 4000).
-func unusedPort(usedPorts sets.Set[int32]) int32 {
-	foundFreePort := false
-	var suggestPort int32
-	for !foundFreePort {
-		suggestPort = rand.Int32N(maxPorts) + 3000
-		if !usedPorts.Has(suggestPort) {
-			foundFreePort = true
-		}
-	}
-	return suggestPort
-}
-
-// tailnetTargetFromSvc returns a tailnet target for the given egress Service.
-// Service must contain exactly one of tailscale.com/tailnet-ip,
-// tailscale.com/tailnet-fqdn annotations.
-func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
-	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
-		return egressservices.TailnetTarget{
-			FQDN: fqdn,
-		}
-	}
-	return egressservices.TailnetTarget{
-		IP: svc.Annotations[AnnotationTailnetTargetIP],
-	}
-}
-
-func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service) egressservices.Config {
-	tt := tailnetTargetFromSvc(externalNameSvc)
-	cfg := egressservices.Config{TailnetTarget: tt}
-	for _, svcPort := range clusterIPSvc.Spec.Ports {
-		pm := portMap(svcPort)
-		mak.Set(&cfg.Ports, pm, struct{}{})
-	}
-	return cfg
-}
-
-func portMap(p corev1.ServicePort) egressservices.PortMap {
-	// TODO (irbekrm): out of bounds check?
-	return egressservices.PortMap{Protocol: string(p.Protocol), MatchPort: uint16(p.TargetPort.IntVal), TargetPort: uint16(p.Port)}
-}
-
-func isEgressSvcForProxyGroup(obj client.Object) bool {
-	s, ok := obj.(*corev1.Service)
-	if !ok {
-		return false
-	}
-	annots := s.ObjectMeta.Annotations
-	return annots[AnnotationProxyGroup] != "" && (annots[AnnotationTailnetTargetFQDN] != "" || annots[AnnotationTailnetTargetIP] != "")
-}
-
-// egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well
-// as unmarshalled configuration from the ConfigMap.
-func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) {
-	name := pgEgressCMName(proxyGroupName)
-	cm = &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: tsNamespace,
-		},
-	}
-	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
-		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
-	}
-	cfgs = &egressservices.Configs{}
-	if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 {
-		if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], cfgs); err != nil {
-			return nil, nil, fmt.Errorf("error unmarshaling egress services config %v: %w", cm.BinaryData[egressservices.KeyEgressServices], err)
-		}
-	}
-	return cm, cfgs, nil
-}
-
-// egressSvcChildResourceLabels returns labels that should be applied to the
-// ClusterIP Service and the EndpointSlice created for the egress service.
-// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes
-// resource names (ProxyGroup, Service). Maximum allowed label length is 63
-// chars whilst the maximum allowed resource name length is 253 chars, so we
-// should probably validate and truncate (?) the names is they are too long.
-func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
-	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentType:      "svc",
-		LabelParentName:      svc.Name,
-		LabelParentNamespace: svc.Namespace,
-		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
-		labelSvcType:         typeEgress,
-	}
-}
-
-// egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service.
-func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string {
-	l := egressSvcChildResourceLabels(extNSvc)
-	// Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the
-	// endpoints defined on this EndpointSlice.
-	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
-	l[discoveryv1.LabelServiceName] = clusterIPSvc.Name
-	// Kubernetes recommends setting this label.
-	// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management
-	l[discoveryv1.LabelManagedBy] = "tailscale.com"
-	return l
-}
-
-func svcConfigurationUpToDate(svc *corev1.Service, l *zap.SugaredLogger) bool {
-	cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
-	if cond == nil {
-		return false
-	}
-	if cond.Status != metav1.ConditionTrue {
-		return false
-	}
-	wantsReadyReason := svcConfiguredReason(svc, true, l)
-	return strings.EqualFold(wantsReadyReason, cond.Reason)
-}
-
-func cfgHash(c cfg, l *zap.SugaredLogger) string {
-	bs, err := json.Marshal(c)
-	if err != nil {
-		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
-		l.Infof("error marhsalling Config: %v", err)
-		return ""
-	}
-	h := sha256.New()
-	if _, err := h.Write(bs); err != nil {
-		// Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace.
-		l.Infof("error producing Config hash: %v", err)
-		return ""
-	}
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-type cfg struct {
-	Ports         []corev1.ServicePort         `json:"ports"`
-	TailnetTarget egressservices.TailnetTarget `json:"tailnetTarget"`
-	ProxyGroup    string                       `json:"proxyGroup"`
-}
-
-func svcConfiguredReason(svc *corev1.Service, configured bool, l *zap.SugaredLogger) string {
-	var r string
-	if configured {
-		r = "ConfiguredFor:"
-	} else {
-		r = fmt.Sprintf("ConfigurationFailed:%s", r)
-	}
-	r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[AnnotationProxyGroup])
-	tt := tailnetTargetFromSvc(svc)
-	s := cfg{
-		Ports:         svc.Spec.Ports,
-		TailnetTarget: tt,
-		ProxyGroup:    svc.Annotations[AnnotationProxyGroup],
-	}
-	r += fmt.Sprintf(":Config:%s", cfgHash(s, l))
-	return r
-}
-
-// tailnetSvc accepts and ExternalName Service name and returns a name that will be used to distinguish this tailnet
-// service from other tailnet services exposed to cluster workloads.
-func tailnetSvcName(extNSvc *corev1.Service) string {
-	return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name)
-}
-
-// epsPortsFromSvc takes the ClusterIP Service created for an egress service and
-// returns its Port array in a form that can be used for an EndpointSlice.
-func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
-	for _, p := range svc.Spec.Ports {
-		ep = append(ep, discoveryv1.EndpointPort{
-			Protocol: &p.Protocol,
-			Port:     &p.TargetPort.IntVal,
-			Name:     &p.Name,
-		})
-	}
-	return ep
-}

cmd/k8s-operator/egress-services_test.go

@@ -1,268 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"testing"
-
-	"github.com/AlekSi/pointer"
-	"github.com/google/go-cmp/cmp"
-	"go.uber.org/zap"
-	corev1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/tstest"
-	"tailscale.com/tstime"
-)
-
-func TestTailscaleEgressServices(t *testing.T) {
-	pg := &tsapi.ProxyGroup{
-		TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "foo",
-			UID:  types.UID("1234-UID"),
-		},
-		Spec: tsapi.ProxyGroupSpec{
-			Replicas: pointer.To[int32](3),
-			Type:     tsapi.ProxyGroupTypeEgress,
-		},
-	}
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      pgEgressCMName("foo"),
-			Namespace: "operator-ns",
-		},
-	}
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pg, cm).
-		WithStatusSubresource(pg).
-		Build()
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-
-	esr := &egressSvcsReconciler{
-		Client:      fc,
-		logger:      zl.Sugar(),
-		clock:       clock,
-		tsNamespace: "operator-ns",
-	}
-	tailnetTargetFQDN := "foo.bar.ts.net."
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
-				AnnotationProxyGroup:        "foo",
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ExternalName: "placeholder",
-			Type:         corev1.ServiceTypeExternalName,
-			Selector:     nil,
-			Ports: []corev1.ServicePort{
-				{
-					Name:     "http",
-					Protocol: "TCP",
-					Port:     80,
-				},
-				{
-					Name:     "https",
-					Protocol: "TCP",
-					Port:     443,
-				},
-			},
-		},
-	}
-
-	t.Run("proxy_group_not_ready", func(t *testing.T) {
-		mustCreate(t, fc, svc)
-		expectReconciled(t, esr, "default", "test")
-		// Service should have EgressSvcValid condition set to Unknown.
-		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
-		expectEqual(t, fc, svc, nil)
-	})
-
-	t.Run("proxy_group_ready", func(t *testing.T) {
-		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
-			pg.Status.Conditions = []metav1.Condition{
-				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
-			}
-		})
-		// Quirks of the fake client.
-		mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
-			svc.Status.Conditions = []metav1.Condition{}
-		})
-		expectReconciled(t, esr, "default", "test")
-		// Verify that a ClusterIP Service has been created.
-		name := findGenNameForEgressSvcResources(t, fc, svc)
-		expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
-		clusterSvc := mustGetClusterIPSvc(t, fc, name)
-		// Verify that an EndpointSlice has been created.
-		expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
-		// Verify that ConfigMap contains configuration for the new egress service.
-		mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
-		r := svcConfiguredReason(svc, true, zl.Sugar())
-		// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
-		// CluterIP Service.
-		svc.Status.Conditions = []metav1.Condition{
-			condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
-		}
-		svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
-		svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
-		expectEqual(t, fc, svc, nil)
-	})
-
-	t.Run("delete_external_name_service", func(t *testing.T) {
-		name := findGenNameForEgressSvcResources(t, fc, svc)
-		if err := fc.Delete(context.Background(), svc); err != nil {
-			t.Fatalf("error deleting ExternalName Service: %v", err)
-		}
-		expectReconciled(t, esr, "default", "test")
-		// Verify that ClusterIP Service and EndpointSlice have been deleted.
-		expectMissing[corev1.Service](t, fc, "operator-ns", name)
-		expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
-		// Verify that service config has been deleted from the ConfigMap.
-		mustNotHaveConfigForSvc(t, fc, svc, cm)
-	})
-}
-
-func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
-	return metav1.Condition{
-		Type:               string(typ),
-		Status:             st,
-		LastTransitionTime: conditionTime(clock),
-		Reason:             r,
-		Message:            msg,
-	}
-}
-
-func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
-	t.Helper()
-	labels := egressSvcChildResourceLabels(svc)
-	s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
-	if err != nil {
-		t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
-	}
-	if s == nil {
-		t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name)
-	}
-	return s.GetName()
-}
-
-func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
-	labels := egressSvcChildResourceLabels(extNSvc)
-	return &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:         name,
-			Namespace:    "operator-ns",
-			GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name),
-			Labels:       labels,
-		},
-		Spec: corev1.ServiceSpec{
-			Type:  corev1.ServiceTypeClusterIP,
-			Ports: extNSvc.Spec.Ports,
-		},
-	}
-}
-
-func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service {
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: "operator-ns",
-		},
-	}
-	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil {
-		t.Fatalf("error retrieving Service")
-	}
-	return svc
-}
-
-func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
-	labels := egressSvcChildResourceLabels(extNSvc)
-	labels[discoveryv1.LabelManagedBy] = "tailscale.com"
-	labels[discoveryv1.LabelServiceName] = name
-	return &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ipv4", name),
-			Namespace: "operator-ns",
-			Labels:    labels,
-		},
-		Ports:       portsForEndpointSlice(clusterIPSvc),
-		AddressType: discoveryv1.AddressTypeIPv4,
-	}
-}
-
-func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
-	ports := make([]discoveryv1.EndpointPort, 0)
-	for _, p := range svc.Spec.Ports {
-		ports = append(ports, discoveryv1.EndpointPort{
-			Name:     &p.Name,
-			Protocol: &p.Protocol,
-			Port:     pointer.ToInt32(p.TargetPort.IntVal),
-		})
-	}
-	return ports
-}
-
-func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
-	t.Helper()
-	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
-	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
-		t.Fatalf("Error retrieving ConfigMap: %v", err)
-	}
-	name := tailnetSvcName(extNSvc)
-	gotCfg := configFromCM(t, cm, name)
-	if gotCfg == nil {
-		t.Fatalf("No config found for service %q", name)
-	}
-	if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" {
-		t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff)
-	}
-}
-
-func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) {
-	t.Helper()
-	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
-		t.Fatalf("Error retrieving ConfigMap: %v", err)
-	}
-	name := tailnetSvcName(extNSvc)
-	gotCfg := configFromCM(t, cm, name)
-	if gotCfg != nil {
-		t.Fatalf("Config  %#+v for service %q found when it should not be present", gotCfg, name)
-	}
-}
-
-func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config {
-	t.Helper()
-	cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices]
-	if !ok {
-		return nil
-	}
-	cfgs := &egressservices.Configs{}
-	if err := json.Unmarshal(cfgBs, cfgs); err != nil {
-		t.Fatalf("error unmarshalling config: %v", err)
-	}
-	cfg, ok := (*cfgs)[svcName]
-	if ok {
-		return &cfg
-	}
-	return nil
-}

cmd/k8s-operator/generate/main.go

@@ -25,13 +25,11 @@ const (
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
 	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
-	proxyGroupCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
 	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
-	proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -148,7 +146,6 @@ func generate(baseDir string) error {
 		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
 		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
 		{recorderCRDPath, recorderCRDHelmTemplatePath},
-		{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
 	} {
 		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
 			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
@@ -164,7 +161,6 @@ func cleanup(baseDir string) error {
 		proxyClassCRDHelmTemplatePath,
 		dnsConfigCRDHelmTemplatePath,
 		recorderCRDHelmTemplatePath,
-		proxyGroupCRDHelmTemplatePath,
 	} {
 		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
 			return fmt.Errorf("error cleaning up %s: %w", path, err)

cmd/k8s-operator/generate/main_test.go

@@ -62,9 +62,6 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
 		t.Errorf("Recorder CRD not found in default chart install")
 	}
-	if !strings.Contains(installContentsWithCRD.String(), "name: proxygroups.tailscale.com") {
-		t.Errorf("ProxyGroup CRD not found in default chart install")
-	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -86,7 +83,4 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
 		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
 	}
-	if strings.Contains(installContentsWithoutCRD.String(), "name: proxygroups.tailscale.com") {
-		t.Errorf("ProxyGroup CRD found in chart install that should not contain a CRD")
-	}
 }

cmd/k8s-operator/ingress.go

@@ -48,7 +48,7 @@ type IngressReconciler struct {
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
 
-	defaultProxyClass string
+	proxyDefaultClass string
 }
 
 var (
@@ -136,7 +136,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing, a.defaultProxyClass)
+	proxyClass := proxyClassForObject(ing, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/ingress_test.go

@@ -253,7 +253,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassReady),
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/operator.go

@@ -11,7 +11,6 @@ import (
 	"context"
 	"os"
 	"regexp"
-	"strconv"
 	"strings"
 	"time"
 
@@ -110,7 +109,7 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
-		defaultProxyClass:             defaultProxyClass,
+		proxyDefaultClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -144,20 +143,12 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
 	}
 	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-k8s-operator"
 	tsClient.HTTPClient = credentials.Client(context.Background())
 
 	s := &tsnet.Server{
 		Hostname: hostname,
 		Logf:     zlog.Named("tailscaled").Debugf,
 	}
-	if p := os.Getenv("TS_PORT"); p != "" {
-		port, err := strconv.ParseUint(p, 10, 16)
-		if err != nil {
-			startlog.Fatalf("TS_PORT %q cannot be parsed as uint16: %v", p, err)
-		}
-		s.Port = uint16(port)
-	}
 	if kubeSecret != "" {
 		st, err := kubestore.New(logger.Discard, kubeSecret)
 		if err != nil {
@@ -247,7 +238,6 @@ func runReconcilers(opts reconcilerOpts) {
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.Secret{}:             nsFilter,
 				&corev1.ServiceAccount{}:     nsFilter,
-				&corev1.Pod{}:                nsFilter,
 				&corev1.ConfigMap{}:          nsFilter,
 				&appsv1.StatefulSet{}:        nsFilter,
 				&appsv1.Deployment{}:         nsFilter,
@@ -295,7 +285,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
-			defaultProxyClass:     opts.defaultProxyClass,
+			proxyDefaultClass:     opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -318,7 +308,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:          eventRecorder,
 			Client:            mgr.GetClient(),
 			logger:            opts.log.Named("ingress-reconciler"),
-			defaultProxyClass: opts.defaultProxyClass,
+			proxyDefaultClass: opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -363,65 +353,6 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create nameserver reconciler: %v", err)
 	}
-
-	egressSvcFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsHandler)
-	egressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsFromEgressProxyGroup(mgr.GetClient(), opts.log))
-	err = builder.
-		ControllerManagedBy(mgr).
-		Named("egress-svcs-reconciler").
-		Watches(&corev1.Service{}, egressSvcFilter).
-		Watches(&tsapi.ProxyGroup{}, egressProxyGroupFilter).
-		Complete(&egressSvcsReconciler{
-			Client:      mgr.GetClient(),
-			tsNamespace: opts.tailscaleNamespace,
-			recorder:    eventRecorder,
-			clock:       tstime.DefaultClock{},
-			logger:      opts.log.Named("egress-svcs-reconciler"),
-		})
-	if err != nil {
-		startlog.Fatalf("could not create egress Services reconciler: %v", err)
-	}
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexEgressProxyGroup, indexEgressServices); err != nil {
-		startlog.Fatalf("failed setting up indexer for egress Services: %v", err)
-	}
-
-	egressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(egressSvcFromEps)
-	err = builder.
-		ControllerManagedBy(mgr).
-		Named("egress-svcs-readiness-reconciler").
-		Watches(&corev1.Service{}, egressSvcFilter).
-		Watches(&discoveryv1.EndpointSlice{}, egressSvcFromEpsFilter).
-		Complete(&egressSvcsReadinessReconciler{
-			Client:      mgr.GetClient(),
-			tsNamespace: opts.tailscaleNamespace,
-			clock:       tstime.DefaultClock{},
-			logger:      opts.log.Named("egress-svcs-readiness-reconciler"),
-		})
-	if err != nil {
-		startlog.Fatalf("could not create egress Services readiness reconciler: %v", err)
-	}
-
-	epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
-	podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace))
-	secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace))
-	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
-
-	err = builder.
-		ControllerManagedBy(mgr).
-		Named("egress-eps-reconciler").
-		Watches(&discoveryv1.EndpointSlice{}, epsFilter).
-		Watches(&corev1.Pod{}, podsFilter).
-		Watches(&corev1.Secret{}, secretsFilter).
-		Watches(&corev1.Service{}, epsFromExtNSvcFilter).
-		Complete(&egressEpsReconciler{
-			Client:      mgr.GetClient(),
-			tsNamespace: opts.tailscaleNamespace,
-			logger:      opts.log.Named("egress-eps-reconciler"),
-		})
-	if err != nil {
-		startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
-	}
-
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
 		Complete(&ProxyClassReconciler{
@@ -483,34 +414,6 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create Recorder reconciler: %v", err)
 	}
 
-	// Recorder reconciler.
-	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
-	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
-	err = builder.ControllerManagedBy(mgr).
-		For(&tsapi.ProxyGroup{}).
-		Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
-		Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
-		Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
-		Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
-		Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
-		Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
-		Complete(&ProxyGroupReconciler{
-			recorder: eventRecorder,
-			Client:   mgr.GetClient(),
-			l:        opts.log.Named("proxygroup-reconciler"),
-			clock:    tstime.DefaultClock{},
-			tsClient: opts.tsClient,
-
-			tsNamespace:       opts.tailscaleNamespace,
-			proxyImage:        opts.proxyImage,
-			defaultTags:       strings.Split(opts.proxyTags, ","),
-			tsFirewallMode:    opts.proxyFirewallMode,
-			defaultProxyClass: opts.defaultProxyClass,
-		})
-	if err != nil {
-		startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)
-	}
-
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
@@ -551,10 +454,10 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
-	// defaultProxyClass is the name of the ProxyClass to use as the default
+	// proxyDefaultClass is the name of the ProxyClass to use as the default
 	// class for proxies that do not have a ProxyClass set.
 	// this is defined by an operator env variable.
-	defaultProxyClass string
+	proxyDefaultClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
@@ -743,27 +646,6 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass,
-// returns a list of reconcile requests for all Connectors that have
-// .spec.proxyClass set.
-func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(ctx context.Context, o client.Object) []reconcile.Request {
-		pgList := new(tsapi.ProxyGroupList)
-		if err := cl.List(ctx, pgList); err != nil {
-			logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		proxyClassName := o.GetName()
-		for _, pg := range pgList.Items {
-			if pg.Spec.ProxyClass == proxyClassName {
-				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
-			}
-		}
-		return reqs
-	}
-}
-
 // serviceHandlerForIngress returns a handler for Service events for ingress
 // reconciler that ensures that if the Service associated with an event is of
 // interest to the reconciler, the associated Ingress(es) gets be reconciled.
@@ -805,10 +687,6 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl
 }
 
 func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
-	if _, ok := o.GetAnnotations()[AnnotationProxyGroup]; ok {
-		// Do not reconcile Services for ProxyGroup.
-		return nil
-	}
 	if isManagedByType(o, "svc") {
 		// If this is a Service managed by a Service we want to enqueue its parent
 		return []reconcile.Request{{NamespacedName: parentFromObjectLabels(o)}}
@@ -834,195 +712,3 @@ func isMagicDNSName(name string) bool {
 	validMagicDNSName := regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`)
 	return validMagicDNSName.MatchString(name)
 }
-
-// egressSvcsHandler returns accepts a Kubernetes object and returns a reconcile
-// request for it , if the object is a Tailscale egress Service meant to be
-// exposed on a ProxyGroup.
-func egressSvcsHandler(_ context.Context, o client.Object) []reconcile.Request {
-	if !isEgressSvcForProxyGroup(o) {
-		return nil
-	}
-	return []reconcile.Request{
-		{
-			NamespacedName: types.NamespacedName{
-				Namespace: o.GetNamespace(),
-				Name:      o.GetName(),
-			},
-		},
-	}
-}
-
-// egressEpsHandler returns accepts an EndpointSlice and, if the EndpointSlice
-// is for an egress service, returns a reconcile request for it.
-func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
-	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
-		return nil
-	}
-	return []reconcile.Request{
-		{
-			NamespacedName: types.NamespacedName{
-				Namespace: o.GetNamespace(),
-				Name:      o.GetName(),
-			},
-		},
-	}
-}
-
-// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
-// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
-func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
-			return nil
-		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
-		if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
-			return nil
-		}
-		pg, ok := o.GetLabels()[LabelParentName]
-		if !ok {
-			return nil
-		}
-		return reconcileRequestsForPG(pg, cl, ns)
-	}
-}
-
-// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is,
-// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
-func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
-			return nil
-		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
-		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
-			return nil
-		}
-		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
-			return nil
-		}
-		pg, ok := o.GetLabels()[LabelParentName]
-		if !ok {
-			return nil
-		}
-		return reconcileRequestsForPG(pg, cl, ns)
-	}
-}
-
-// egressSvcFromEps is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service
-// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service.
-func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
-	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
-		return nil
-	}
-	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
-		return nil
-	}
-	svcName, ok := o.GetLabels()[LabelParentName]
-	if !ok {
-		return nil
-	}
-	svcNs, ok := o.GetLabels()[LabelParentNamespace]
-	if !ok {
-		return nil
-	}
-	return []reconcile.Request{
-		{
-			NamespacedName: types.NamespacedName{
-				Namespace: svcNs,
-				Name:      svcName,
-			},
-		},
-	}
-}
-
-func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request {
-	epsList := discoveryv1.EndpointSliceList{}
-	if err := cl.List(context.Background(), &epsList,
-		client.InNamespace(ns),
-		client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
-		return nil
-	}
-	reqs := make([]reconcile.Request, 0)
-	for _, ep := range epsList.Items {
-		reqs = append(reqs, reconcile.Request{
-			NamespacedName: types.NamespacedName{
-				Namespace: ep.Namespace,
-				Name:      ep.Name,
-			},
-		})
-	}
-	return reqs
-}
-
-// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
-// user-created ExternalName Services that should be exposed on this ProxyGroup.
-func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		pg, ok := o.(*tsapi.ProxyGroup)
-		if !ok {
-			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
-			return nil
-		}
-		if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
-			return nil
-		}
-		svcList := &corev1.ServiceList{}
-		if err := cl.List(context.Background(), svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
-			logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, svc := range svcList.Items {
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: svc.Namespace,
-					Name:      svc.Name,
-				},
-			})
-		}
-		return reqs
-	}
-}
-
-// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
-// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
-func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
-	return func(_ context.Context, o client.Object) []reconcile.Request {
-		svc, ok := o.(*corev1.Service)
-		if !ok {
-			logger.Infof("[unexpected] Service handler triggered for an object that is not a Service")
-			return nil
-		}
-		if !isEgressSvcForProxyGroup(svc) {
-			return nil
-		}
-		epsList := &discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
-			client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
-			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, eps := range epsList.Items {
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: eps.Namespace,
-					Name:      eps.Name,
-				},
-			})
-		}
-		return reqs
-	}
-}
-
-// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
-// index is used a list filter.
-func indexEgressServices(o client.Object) []string {
-	if !isEgressSvcForProxyGroup(o) {
-		return nil
-	}
-	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
-}

cmd/k8s-operator/operator_test.go

@@ -432,148 +432,6 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 }
 
-func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:   zl.Sugar(),
-		clock:    clock,
-		recorder: record.NewFakeRecorder(100),
-	}
-	tailnetTargetIP := "invalid-ip"
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-
-			UID: types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	t0 := conditionTime(clock)
-
-	want := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-		Status: corev1.ServiceStatus{
-			Conditions: []metav1.Condition{{
-				Type:               string(tsapi.ProxyReady),
-				Status:             metav1.ConditionFalse,
-				LastTransitionTime: t0,
-				Reason:             reasonProxyInvalid,
-				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "invalid-ip" could not be parsed as a valid IP Address, error: ParseAddr("invalid-ip"): unable to parse IP`,
-			}},
-		},
-	}
-
-	expectEqual(t, fc, want, nil)
-}
-
-func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger:   zl.Sugar(),
-		clock:    clock,
-		recorder: record.NewFakeRecorder(100),
-	}
-	tailnetTargetIP := "999.999.999.999"
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-
-			UID: types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	t0 := conditionTime(clock)
-
-	want := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-			Annotations: map[string]string{
-				AnnotationTailnetTargetIP: tailnetTargetIP,
-			},
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-		Status: corev1.ServiceStatus{
-			Conditions: []metav1.Condition{{
-				Type:               string(tsapi.ProxyReady),
-				Status:             metav1.ConditionFalse,
-				LastTransitionTime: t0,
-				Reason:             reasonProxyInvalid,
-				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "999.999.999.999" could not be parsed as a valid IP Address, error: ParseAddr("999.999.999.999"): IPv4 field has value >255`,
-			}},
-		},
-	}
-
-	expectEqual(t, fc, want, nil)
-}
-
 func TestAnnotations(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
@@ -1206,7 +1064,7 @@ func TestProxyClassForService(t *testing.T) {
 		pc.Status = tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               string(tsapi.ProxyClassReady),
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
@@ -1388,7 +1246,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "a67b5ad3ff605531c822327e8f1a23dd0846e1075b722c13402f7d5d0ba32ba2",
+		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
 		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
@@ -1399,7 +1257,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "888a993ebee20ad6be99623b45015339de117946850cf1252bede0b570e04293"
+	o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }
@@ -1629,72 +1487,6 @@ func Test_clusterDomainFromResolverConf(t *testing.T) {
 		})
 	}
 }
-func Test_authKeyRemoval(t *testing.T) {
-	fc := fake.NewFakeClient()
-	ft := &fakeTSClient{}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// 1. A new Service that should be exposed via Tailscale gets created, a Secret with a config that contains auth
-	// key is generated.
-	clock := tstest.NewClock(tstest.ClockOpts{})
-	sr := &ServiceReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger: zl.Sugar(),
-		clock:  clock,
-	}
-
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			UID:       types.UID("1234-UID"),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-	})
-
-	expectReconciled(t, sr, "default", "test")
-
-	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
-	opts := configOpts{
-		stsName:         shortName,
-		secretName:      fullName,
-		namespace:       "default",
-		parentType:      "svc",
-		hostname:        "default-test",
-		clusterTargetIP: "10.20.30.40",
-		app:             kubetypes.AppIngressProxy,
-	}
-
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
-	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
-
-	// 2. Apply update to the Secret that imitates the proxy setting device_id.
-	s := expectedSecret(t, fc, opts)
-	mustUpdate(t, fc, s.Namespace, s.Name, func(s *corev1.Secret) {
-		mak.Set(&s.Data, "device_id", []byte("dkkdi4CNTRL"))
-	})
-
-	// 3. Config should no longer contain auth key
-	expectReconciled(t, sr, "default", "test")
-	opts.shouldRemoveAuthKey = true
-	opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")}
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-}
 
 func Test_externalNameService(t *testing.T) {
 	fc := fake.NewFakeClient()

cmd/k8s-operator/proxyclass.go

@@ -98,9 +98,9 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	if errs := pcr.validate(pc); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
 		pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
-		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
+		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
 	} else {
-		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
+		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
 	}
 	if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) {
 		if err := pcr.Client.Status().Update(ctx, pc); err != nil {

cmd/k8s-operator/proxyclass_test.go

@@ -69,7 +69,7 @@ func TestProxyClass(t *testing.T) {
 	// 1. A valid ProxyClass resource gets its status updated to Ready.
 	expectReconciled(t, pcr, "", "test")
 	pc.Status.Conditions = append(pc.Status.Conditions, metav1.Condition{
-		Type:               string(tsapi.ProxyClassReady),
+		Type:               string(tsapi.ProxyClassready),
 		Status:             metav1.ConditionTrue,
 		Reason:             reasonProxyClassValid,
 		Message:            reasonProxyClassValid,
@@ -85,7 +85,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
 	expectEvents(t, fr, []string{expectedEvent})
@@ -99,7 +99,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
 	expectEvents(t, fr, []string{expectedEvent})
@@ -118,7 +118,7 @@ func TestProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, pcr, "", "test")
 	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
-	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
 	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
 	expectEvents(t, fr, []string{expectedEvent})

cmd/k8s-operator/proxygroup.go

@@ -1,549 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"slices"
-	"sync"
-
-	"github.com/pkg/errors"
-	"go.uber.org/zap"
-	xslices "golang.org/x/exp/slices"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"tailscale.com/client/tailscale"
-	"tailscale.com/ipn"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
-	"tailscale.com/tailcfg"
-	"tailscale.com/tstime"
-	"tailscale.com/types/ptr"
-	"tailscale.com/util/clientmetric"
-	"tailscale.com/util/mak"
-	"tailscale.com/util/set"
-)
-
-const (
-	reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
-	reasonProxyGroupReady          = "ProxyGroupReady"
-	reasonProxyGroupCreating       = "ProxyGroupCreating"
-	reasonProxyGroupInvalid        = "ProxyGroupInvalid"
-)
-
-var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
-
-// ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
-type ProxyGroupReconciler struct {
-	client.Client
-	l        *zap.SugaredLogger
-	recorder record.EventRecorder
-	clock    tstime.Clock
-	tsClient tsClient
-
-	// User-specified defaults from the helm installation.
-	tsNamespace       string
-	proxyImage        string
-	defaultTags       []string
-	tsFirewallMode    string
-	defaultProxyClass string
-
-	mu          sync.Mutex           // protects following
-	proxyGroups set.Slice[types.UID] // for proxygroups gauge
-}
-
-func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
-	return r.l.With("ProxyGroup", name)
-}
-
-func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := r.logger(req.Name)
-	logger.Debugf("starting reconcile")
-	defer logger.Debugf("reconcile finished")
-
-	pg := new(tsapi.ProxyGroup)
-	err = r.Get(ctx, req.NamespacedName, pg)
-	if apierrors.IsNotFound(err) {
-		logger.Debugf("ProxyGroup not found, assuming it was deleted")
-		return reconcile.Result{}, nil
-	} else if err != nil {
-		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyGroup: %w", err)
-	}
-	if markedForDeletion(pg) {
-		logger.Debugf("ProxyGroup is being deleted, cleaning up resources")
-		ix := xslices.Index(pg.Finalizers, FinalizerName)
-		if ix < 0 {
-			logger.Debugf("no finalizer, nothing to do")
-			return reconcile.Result{}, nil
-		}
-
-		if done, err := r.maybeCleanup(ctx, pg); err != nil {
-			return reconcile.Result{}, err
-		} else if !done {
-			logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...")
-			return reconcile.Result{RequeueAfter: shortRequeue}, nil
-		}
-
-		pg.Finalizers = slices.Delete(pg.Finalizers, ix, ix+1)
-		if err := r.Update(ctx, pg); err != nil {
-			return reconcile.Result{}, err
-		}
-		return reconcile.Result{}, nil
-	}
-
-	oldPGStatus := pg.Status.DeepCopy()
-	setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldPGStatus, pg.Status) {
-			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
-		}
-		return reconcile.Result{}, err
-	}
-
-	if !slices.Contains(pg.Finalizers, FinalizerName) {
-		// This log line is printed exactly once during initial provisioning,
-		// because once the finalizer is in place this block gets skipped. So,
-		// this is a nice place to log that the high level, multi-reconcile
-		// operation is underway.
-		logger.Infof("ensuring ProxyGroup is set up")
-		pg.Finalizers = append(pg.Finalizers, FinalizerName)
-		if err = r.Update(ctx, pg); err != nil {
-			err = fmt.Errorf("error adding finalizer: %w", err)
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, reasonProxyGroupCreationFailed)
-		}
-	}
-
-	if err = r.validate(pg); err != nil {
-		message := fmt.Sprintf("ProxyGroup is invalid: %s", err)
-		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupInvalid, message)
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupInvalid, message)
-	}
-
-	proxyClassName := r.defaultProxyClass
-	if pg.Spec.ProxyClass != "" {
-		proxyClassName = pg.Spec.ProxyClass
-	}
-
-	var proxyClass *tsapi.ProxyClass
-	if proxyClassName != "" {
-		proxyClass = new(tsapi.ProxyClass)
-		err := r.Get(ctx, types.NamespacedName{Name: proxyClassName}, proxyClass)
-		if apierrors.IsNotFound(err) {
-			err = nil
-			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s does not (yet) exist", proxyClassName)
-			logger.Info(message)
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-		}
-		if err != nil {
-			err = fmt.Errorf("error getting ProxyGroup's ProxyClass %s: %s", proxyClassName, err)
-			r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
-		}
-		if !tsoperator.ProxyClassIsReady(proxyClass) {
-			message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName)
-			logger.Info(message)
-			return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-		}
-	}
-
-	if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
-		err = fmt.Errorf("error provisioning ProxyGroup resources: %w", err)
-		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
-	}
-
-	desiredReplicas := int(pgReplicas(pg))
-	if len(pg.Status.Devices) < desiredReplicas {
-		message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
-		logger.Debug(message)
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-	}
-
-	if len(pg.Status.Devices) > desiredReplicas {
-		message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas)
-		logger.Debug(message)
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
-	}
-
-	logger.Info("ProxyGroup resources synced")
-	return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady)
-}
-
-func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
-	logger := r.logger(pg.Name)
-	r.mu.Lock()
-	r.proxyGroups.Add(pg.UID)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
-	r.mu.Unlock()
-
-	cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass)
-	if err != nil {
-		return fmt.Errorf("error provisioning config Secrets: %w", err)
-	}
-	// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
-	stateSecrets := pgStateSecrets(pg, r.tsNamespace)
-	for _, sec := range stateSecrets {
-		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
-			s.ObjectMeta.Labels = sec.ObjectMeta.Labels
-			s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
-			s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
-		}); err != nil {
-			return fmt.Errorf("error provisioning state Secrets: %w", err)
-		}
-	}
-	sa := pgServiceAccount(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
-		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
-	}); err != nil {
-		return fmt.Errorf("error provisioning ServiceAccount: %w", err)
-	}
-	role := pgRole(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
-		r.ObjectMeta.Labels = role.ObjectMeta.Labels
-		r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
-		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
-		r.Rules = role.Rules
-	}); err != nil {
-		return fmt.Errorf("error provisioning Role: %w", err)
-	}
-	roleBinding := pgRoleBinding(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
-		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
-		r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
-		r.ObjectMeta.OwnerReferences = roleBinding.ObjectMeta.OwnerReferences
-		r.RoleRef = roleBinding.RoleRef
-		r.Subjects = roleBinding.Subjects
-	}); err != nil {
-		return fmt.Errorf("error provisioning RoleBinding: %w", err)
-	}
-	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-		cm := pgEgressCM(pg, r.tsNamespace)
-		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
-			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
-			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
-		}); err != nil {
-			return fmt.Errorf("error provisioning ConfigMap: %w", err)
-		}
-	}
-	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
-	if err != nil {
-		return fmt.Errorf("error generating StatefulSet spec: %w", err)
-	}
-	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
-		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
-		s.Spec = ss.Spec
-	}); err != nil {
-		return fmt.Errorf("error provisioning StatefulSet: %w", err)
-	}
-
-	if err := r.cleanupDanglingResources(ctx, pg); err != nil {
-		return fmt.Errorf("error cleaning up dangling resources: %w", err)
-	}
-
-	devices, err := r.getDeviceInfo(ctx, pg)
-	if err != nil {
-		return fmt.Errorf("failed to get device info: %w", err)
-	}
-
-	pg.Status.Devices = devices
-
-	return nil
-}
-
-// cleanupDanglingResources ensures we don't leak config secrets, state secrets, and
-// tailnet devices when the number of replicas specified is reduced.
-func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error {
-	logger := r.logger(pg.Name)
-	metadata, err := r.getNodeMetadata(ctx, pg)
-	if err != nil {
-		return err
-	}
-
-	for _, m := range metadata {
-		if m.ordinal+1 <= int(pgReplicas(pg)) {
-			continue
-		}
-
-		// Dangling resource, delete the config + state Secrets, as well as
-		// deleting the device from the tailnet.
-		if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
-			return err
-		}
-		if err := r.Delete(ctx, m.stateSecret); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err)
-			}
-		}
-		configSecret := m.stateSecret.DeepCopy()
-		configSecret.Name += "-config"
-		if err := r.Delete(ctx, configSecret); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err)
-			}
-		}
-	}
-
-	return nil
-}
-
-// maybeCleanup just deletes the device from the tailnet. All the kubernetes
-// resources linked to a ProxyGroup will get cleaned up via owner references
-// (which we can use because they are all in the same namespace).
-func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.ProxyGroup) (bool, error) {
-	logger := r.logger(pg.Name)
-
-	metadata, err := r.getNodeMetadata(ctx, pg)
-	if err != nil {
-		return false, err
-	}
-
-	for _, m := range metadata {
-		if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
-			return false, err
-		}
-	}
-
-	logger.Infof("cleaned up ProxyGroup resources")
-	r.mu.Lock()
-	r.proxyGroups.Remove(pg.UID)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
-	r.mu.Unlock()
-	return true, nil
-}
-
-func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error {
-	logger.Debugf("deleting device %s from control", string(id))
-	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
-		errResp := &tailscale.ErrResponse{}
-		if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
-			logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
-		} else {
-			return fmt.Errorf("error deleting device: %w", err)
-		}
-	} else {
-		logger.Debugf("device %s deleted from control", string(id))
-	}
-
-	return nil
-}
-
-func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) {
-	logger := r.logger(pg.Name)
-	var configSHA256Sum string
-	for i := range pgReplicas(pg) {
-		cfgSecret := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:            fmt.Sprintf("%s-%d-config", pg.Name, i),
-				Namespace:       r.tsNamespace,
-				Labels:          pgSecretLabels(pg.Name, "config"),
-				OwnerReferences: pgOwnerReference(pg),
-			},
-		}
-
-		var existingCfgSecret *corev1.Secret // unmodified copy of secret
-		if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
-			logger.Debugf("secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
-			existingCfgSecret = cfgSecret.DeepCopy()
-		} else if !apierrors.IsNotFound(err) {
-			return "", err
-		}
-
-		var authKey string
-		if existingCfgSecret == nil {
-			logger.Debugf("creating authkey for new ProxyGroup proxy")
-			tags := pg.Spec.Tags.Stringify()
-			if len(tags) == 0 {
-				tags = r.defaultTags
-			}
-			authKey, err = newAuthKey(ctx, r.tsClient, tags)
-			if err != nil {
-				return "", err
-			}
-		}
-
-		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret)
-		if err != nil {
-			return "", fmt.Errorf("error creating tailscaled config: %w", err)
-		}
-
-		for cap, cfg := range configs {
-			cfgJSON, err := json.Marshal(cfg)
-			if err != nil {
-				return "", fmt.Errorf("error marshalling tailscaled config: %w", err)
-			}
-			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
-		}
-
-		// The config sha256 sum is a value for a hash annotation used to trigger
-		// pod restarts when tailscaled config changes. Any config changes apply
-		// to all replicas, so it is sufficient to only hash the config for the
-		// first replica.
-		//
-		// In future, we're aiming to eliminate restarts altogether and have
-		// pods dynamically reload their config when it changes.
-		if i == 0 {
-			sum := sha256.New()
-			for _, cfg := range configs {
-				// Zero out the auth key so it doesn't affect the sha256 hash when we
-				// remove it from the config after the pods have all authed. Otherwise
-				// all the pods will need to restart immediately after authing.
-				cfg.AuthKey = nil
-				b, err := json.Marshal(cfg)
-				if err != nil {
-					return "", err
-				}
-				if _, err := sum.Write(b); err != nil {
-					return "", err
-				}
-			}
-
-			configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil))
-		}
-
-		if existingCfgSecret != nil {
-			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
-			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
-				return "", err
-			}
-		} else {
-			logger.Debugf("creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
-			if err := r.Create(ctx, cfgSecret); err != nil {
-				return "", err
-			}
-		}
-	}
-
-	return configSHA256Sum, nil
-}
-
-func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
-	conf := &ipn.ConfigVAlpha{
-		Version:      "alpha0",
-		AcceptDNS:    "false",
-		AcceptRoutes: "false", // AcceptRoutes defaults to true
-		Locked:       "false",
-		Hostname:     ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)),
-	}
-
-	if pg.Spec.HostnamePrefix != "" {
-		conf.Hostname = ptr.To(fmt.Sprintf("%s%d", pg.Spec.HostnamePrefix, idx))
-	}
-
-	if shouldAcceptRoutes(class) {
-		conf.AcceptRoutes = "true"
-	}
-
-	deviceAuthed := false
-	for _, d := range pg.Status.Devices {
-		if d.Hostname == *conf.Hostname {
-			deviceAuthed = true
-			break
-		}
-	}
-
-	if authKey != "" {
-		conf.AuthKey = &authKey
-	} else if !deviceAuthed {
-		key, err := authKeyFromSecret(oldSecret)
-		if err != nil {
-			return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
-		}
-		conf.AuthKey = key
-	}
-	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
-	capVerConfigs[106] = *conf
-	return capVerConfigs, nil
-}
-
-func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error {
-	return nil
-}
-
-// getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by
-// querying their state Secrets. It may not return the same number of items as
-// specified in the ProxyGroup spec if e.g. it is getting scaled up or down, or
-// some pods have failed to write state.
-func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) {
-	// List all state secrets owned by this ProxyGroup.
-	secrets := &corev1.SecretList{}
-	if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil {
-		return nil, fmt.Errorf("failed to list state Secrets: %w", err)
-	}
-	for _, secret := range secrets.Items {
-		var ordinal int
-		if _, err := fmt.Sscanf(secret.Name, pg.Name+"-%d", &ordinal); err != nil {
-			return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err)
-		}
-
-		id, dnsName, ok, err := getNodeMetadata(ctx, &secret)
-		if err != nil {
-			return nil, err
-		}
-		if !ok {
-			continue
-		}
-
-		metadata = append(metadata, nodeMetadata{
-			ordinal:     ordinal,
-			stateSecret: &secret,
-			tsID:        id,
-			dnsName:     dnsName,
-		})
-	}
-
-	return metadata, nil
-}
-
-func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
-	metadata, err := r.getNodeMetadata(ctx, pg)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, m := range metadata {
-		device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret)
-		if err != nil {
-			return nil, err
-		}
-		if !ok {
-			continue
-		}
-		devices = append(devices, tsapi.TailnetDevice{
-			Hostname:   device.Hostname,
-			TailnetIPs: device.TailnetIPs,
-		})
-	}
-
-	return devices, nil
-}
-
-type nodeMetadata struct {
-	ordinal     int
-	stateSecret *corev1.Secret
-	tsID        tailcfg.StableNodeID
-	dnsName     string
-}

cmd/k8s-operator/proxygroup_specs.go

@@ -1,299 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"fmt"
-
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/yaml"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/egressservices"
-	"tailscale.com/kube/kubetypes"
-	"tailscale.com/types/ptr"
-)
-
-// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
-// applied over the top after.
-func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) {
-	ss := new(appsv1.StatefulSet)
-	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
-	}
-	// Validate some base assumptions.
-	if len(ss.Spec.Template.Spec.InitContainers) != 1 {
-		return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers))
-	}
-	if len(ss.Spec.Template.Spec.Containers) != 1 {
-		return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers))
-	}
-
-	// StatefulSet config.
-	ss.ObjectMeta = metav1.ObjectMeta{
-		Name:            pg.Name,
-		Namespace:       namespace,
-		Labels:          pgLabels(pg.Name, nil),
-		OwnerReferences: pgOwnerReference(pg),
-	}
-	ss.Spec.Replicas = ptr.To(pgReplicas(pg))
-	ss.Spec.Selector = &metav1.LabelSelector{
-		MatchLabels: pgLabels(pg.Name, nil),
-	}
-
-	// Template config.
-	tmpl := &ss.Spec.Template
-	tmpl.ObjectMeta = metav1.ObjectMeta{
-		Name:                       pg.Name,
-		Namespace:                  namespace,
-		Labels:                     pgLabels(pg.Name, nil),
-		DeletionGracePeriodSeconds: ptr.To[int64](10),
-		Annotations: map[string]string{
-			podAnnotationLastSetConfigFileHash: cfgHash,
-		},
-	}
-	tmpl.Spec.ServiceAccountName = pg.Name
-	tmpl.Spec.InitContainers[0].Image = image
-	tmpl.Spec.Volumes = func() []corev1.Volume {
-		var volumes []corev1.Volume
-		for i := range pgReplicas(pg) {
-			volumes = append(volumes, corev1.Volume{
-				Name: fmt.Sprintf("tailscaledconfig-%d", i),
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i),
-					},
-				},
-			})
-		}
-
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			volumes = append(volumes, corev1.Volume{
-				Name: pgEgressCMName(pg.Name),
-				VolumeSource: corev1.VolumeSource{
-					ConfigMap: &corev1.ConfigMapVolumeSource{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: pgEgressCMName(pg.Name),
-						},
-					},
-				},
-			})
-		}
-
-		return volumes
-	}()
-
-	// Main container config.
-	c := &ss.Spec.Template.Spec.Containers[0]
-	c.Image = image
-	c.VolumeMounts = func() []corev1.VolumeMount {
-		var mounts []corev1.VolumeMount
-
-		// TODO(tomhjp): Read config directly from the secret instead. The
-		// mounts change on scaling up/down which causes unnecessary restarts
-		// for pods that haven't meaningfully changed.
-		for i := range pgReplicas(pg) {
-			mounts = append(mounts, corev1.VolumeMount{
-				Name:      fmt.Sprintf("tailscaledconfig-%d", i),
-				ReadOnly:  true,
-				MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
-			})
-		}
-
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			mounts = append(mounts, corev1.VolumeMount{
-				Name:      pgEgressCMName(pg.Name),
-				MountPath: "/etc/proxies",
-				ReadOnly:  true,
-			})
-		}
-
-		return mounts
-	}()
-	c.Env = func() []corev1.EnvVar {
-		envs := []corev1.EnvVar{
-			{
-				// TODO(irbekrm): verify that .status.podIPs are always set, else read in .status.podIP as well.
-				Name: "POD_IPS", // this will be a comma separate list i.e 10.136.0.6,2600:1900:4011:161:0:e:0:6
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						FieldPath: "status.podIPs",
-					},
-				},
-			},
-			{
-				Name:  "TS_KUBE_SECRET",
-				Value: "$(POD_NAME)",
-			},
-			{
-				Name:  "TS_STATE",
-				Value: "kube:$(POD_NAME)",
-			},
-			{
-				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
-				Value: "/etc/tsconfig/$(POD_NAME)",
-			},
-			{
-				Name:  "TS_INTERNAL_APP",
-				Value: kubetypes.AppProxyGroupEgress,
-			},
-		}
-
-		if tsFirewallMode != "" {
-			envs = append(envs, corev1.EnvVar{
-				Name:  "TS_DEBUG_FIREWALL_MODE",
-				Value: tsFirewallMode,
-			})
-		}
-
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-			envs = append(envs, corev1.EnvVar{
-				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
-				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
-			})
-		}
-
-		return append(c.Env, envs...)
-	}()
-
-	return ss, nil
-}
-
-func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount {
-	return &corev1.ServiceAccount{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pg.Name,
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-	}
-}
-
-func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
-	return &rbacv1.Role{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pg.Name,
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-		Rules: []rbacv1.PolicyRule{
-			{
-				APIGroups: []string{""},
-				Resources: []string{"secrets"},
-				Verbs: []string{
-					"get",
-					"patch",
-					"update",
-				},
-				ResourceNames: func() (secrets []string) {
-					for i := range pgReplicas(pg) {
-						secrets = append(secrets,
-							fmt.Sprintf("%s-%d-config", pg.Name, i), // Config with auth key.
-							fmt.Sprintf("%s-%d", pg.Name, i),        // State.
-						)
-					}
-					return secrets
-				}(),
-			},
-			{
-				APIGroups: []string{""},
-				Resources: []string{"events"},
-				Verbs: []string{
-					"create",
-					"patch",
-					"get",
-				},
-			},
-		},
-	}
-}
-
-func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
-	return &rbacv1.RoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pg.Name,
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      pg.Name,
-				Namespace: namespace,
-			},
-		},
-		RoleRef: rbacv1.RoleRef{
-			Kind: "Role",
-			Name: pg.Name,
-		},
-	}
-}
-
-func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) {
-	for i := range pgReplicas(pg) {
-		secrets = append(secrets, &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:            fmt.Sprintf("%s-%d", pg.Name, i),
-				Namespace:       namespace,
-				Labels:          pgSecretLabels(pg.Name, "state"),
-				OwnerReferences: pgOwnerReference(pg),
-			},
-		})
-	}
-
-	return secrets
-}
-
-func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
-	return &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            pgEgressCMName(pg.Name),
-			Namespace:       namespace,
-			Labels:          pgLabels(pg.Name, nil),
-			OwnerReferences: pgOwnerReference(pg),
-		},
-	}
-}
-
-func pgSecretLabels(pgName, typ string) map[string]string {
-	return pgLabels(pgName, map[string]string{
-		labelSecretType: typ, // "config" or "state".
-	})
-}
-
-func pgLabels(pgName string, customLabels map[string]string) map[string]string {
-	l := make(map[string]string, len(customLabels)+3)
-	for k, v := range customLabels {
-		l[k] = v
-	}
-
-	l[LabelManaged] = "true"
-	l[LabelParentType] = "proxygroup"
-	l[LabelParentName] = pgName
-
-	return l
-}
-
-func pgOwnerReference(owner *tsapi.ProxyGroup) []metav1.OwnerReference {
-	return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("ProxyGroup"))}
-}
-
-func pgReplicas(pg *tsapi.ProxyGroup) int32 {
-	if pg.Spec.Replicas != nil {
-		return *pg.Spec.Replicas
-	}
-
-	return 2
-}
-
-func pgEgressCMName(pg string) string {
-	return fmt.Sprintf("%s-egress-config", pg)
-}

cmd/k8s-operator/proxygroup_test.go

@@ -1,287 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"go.uber.org/zap"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/tools/record"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	"tailscale.com/client/tailscale"
-	tsoperator "tailscale.com/k8s-operator"
-	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/tstest"
-	"tailscale.com/types/ptr"
-)
-
-const testProxyImage = "tailscale/tailscale:test"
-
-var defaultProxyClassAnnotations = map[string]string{
-	"some-annotation": "from-the-proxy-class",
-}
-
-func TestProxyGroup(t *testing.T) {
-	const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196"
-
-	pc := &tsapi.ProxyClass{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "default-pc",
-		},
-		Spec: tsapi.ProxyClassSpec{
-			StatefulSet: &tsapi.StatefulSet{
-				Annotations: defaultProxyClassAnnotations,
-			},
-		},
-	}
-	pg := &tsapi.ProxyGroup{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:       "test",
-			Finalizers: []string{"tailscale.com/finalizer"},
-		},
-	}
-
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pg, pc).
-		WithStatusSubresource(pg, pc).
-		Build()
-	tsClient := &fakeTSClient{}
-	zl, _ := zap.NewDevelopment()
-	fr := record.NewFakeRecorder(1)
-	cl := tstest.NewClock(tstest.ClockOpts{})
-	reconciler := &ProxyGroupReconciler{
-		tsNamespace:       tsNamespace,
-		proxyImage:        testProxyImage,
-		defaultTags:       []string{"tag:test-tag"},
-		tsFirewallMode:    "auto",
-		defaultProxyClass: "default-pc",
-
-		Client:   fc,
-		tsClient: tsClient,
-		recorder: fr,
-		l:        zl.Sugar(),
-		clock:    cl,
-	}
-
-	t.Run("proxyclass_not_ready", func(t *testing.T) {
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, false, "")
-	})
-
-	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
-		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []metav1.Condition{{
-				Type:               string(tsapi.ProxyClassReady),
-				Status:             metav1.ConditionTrue,
-				Reason:             reasonProxyClassValid,
-				Message:            reasonProxyClassValid,
-				LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
-			}},
-		}
-		if err := fc.Status().Update(context.Background(), pc); err != nil {
-			t.Fatal(err)
-		}
-
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-		if expected := 1; reconciler.proxyGroups.Len() != expected {
-			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
-		}
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-		keyReq := tailscale.KeyCapabilities{
-			Devices: tailscale.KeyDeviceCapabilities{
-				Create: tailscale.KeyDeviceCreateCapabilities{
-					Reusable:      false,
-					Ephemeral:     false,
-					Preauthorized: true,
-					Tags:          []string{"tag:test-tag"},
-				},
-			},
-		}
-		if diff := cmp.Diff(tsClient.KeyRequests(), []tailscale.KeyCapabilities{keyReq, keyReq}); diff != "" {
-			t.Fatalf("unexpected secrets (-got +want):\n%s", diff)
-		}
-	})
-
-	t.Run("simulate_successful_device_auth", func(t *testing.T) {
-		addNodeIDToStateSecrets(t, fc, pg)
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		pg.Status.Devices = []tsapi.TailnetDevice{
-			{
-				Hostname:   "hostname-nodeid-0",
-				TailnetIPs: []string{"1.2.3.4", "::1"},
-			},
-			{
-				Hostname:   "hostname-nodeid-1",
-				TailnetIPs: []string{"1.2.3.4", "::1"},
-			},
-		}
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-	})
-
-	t.Run("scale_up_to_3", func(t *testing.T) {
-		pg.Spec.Replicas = ptr.To[int32](3)
-		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
-			p.Spec = pg.Spec
-		})
-		expectReconciled(t, reconciler, "", pg.Name)
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-
-		addNodeIDToStateSecrets(t, fc, pg)
-		expectReconciled(t, reconciler, "", pg.Name)
-		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
-		pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{
-			Hostname:   "hostname-nodeid-2",
-			TailnetIPs: []string{"1.2.3.4", "::1"},
-		})
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-	})
-
-	t.Run("scale_down_to_1", func(t *testing.T) {
-		pg.Spec.Replicas = ptr.To[int32](1)
-		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
-			p.Spec = pg.Spec
-		})
-
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-	})
-
-	t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) {
-		pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{
-			AcceptRoutes: true,
-		}
-		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
-			p.Spec = pc.Spec
-		})
-
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74")
-	})
-
-	t.Run("delete_and_cleanup", func(t *testing.T) {
-		if err := fc.Delete(context.Background(), pg); err != nil {
-			t.Fatal(err)
-		}
-
-		expectReconciled(t, reconciler, "", pg.Name)
-
-		expectMissing[tsapi.Recorder](t, fc, "", pg.Name)
-		if expected := 0; reconciler.proxyGroups.Len() != expected {
-			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
-		}
-		// 2 nodes should get deleted as part of the scale down, and then finally
-		// the first node gets deleted with the ProxyGroup cleanup.
-		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" {
-			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
-		}
-		// The fake client does not clean up objects whose owner has been
-		// deleted, so we can't test for the owned resources getting deleted.
-	})
-}
-
-func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
-	t.Helper()
-
-	role := pgRole(pg, tsNamespace)
-	roleBinding := pgRoleBinding(pg, tsNamespace)
-	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", cfgHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	statefulSet.Annotations = defaultProxyClassAnnotations
-
-	if shouldExist {
-		expectEqual(t, fc, role, nil)
-		expectEqual(t, fc, roleBinding, nil)
-		expectEqual(t, fc, serviceAccount, nil)
-		expectEqual(t, fc, statefulSet, nil)
-	} else {
-		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
-		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
-		expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
-		expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
-	}
-
-	var expectedSecrets []string
-	if shouldExist {
-		for i := range pgReplicas(pg) {
-			expectedSecrets = append(expectedSecrets,
-				fmt.Sprintf("%s-%d", pg.Name, i),
-				fmt.Sprintf("%s-%d-config", pg.Name, i),
-			)
-		}
-	}
-	expectSecrets(t, fc, expectedSecrets)
-}
-
-func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) {
-	t.Helper()
-
-	secrets := &corev1.SecretList{}
-	if err := fc.List(context.Background(), secrets); err != nil {
-		t.Fatal(err)
-	}
-
-	var actual []string
-	for _, secret := range secrets.Items {
-		actual = append(actual, secret.Name)
-	}
-
-	if diff := cmp.Diff(actual, expected); diff != "" {
-		t.Fatalf("unexpected secrets (-got +want):\n%s", diff)
-	}
-}
-
-func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) {
-	const key = "profile-abc"
-	for i := range pgReplicas(pg) {
-		bytes, err := json.Marshal(map[string]any{
-			"Config": map[string]any{
-				"NodeID": fmt.Sprintf("nodeid-%d", i),
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) {
-			s.Data = map[string][]byte{
-				currentProfileKey: []byte(key),
-				key:               bytes,
-			}
-		})
-	}
-}

cmd/k8s-operator/sts.go

@@ -47,11 +47,11 @@ const (
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
-	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
-	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
-	// the Ingress or Service.
+	// LabelProxyClass can be set by users on Connectors, tailscale
+	// Ingresses and Services that define cluster ingress or cluster egress,
+	// to specify that configuration in this ProxyClass should be applied to
+	// resources created for the Connector, Ingress or Service.
 	LabelProxyClass = "tailscale.com/proxy-class"
 
 	FinalizerName = "tailscale.com/finalizer"
@@ -65,8 +65,6 @@ const (
 	//MagicDNS name of tailnet node.
 	AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
 
-	AnnotationProxyGroup = "tailscale.com/proxy-group"
-
 	// Annotations settable by users on ingresses.
 	AnnotationFunnel = "tailscale.com/funnel"
 
@@ -132,13 +130,10 @@ type tailscaleSTSConfig struct {
 }
 
 type connector struct {
-	// routes is a list of routes that this Connector should advertise either as a subnet router or as an app
-	// connector.
+	// routes is a list of subnet routes that this Connector should expose.
 	routes string
 	// isExitNode defines whether this Connector should act as an exit node.
 	isExitNode bool
-	// isAppConnector defines whether this Connector should act as an app connector.
-	isAppConnector bool
 }
 type tsnetServer interface {
 	CertDomains() []string
@@ -307,7 +302,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaledConfigs, _ error) {
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaleConfigs, _ error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			// Hardcode a -0 suffix so that in future, if we support
@@ -365,7 +360,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 	latest := tailcfg.CapabilityVersion(-1)
 	var latestConfig ipn.ConfigVAlpha
 	for key, val := range configs {
-		fn := tsoperator.TailscaledConfigFileName(key)
+		fn := tsoperator.TailscaledConfigFileNameForCap(key)
 		b, err := json.Marshal(val)
 		if err != nil {
 			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
@@ -521,6 +516,11 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_KUBE_SECRET",
 			Value: proxySecret,
 		},
+		corev1.EnvVar{
+			// Old tailscaled config key is still used for backwards compatibility.
+			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
+			Value: "/etc/tsconfig/tailscaled",
+		},
 		corev1.EnvVar{
 			// New style is in the form of cap-<capability-version>.hujson.
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
@@ -670,9 +670,9 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	if pc == nil || ss == nil {
 		return ss
 	}
-	if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
+	if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
 		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
-			enableMetrics(ss)
+			enableMetrics(ss, pc)
 		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
 			// TODO (irbekrm): fix this
 			// For Ingress proxies that have been configured with
@@ -716,7 +716,6 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
 	ss.Spec.Template.Spec.Affinity = wantsPod.Affinity
 	ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
-	ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints
 
 	// Update containers.
 	updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container {
@@ -761,7 +760,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	return ss
 }
 
-func enableMetrics(ss *appsv1.StatefulSet) {
+func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) {
 	for i, c := range ss.Spec.Template.Spec.Containers {
 		if c.Name == "tailscale" {
 			// Serve metrics on on <pod-ip>:9001/debug/metrics. If
@@ -784,10 +783,16 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 	return origConf.AuthKey, nil
 }
 
-// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
-// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that
-// configuration.
-func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
+// tailscaledConfig takes a proxy config, a newly generated auth key if
+// generated and a Secret with the previous proxy state and auth key and
+// returns tailscaled configuration and a hash of that configuration.
+//
+// As of 2024-05-09 it also returns legacy tailscaled config without the
+// later added NoStatefulFilter field to support proxies older than cap95.
+// TODO (irbekrm): remove the legacy config once we no longer need to support
+// versions older than cap94,
+// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaleConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
 		AcceptDNS:           "false",
@@ -795,13 +800,11 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		Locked:              "false",
 		Hostname:            &stsC.Hostname,
 		NoStatefulFiltering: "false",
-		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
 	}
 
 	// For egress proxies only, we need to ensure that stateful filtering is
 	// not in place so that traffic from cluster can be forwarded via
 	// Tailscale IPs.
-	// TODO (irbekrm): set it to true always as this is now the default in core.
 	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	}
@@ -811,9 +814,6 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
 		conf.AdvertiseRoutes = routes
-		if stsC.Connector.isAppConnector {
-			conf.AppConnector.Advertise = true
-		}
 	}
 	if shouldAcceptRoutes(stsC.ProxyClass) {
 		conf.AcceptRoutes = "true"
@@ -821,58 +821,42 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 
 	if newAuthkey != "" {
 		conf.AuthKey = &newAuthkey
-	} else if shouldRetainAuthKey(oldSecret) {
-		key, err := authKeyFromSecret(oldSecret)
-		if err != nil {
-			return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
+	} else if oldSecret != nil {
+		var err error
+		latest := tailcfg.CapabilityVersion(-1)
+		latestStr := ""
+		for k, data := range oldSecret.Data {
+			// write to StringData, read from Data as StringData is write-only
+			if len(data) == 0 {
+				continue
+			}
+			v, err := tsoperator.CapVerFromFileName(k)
+			if err != nil {
+				continue
+			}
+			if v > latest {
+				latestStr = k
+				latest = v
+			}
+		}
+		// Allow for configs that don't contain an auth key. Perhaps
+		// users have some mechanisms to delete them. Auth key is
+		// normally not needed after the initial login.
+		if latestStr != "" {
+			conf.AuthKey, err = readAuthKey(oldSecret, latestStr)
+			if err != nil {
+				return nil, err
+			}
 		}
-		conf.AuthKey = key
 	}
-
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
-	capVerConfigs[107] = *conf
-
-	// AppConnector config option is only understood by clients of capver 107 and newer.
-	conf.AppConnector = nil
 	capVerConfigs[95] = *conf
+	// legacy config should not contain NoStatefulFiltering field.
+	conf.NoStatefulFiltering.Clear()
+	capVerConfigs[94] = *conf
 	return capVerConfigs, nil
 }
 
-func authKeyFromSecret(s *corev1.Secret) (key *string, err error) {
-	latest := tailcfg.CapabilityVersion(-1)
-	latestStr := ""
-	for k, data := range s.Data {
-		// write to StringData, read from Data as StringData is write-only
-		if len(data) == 0 {
-			continue
-		}
-		v, err := tsoperator.CapVerFromFileName(k)
-		if err != nil {
-			continue
-		}
-		if v > latest {
-			latestStr = k
-			latest = v
-		}
-	}
-	// Allow for configs that don't contain an auth key. Perhaps
-	// users have some mechanisms to delete them. Auth key is
-	// normally not needed after the initial login.
-	if latestStr != "" {
-		return readAuthKey(s, latestStr)
-	}
-	return key, nil
-}
-
-// shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be
-// retained (because the proxy has not yet successfully authenticated).
-func shouldRetainAuthKey(s *corev1.Secret) bool {
-	if s == nil {
-		return false // nothing to retain here
-	}
-	return len(s.Data["device_id"]) == 0 // proxy has not authed yet
-}
-
 func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
 	return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
 }
@@ -884,7 +868,7 @@ type ptrObject[T any] interface {
 	*T
 }
 
-type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
+type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
 
 // hashBytes produces a hash for the provided tailscaled config that is the same across
 // different invocations of this code. We do not use the
@@ -895,7 +879,7 @@ type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
 // thing that changed is operator version (the hash is also exposed to users via
 // an annotation and might be confusing if it changes without the config having
 // changed).
-func tailscaledConfigHash(c tailscaledConfigs) (string, error) {
+func tailscaledConfigHash(c tailscaleConfigs) (string, error) {
 	b, err := json.Marshal(c)
 	if err != nil {
 		return "", fmt.Errorf("error marshalling tailscaled configs: %w", err)

cmd/k8s-operator/sts_test.go

@@ -18,7 +18,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/types/ptr"
@@ -74,16 +73,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 					NodeSelector:     map[string]string{"beta.kubernetes.io/os": "linux"},
 					Affinity:         &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}},
 					Tolerations:      []corev1.Toleration{{Key: "", Operator: "Exists"}},
-					TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
-						{
-							WhenUnsatisfiable: "DoNotSchedule",
-							TopologyKey:       "kubernetes.io/hostname",
-							MaxSkew:           3,
-							LabelSelector: &metav1.LabelSelector{
-								MatchLabels: map[string]string{"foo": "bar"},
-							},
-						},
-					},
 					TailscaleContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
 							Privileged: ptr.To(true),
@@ -170,7 +159,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
-	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
@@ -213,7 +201,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
-	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)

cmd/k8s-operator/svc.go

@@ -64,7 +64,7 @@ type ServiceReconciler struct {
 
 	clock tstime.Clock
 
-	defaultProxyClass string
+	proxyDefaultClass string
 }
 
 var (
@@ -112,10 +112,6 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
 
-	if _, ok := svc.Annotations[AnnotationProxyGroup]; ok {
-		return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup
-	}
-
 	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
@@ -215,7 +211,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc, a.defaultProxyClass)
+	proxyClass := proxyClassForObject(svc, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -358,15 +354,6 @@ func validateService(svc *corev1.Service) []string {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
-	if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" {
-		ip, err := netip.ParseAddr(ipStr)
-		if err != nil {
-			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err))
-		} else if !ip.IsValid() {
-			violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr))
-		}
-	}
-
 	svcName := nameForService(svc)
 	if err := dnsname.ValidLabel(svcName); err != nil {
 		if _, ok := svc.Annotations[AnnotationHostname]; ok {

cmd/k8s-operator/testutils_test.go

@@ -48,14 +48,11 @@ type configOpts struct {
 	clusterTargetDNS                               string
 	subnetRoutes                                   string
 	isExitNode                                     bool
-	isAppConnector                                 bool
 	confFileHash                                   string
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
 	proxyClass                                     string // configuration from the named ProxyClass should be applied to proxy resources
 	app                                            string
-	shouldRemoveAuthKey                            bool
-	secretExtraData                                map[string][]byte
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -70,9 +67,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "false"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
-			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
-			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
+			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
@@ -230,9 +226,8 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
-			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
-			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
+			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 			{Name: "TS_INTERNAL_APP", Value: opts.app},
@@ -359,7 +354,6 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		Locked:       "false",
 		AuthKey:      ptr.To("secret-authkey"),
 		AcceptRoutes: "false",
-		AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
 	}
 	if opts.proxyClass != "" {
 		t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
@@ -371,12 +365,6 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 			conf.AcceptRoutes = "true"
 		}
 	}
-	if opts.shouldRemoveAuthKey {
-		conf.AuthKey = nil
-	}
-	if opts.isAppConnector {
-		conf.AppConnector = &ipn.AppConnectorPrefs{Advertise: true}
-	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -391,23 +379,22 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 			routes = append(routes, prefix)
 		}
 	}
+	conf.AdvertiseRoutes = routes
+	b, err := json.Marshal(conf)
+	if err != nil {
+		t.Fatalf("error marshalling tailscaled config")
+	}
 	if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	} else {
 		conf.NoStatefulFiltering = "false"
 	}
-	conf.AdvertiseRoutes = routes
-	bnn, err := json.Marshal(conf)
-	if err != nil {
-		t.Fatalf("error marshalling tailscaled config")
-	}
-	conf.AppConnector = nil
 	bn, err := json.Marshal(conf)
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
+	mak.Set(&s.StringData, "tailscaled", string(b))
 	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
-	mak.Set(&s.StringData, "cap-107.hujson", string(bnn))
 	labels := map[string]string{
 		"tailscale.com/managed":              "true",
 		"tailscale.com/parent-resource":      "test",
@@ -418,9 +405,6 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		labels["tailscale.com/parent-resource-ns"] = "" // Connector is cluster scoped
 	}
 	s.Labels = labels
-	for key, val := range opts.secretExtraData {
-		mak.Set(&s.Data, key, val)
-	}
 	return s
 }
 
@@ -612,7 +596,7 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili
 func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) {
 	return &tailscale.Device{
 		DeviceID: deviceID,
-		Hostname: "hostname-" + deviceID,
+		Hostname: "test-device",
 		Addresses: []string{
 			"1.2.3.4",
 			"::1",
@@ -647,17 +631,21 @@ func removeHashAnnotation(sts *appsv1.StatefulSet) {
 	delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
 }
 
-func removeTargetPortsFromSvc(svc *corev1.Service) {
-	newPorts := make([]corev1.ServicePort, 0)
-	for _, p := range svc.Spec.Ports {
-		newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port})
-	}
-	svc.Spec.Ports = newPorts
-}
-
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()
+		if len(secret.StringData["tailscaled"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
+				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
+			}
+			mak.Set(&secret.StringData, "tailscaled", string(b))
+		}
 		if len(secret.StringData["cap-95.hujson"]) != 0 {
 			conf := &ipn.ConfigVAlpha{}
 			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
@@ -670,17 +658,5 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 			}
 			mak.Set(&secret.StringData, "cap-95.hujson", string(b))
 		}
-		if len(secret.StringData["cap-107.hujson"]) != 0 {
-			conf := &ipn.ConfigVAlpha{}
-			if err := json.Unmarshal([]byte(secret.StringData["cap-107.hujson"]), conf); err != nil {
-				t.Fatalf("error umarshalling 'cap-107.hujson' contents: %v", err)
-			}
-			conf.AuthKey = nil
-			b, err := json.Marshal(conf)
-			if err != nil {
-				t.Fatalf("error marshalling 'cap-107.huson' contents: %v", err)
-			}
-			mak.Set(&secret.StringData, "cap-107.hujson", string(b))
-		}
 	}
 }

cmd/k8s-operator/tsrecorder.go

@@ -199,7 +199,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
 		return fmt.Errorf("error creating StatefulSet: %w", err)
 	}
 
-	var devices []tsapi.RecorderTailnetDevice
+	var devices []tsapi.TailnetDevice
 
 	device, ok, err := r.getDeviceInfo(ctx, tsr.Name)
 	if err != nil {
@@ -302,7 +302,9 @@ func (r *RecorderReconciler) validate(tsr *tsapi.Recorder) error {
 	return nil
 }
 
-func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string) (*corev1.Secret, error) {
+// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// is expected to always be non-empty if the node ID is, but not required.
+func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: r.tsNamespace,
@@ -311,27 +313,12 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string)
 	}
 	if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
 		if apierrors.IsNotFound(err) {
-			return nil, nil
+			return "", "", false, nil
 		}
 
-		return nil, fmt.Errorf("error getting state Secret: %w", err)
-	}
-
-	return secret, nil
-}
-
-func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
-	secret, err := r.getStateSecret(ctx, tsrName)
-	if err != nil || secret == nil {
 		return "", "", false, err
 	}
 
-	return getNodeMetadata(ctx, secret)
-}
-
-// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
-// is expected to always be non-empty if the node ID is, but not required.
-func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
 	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
 	currentProfile, ok := secret.Data[currentProfileKey]
 	if !ok {
@@ -350,29 +337,20 @@ func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.Sta
 	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
 }
 
-func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	secret, err := r.getStateSecret(ctx, tsrName)
-	if err != nil || secret == nil {
-		return tsapi.RecorderTailnetDevice{}, false, err
-	}
-
-	return getDeviceInfo(ctx, r.tsClient, secret)
-}
-
-func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	nodeID, dnsName, ok, err := getNodeMetadata(ctx, secret)
+func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.TailnetDevice, ok bool, err error) {
+	nodeID, dnsName, ok, err := r.getNodeMetadata(ctx, tsrName)
 	if !ok || err != nil {
-		return tsapi.RecorderTailnetDevice{}, false, err
+		return tsapi.TailnetDevice{}, false, err
 	}
 
 	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
 	// need the API. Should we instead update the profile to include addresses?
-	device, err := tsClient.Device(ctx, string(nodeID), nil)
+	device, err := r.tsClient.Device(ctx, string(nodeID), nil)
 	if err != nil {
-		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
+		return tsapi.TailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
 	}
 
-	d = tsapi.RecorderTailnetDevice{
+	d = tsapi.TailnetDevice{
 		Hostname:   device.Hostname,
 		TailnetIPs: device.Addresses,
 	}
@@ -392,6 +370,6 @@ type profile struct {
 	} `json:"Config"`
 }
 
-func markedForDeletion(obj metav1.Object) bool {
-	return !obj.GetDeletionTimestamp().IsZero()
+func markedForDeletion(tsr *tsapi.Recorder) bool {
+	return !tsr.DeletionTimestamp.IsZero()
 }

cmd/k8s-operator/tsrecorder_test.go

@@ -105,9 +105,9 @@ func TestRecorder(t *testing.T) {
 		})
 
 		expectReconciled(t, reconciler, "", tsr.Name)
-		tsr.Status.Devices = []tsapi.RecorderTailnetDevice{
+		tsr.Status.Devices = []tsapi.TailnetDevice{
 			{
-				Hostname:   "hostname-nodeid-123",
+				Hostname:   "test-device",
 				TailnetIPs: []string{"1.2.3.4", "::1"},
 				URL:        "https://test-0.example.ts.net",
 			},

cmd/stund/depaware.txt

@@ -67,7 +67,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/types/logger                                   from tailscale.com/tsweb
         tailscale.com/types/opt                                      from tailscale.com/envknob+
         tailscale.com/types/ptr                                      from tailscale.com/tailcfg+
-        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/tailcfg+
         tailscale.com/types/tkatype                                  from tailscale.com/tailcfg+
         tailscale.com/types/views                                    from tailscale.com/net/tsaddr+
@@ -75,7 +74,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/tailcfg
         tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
-        tailscale.com/util/lineiter                                  from tailscale.com/version/distro
+        tailscale.com/util/lineread                                  from tailscale.com/version/distro
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
         tailscale.com/util/slicesx                                   from tailscale.com/tailcfg
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
@@ -92,7 +91,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
-        golang.org/x/net/dns/dnsmessage                              from net+
+        golang.org/x/net/dns/dnsmessage                              from net
         golang.org/x/net/http/httpguts                               from net/http
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http

cmd/tailscale/cli/advertise.go

@@ -1,78 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package cli
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"strings"
-
-	"github.com/peterbourgon/ff/v3/ffcli"
-	"tailscale.com/envknob"
-	"tailscale.com/ipn"
-	"tailscale.com/tailcfg"
-)
-
-var advertiseArgs struct {
-	services string // comma-separated list of services to advertise
-}
-
-// TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode
-// envknob is not needed.
-var advertiseCmd = &ffcli.Command{
-	Name:       "advertise",
-	ShortUsage: "tailscale advertise --services=<services>",
-	ShortHelp:  "Advertise this node as a destination for a service",
-	Exec:       runAdvertise,
-	FlagSet: (func() *flag.FlagSet {
-		fs := newFlagSet("advertise")
-		fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")")
-		return fs
-	})(),
-}
-
-func maybeAdvertiseCmd() []*ffcli.Command {
-	if !envknob.UseWIPCode() {
-		return nil
-	}
-	return []*ffcli.Command{advertiseCmd}
-}
-
-func runAdvertise(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		return flag.ErrHelp
-	}
-
-	services, err := parseServiceNames(advertiseArgs.services)
-	if err != nil {
-		return err
-	}
-
-	_, err = localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
-		AdvertiseServicesSet: true,
-		Prefs: ipn.Prefs{
-			AdvertiseServices: services,
-		},
-	})
-	return err
-}
-
-// parseServiceNames takes a comma-separated list of service names
-// (eg. "svc:hello,svc:webserver,svc:catphotos"), splits them into
-// a list and validates each service name. If valid, it returns
-// the service names in a slice of strings.
-func parseServiceNames(servicesArg string) ([]string, error) {
-	var services []string
-	if servicesArg != "" {
-		services = strings.Split(servicesArg, ",")
-		for _, svc := range services {
-			err := tailcfg.CheckServiceName(svc)
-			if err != nil {
-				return nil, fmt.Errorf("service %q: %s", svc, err)
-			}
-		}
-	}
-	return services, nil
-}

cmd/tailscale/cli/cli.go

@@ -93,13 +93,8 @@ func Run(args []string) (err error) {
 
 	args = CleanUpArgs(args)
 
-	if len(args) == 1 {
-		switch args[0] {
-		case "-V", "--version":
-			args = []string{"version"}
-		case "help":
-			args = []string{"--help"}
-		}
+	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
+		args = []string{"version"}
 	}
 
 	var warnOnce sync.Once
@@ -182,7 +177,7 @@ For help on subcommands, add --help after: "tailscale status --help".
 This CLI is still under active development. Commands and flags will
 change in the future.
 `),
-		Subcommands: append([]*ffcli.Command{
+		Subcommands: []*ffcli.Command{
 			upCmd,
 			downCmd,
 			setCmd,
@@ -190,12 +185,10 @@ change in the future.
 			logoutCmd,
 			switchCmd,
 			configureCmd,
-			syspolicyCmd,
 			netcheckCmd,
 			ipCmd,
 			dnsCmd,
 			statusCmd,
-			metricsCmd,
 			pingCmd,
 			ncCmd,
 			sshCmd,
@@ -214,7 +207,7 @@ change in the future.
 			debugCmd,
 			driveCmd,
 			idTokenCmd,
-		}, maybeAdvertiseCmd()...),
+		},
 		FlagSet: rootfs,
 		Exec: func(ctx context.Context, args []string) error {
 			if len(args) > 0 {

cmd/tailscale/cli/cli_test.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io"
 	"net/netip"
 	"reflect"
 	"strings"
@@ -947,10 +946,6 @@ func TestPrefFlagMapping(t *testing.T) {
 			// Handled by the tailscale share subcommand, we don't want a CLI
 			// flag for this.
 			continue
-		case "AdvertiseServices":
-			// Handled by the tailscale advertise subcommand, we don't want a
-			// CLI flag for this.
-			continue
 		case "InternalExitNodePrior":
 			// Used internally by LocalBackend as part of exit node usage toggling.
 			// No CLI flag for this.
@@ -1453,7 +1448,7 @@ func TestParseNLArgs(t *testing.T) {
 			name:      "disablements not allowed",
 			input:     []string{"disablement:" + strings.Repeat("02", 32)},
 			parseKeys: true,
-			wantErr:   fmt.Errorf("parsing key 1: key hex string doesn't have expected type prefix tlpub:"),
+			wantErr:   fmt.Errorf("parsing key 1: key hex string doesn't have expected type prefix nlpub:"),
 		},
 		{
 			name:              "keys not allowed",
@@ -1481,33 +1476,3 @@ func TestParseNLArgs(t *testing.T) {
 		})
 	}
 }
-
-func TestHelpAlias(t *testing.T) {
-	var stdout, stderr bytes.Buffer
-	tstest.Replace[io.Writer](t, &Stdout, &stdout)
-	tstest.Replace[io.Writer](t, &Stderr, &stderr)
-
-	gotExit0 := false
-	defer func() {
-		if !gotExit0 {
-			t.Error("expected os.Exit(0) to be called")
-			return
-		}
-		if !strings.Contains(stderr.String(), "SUBCOMMANDS") {
-			t.Errorf("expected help output to contain SUBCOMMANDS; got stderr=%q; stdout=%q", stderr.String(), stdout.String())
-		}
-	}()
-	defer func() {
-		if e := recover(); e != nil {
-			if strings.Contains(fmt.Sprint(e), "unexpected call to os.Exit(0)") {
-				gotExit0 = true
-			} else {
-				t.Errorf("unexpected panic: %v", e)
-			}
-		}
-	}()
-	err := Run([]string{"help"})
-	if err != nil {
-		t.Fatalf("Run: %v", err)
-	}
-}

cmd/tailscale/cli/debug.go

@@ -213,7 +213,6 @@ var debugCmd = &ffcli.Command{
 				fs := newFlagSet("watch-ipn")
 				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
 				fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status")
-				fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags")
 				fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap")
 				fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever")
 				return fs
@@ -501,7 +500,6 @@ var watchIPNArgs struct {
 	netmap         bool
 	initial        bool
 	showPrivateKey bool
-	rateLimit      bool
 	count          int
 }
 
@@ -513,9 +511,6 @@ func runWatchIPN(ctx context.Context, args []string) error {
 	if !watchIPNArgs.showPrivateKey {
 		mask |= ipn.NotifyNoPrivateKeys
 	}
-	if watchIPNArgs.rateLimit {
-		mask |= ipn.NotifyRateLimit
-	}
 	watcher, err := localClient.WatchIPNBus(ctx, mask)
 	if err != nil {
 		return err
@@ -849,8 +844,7 @@ func runTS2021(ctx context.Context, args []string) error {
 	if ts2021Args.verbose {
 		logf = log.Printf
 	}
-
-	noiseDialer := &controlhttp.Dialer{
+	conn, err := (&controlhttp.Dialer{
 		Hostname:        ts2021Args.host,
 		HTTPPort:        "80",
 		HTTPSPort:       "443",
@@ -859,21 +853,7 @@ func runTS2021(ctx context.Context, args []string) error {
 		ProtocolVersion: uint16(ts2021Args.version),
 		Dialer:          dialFunc,
 		Logf:            logf,
-	}
-	const tries = 2
-	for i := range tries {
-		err := tryConnect(ctx, keys.PublicKey, noiseDialer)
-		if err != nil {
-			log.Printf("error on attempt %d/%d: %v", i+1, tries, err)
-			continue
-		}
-		break
-	}
-	return nil
-}
-
-func tryConnect(ctx context.Context, controlPublic key.MachinePublic, noiseDialer *controlhttp.Dialer) error {
-	conn, err := noiseDialer.Dial(ctx)
+	}).Dial(ctx)
 	log.Printf("controlhttp.Dial = %p, %v", conn, err)
 	if err != nil {
 		return err
@@ -881,8 +861,8 @@ func tryConnect(ctx context.Context, controlPublic key.MachinePublic, noiseDiale
 	log.Printf("did noise handshake")
 
 	gotPeer := conn.Peer()
-	if gotPeer != controlPublic {
-		log.Printf("peer = %v, want %v", gotPeer, controlPublic)
+	if gotPeer != keys.PublicKey {
+		log.Printf("peer = %v, want %v", gotPeer, keys.PublicKey)
 		return errors.New("key mismatch")
 	}
 
@@ -914,7 +894,7 @@ func tryConnect(ctx context.Context, controlPublic key.MachinePublic, noiseDiale
 	// Make a /whoami request to the server to verify that we can actually
 	// communicate over the newly-established connection.
 	whoamiURL := "http://" + ts2021Args.host + "/machine/whoami"
-	req, err := http.NewRequestWithContext(ctx, "GET", whoamiURL, nil)
+	req, err = http.NewRequestWithContext(ctx, "GET", whoamiURL, nil)
 	if err != nil {
 		return err
 	}

cmd/tailscale/cli/dns-query.go

@@ -1,163 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package cli
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"net/netip"
-	"os"
-	"text/tabwriter"
-
-	"golang.org/x/net/dns/dnsmessage"
-	"tailscale.com/types/dnstype"
-)
-
-func runDNSQuery(ctx context.Context, args []string) error {
-	if len(args) < 1 {
-		return flag.ErrHelp
-	}
-	name := args[0]
-	queryType := "A"
-	if len(args) >= 2 {
-		queryType = args[1]
-	}
-	fmt.Printf("DNS query for %q (%s) using internal resolver:\n", name, queryType)
-	fmt.Println()
-	bytes, resolvers, err := localClient.QueryDNS(ctx, name, queryType)
-	if err != nil {
-		fmt.Printf("failed to query DNS: %v\n", err)
-		return nil
-	}
-
-	if len(resolvers) == 1 {
-		fmt.Printf("Forwarding to resolver: %v\n", makeResolverString(*resolvers[0]))
-	} else {
-		fmt.Println("Multiple resolvers available:")
-		for _, r := range resolvers {
-			fmt.Printf("  - %v\n", makeResolverString(*r))
-		}
-	}
-	fmt.Println()
-	var p dnsmessage.Parser
-	header, err := p.Start(bytes)
-	if err != nil {
-		fmt.Printf("failed to parse DNS response: %v\n", err)
-		return err
-	}
-	fmt.Printf("Response code: %v\n", header.RCode.String())
-	fmt.Println()
-	p.SkipAllQuestions()
-	if header.RCode != dnsmessage.RCodeSuccess {
-		fmt.Println("No answers were returned.")
-		return nil
-	}
-	answers, err := p.AllAnswers()
-	if err != nil {
-		fmt.Printf("failed to parse DNS answers: %v\n", err)
-		return err
-	}
-	if len(answers) == 0 {
-		fmt.Println("  (no answers found)")
-	}
-
-	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
-	fmt.Fprintln(w, "Name\tTTL\tClass\tType\tBody")
-	fmt.Fprintln(w, "----\t---\t-----\t----\t----")
-	for _, a := range answers {
-		fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", a.Header.Name.String(), a.Header.TTL, a.Header.Class.String(), a.Header.Type.String(), makeAnswerBody(a))
-	}
-	w.Flush()
-
-	fmt.Println()
-	return nil
-}
-
-// makeAnswerBody returns a string with the DNS answer body in a human-readable format.
-func makeAnswerBody(a dnsmessage.Resource) string {
-	switch a.Header.Type {
-	case dnsmessage.TypeA:
-		return makeABody(a.Body)
-	case dnsmessage.TypeAAAA:
-		return makeAAAABody(a.Body)
-	case dnsmessage.TypeCNAME:
-		return makeCNAMEBody(a.Body)
-	case dnsmessage.TypeMX:
-		return makeMXBody(a.Body)
-	case dnsmessage.TypeNS:
-		return makeNSBody(a.Body)
-	case dnsmessage.TypeOPT:
-		return makeOPTBody(a.Body)
-	case dnsmessage.TypePTR:
-		return makePTRBody(a.Body)
-	case dnsmessage.TypeSRV:
-		return makeSRVBody(a.Body)
-	case dnsmessage.TypeTXT:
-		return makeTXTBody(a.Body)
-	default:
-		return a.Body.GoString()
-	}
-}
-
-func makeABody(a dnsmessage.ResourceBody) string {
-	if a, ok := a.(*dnsmessage.AResource); ok {
-		return netip.AddrFrom4(a.A).String()
-	}
-	return ""
-}
-func makeAAAABody(aaaa dnsmessage.ResourceBody) string {
-	if a, ok := aaaa.(*dnsmessage.AAAAResource); ok {
-		return netip.AddrFrom16(a.AAAA).String()
-	}
-	return ""
-}
-func makeCNAMEBody(cname dnsmessage.ResourceBody) string {
-	if c, ok := cname.(*dnsmessage.CNAMEResource); ok {
-		return c.CNAME.String()
-	}
-	return ""
-}
-func makeMXBody(mx dnsmessage.ResourceBody) string {
-	if m, ok := mx.(*dnsmessage.MXResource); ok {
-		return fmt.Sprintf("%s (Priority=%d)", m.MX, m.Pref)
-	}
-	return ""
-}
-func makeNSBody(ns dnsmessage.ResourceBody) string {
-	if n, ok := ns.(*dnsmessage.NSResource); ok {
-		return n.NS.String()
-	}
-	return ""
-}
-func makeOPTBody(opt dnsmessage.ResourceBody) string {
-	if o, ok := opt.(*dnsmessage.OPTResource); ok {
-		return o.GoString()
-	}
-	return ""
-}
-func makePTRBody(ptr dnsmessage.ResourceBody) string {
-	if p, ok := ptr.(*dnsmessage.PTRResource); ok {
-		return p.PTR.String()
-	}
-	return ""
-}
-func makeSRVBody(srv dnsmessage.ResourceBody) string {
-	if s, ok := srv.(*dnsmessage.SRVResource); ok {
-		return fmt.Sprintf("Target=%s, Port=%d, Priority=%d, Weight=%d", s.Target.String(), s.Port, s.Priority, s.Weight)
-	}
-	return ""
-}
-func makeTXTBody(txt dnsmessage.ResourceBody) string {
-	if t, ok := txt.(*dnsmessage.TXTResource); ok {
-		return fmt.Sprintf("%q", t.TXT)
-	}
-	return ""
-}
-func makeResolverString(r dnstype.Resolver) string {
-	if len(r.BootstrapResolution) > 0 {
-		return fmt.Sprintf("%s (bootstrap: %v)", r.Addr, r.BootstrapResolution)
-	}
-	return fmt.Sprintf("%s", r.Addr)
-}

cmd/tailscale/cli/dns-status.go

@@ -75,7 +75,7 @@ func runDNSStatus(ctx context.Context, args []string) error {
 	fmt.Print("\n")
 	fmt.Println("Split DNS Routes:")
 	if len(dnsConfig.Routes) == 0 {
-		fmt.Println("  (no routes configured: split DNS disabled)")
+		fmt.Println("  (no routes configured: split DNS might not be in use)")
 	}
 	for _, k := range slices.Sorted(maps.Keys(dnsConfig.Routes)) {
 		v := dnsConfig.Routes[k]

cmd/tailscale/cli/dns.go

@@ -28,13 +28,8 @@ var dnsCmd = &ffcli.Command{
 				return fs
 			})(),
 		},
-		{
-			Name:       "query",
-			ShortUsage: "tailscale dns query <name> [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]",
-			Exec:       runDNSQuery,
-			ShortHelp:  "Perform a DNS query",
-			LongHelp:   "The 'tailscale dns query' subcommand performs a DNS query for the specified name using the internal DNS forwarder (100.100.100.100).\n\nIt also provides information about the resolver(s) used to resolve the query.",
-		},
+
+		// TODO: implement `tailscale query` here
 
 		// TODO: implement `tailscale log` here
 

cmd/tailscale/cli/metrics.go

@@ -1,88 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package cli
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/peterbourgon/ff/v3/ffcli"
-	"tailscale.com/atomicfile"
-)
-
-var metricsCmd = &ffcli.Command{
-	Name:      "metrics",
-	ShortHelp: "Show Tailscale metrics",
-	LongHelp: strings.TrimSpace(`
-
-The 'tailscale metrics' command shows Tailscale user-facing metrics (as opposed
-to internal metrics printed by 'tailscale debug metrics').
-
-For more information about Tailscale metrics, refer to
-https://tailscale.com/s/client-metrics
-
-`),
-	ShortUsage: "tailscale metrics <subcommand> [flags]",
-	UsageFunc:  usageFuncNoDefaultValues,
-	Exec:       runMetricsNoSubcommand,
-	Subcommands: []*ffcli.Command{
-		{
-			Name:       "print",
-			ShortUsage: "tailscale metrics print",
-			Exec:       runMetricsPrint,
-			ShortHelp:  "Prints current metric values in the Prometheus text exposition format",
-		},
-		{
-			Name:       "write",
-			ShortUsage: "tailscale metrics write <path>",
-			Exec:       runMetricsWrite,
-			ShortHelp:  "Writes metric values to a file",
-			LongHelp: strings.TrimSpace(`
-
-The 'tailscale metrics write' command writes metric values to a text file provided as its
-only argument. It's meant to be used alongside Prometheus node exporter, allowing Tailscale
-metrics to be consumed and exported by the textfile collector.
-
-As an example, to export Tailscale metrics on an Ubuntu system running node exporter, you
-can regularly run 'tailscale metrics write /var/lib/prometheus/node-exporter/tailscaled.prom'
-using cron or a systemd timer.
-
-	`),
-		},
-	},
-}
-
-// runMetricsNoSubcommand prints metric values if no subcommand is specified.
-func runMetricsNoSubcommand(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		return fmt.Errorf("tailscale metrics: unknown subcommand: %s", args[0])
-	}
-
-	return runMetricsPrint(ctx, args)
-}
-
-// runMetricsPrint prints metric values to stdout.
-func runMetricsPrint(ctx context.Context, args []string) error {
-	out, err := localClient.UserMetrics(ctx)
-	if err != nil {
-		return err
-	}
-	Stdout.Write(out)
-	return nil
-}
-
-// runMetricsWrite writes metric values to a file.
-func runMetricsWrite(ctx context.Context, args []string) error {
-	if len(args) != 1 {
-		return errors.New("usage: tailscale metrics write <path>")
-	}
-	path := args[0]
-	out, err := localClient.UserMetrics(ctx)
-	if err != nil {
-		return err
-	}
-	return atomicfile.WriteFile(path, out, 0644)
-}

cmd/tailscale/cli/netcheck.go

@@ -136,7 +136,6 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 
 	printf("\nReport:\n")
-	printf("\t* Time: %v\n", report.Now.Format(time.RFC3339Nano))
 	printf("\t* UDP: %v\n", report.UDP)
 	if report.GlobalV4.IsValid() {
 		printf("\t* IPv4: yes, %s\n", report.GlobalV4)

cmd/tailscale/cli/network-lock.go

@@ -151,15 +151,13 @@ func runNetworkLockInit(ctx context.Context, args []string) error {
 		return nil
 	}
 
-	var successMsg strings.Builder
-
-	fmt.Fprintf(&successMsg, "%d disablement secrets have been generated and are printed below. Take note of them now, they WILL NOT be shown again.\n", nlInitArgs.numDisablements)
+	fmt.Printf("%d disablement secrets have been generated and are printed below. Take note of them now, they WILL NOT be shown again.\n", nlInitArgs.numDisablements)
 	for range nlInitArgs.numDisablements {
 		var secret [32]byte
 		if _, err := rand.Read(secret[:]); err != nil {
 			return err
 		}
-		fmt.Fprintf(&successMsg, "\tdisablement-secret:%X\n", secret[:])
+		fmt.Printf("\tdisablement-secret:%X\n", secret[:])
 		disablementValues = append(disablementValues, tka.DisablementKDF(secret[:]))
 	}
 
@@ -170,7 +168,7 @@ func runNetworkLockInit(ctx context.Context, args []string) error {
 			return err
 		}
 		disablementValues = append(disablementValues, tka.DisablementKDF(supportDisablement))
-		fmt.Fprintln(&successMsg, "A disablement secret for Tailscale support has been generated and transmitted to Tailscale.")
+		fmt.Println("A disablement secret for Tailscale support has been generated and will be transmitted to Tailscale upon initialization.")
 	}
 
 	// The state returned by NetworkLockInit likely doesn't contain the initialized state,
@@ -179,7 +177,6 @@ func runNetworkLockInit(ctx context.Context, args []string) error {
 		return err
 	}
 
-	fmt.Print(successMsg.String())
 	fmt.Println("Initialization complete.")
 	return nil
 }

cmd/tailscale/cli/set.go

@@ -51,6 +51,7 @@ type setArgsT struct {
 	advertiseRoutes        string
 	advertiseDefaultRoute  bool
 	advertiseConnector     bool
+	advertiseNatConnector  bool
 	opUser                 string
 	acceptedRisks          string
 	profileName            string
@@ -151,6 +152,9 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 			AppConnector: ipn.AppConnectorPrefs{
 				Advertise: setArgs.advertiseConnector,
 			},
+			NatConnector: ipn.AppConnectorPrefs{
+				Advertise: setArgs.advertiseNatConnector,
+			},
 			PostureChecking:     setArgs.postureChecking,
 			NoStatefulFiltering: opt.NewBool(!setArgs.statefulFiltering),
 		},

cmd/tailscale/cli/syspolicy.go

@@ -1,110 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package cli
-
-import (
-	"context"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"os"
-	"slices"
-	"text/tabwriter"
-
-	"github.com/peterbourgon/ff/v3/ffcli"
-	"tailscale.com/util/syspolicy/setting"
-)
-
-var syspolicyArgs struct {
-	json bool // JSON output mode
-}
-
-var syspolicyCmd = &ffcli.Command{
-	Name:       "syspolicy",
-	ShortHelp:  "Diagnose the MDM and system policy configuration",
-	LongHelp:   "The 'tailscale syspolicy' command provides tools for diagnosing the MDM and system policy configuration.",
-	ShortUsage: "tailscale syspolicy <subcommand>",
-	UsageFunc:  usageFuncNoDefaultValues,
-	Subcommands: []*ffcli.Command{
-		{
-			Name:       "list",
-			ShortUsage: "tailscale syspolicy list",
-			Exec:       runSysPolicyList,
-			ShortHelp:  "Prints effective policy settings",
-			LongHelp:   "The 'tailscale syspolicy list' subcommand displays the effective policy settings and their sources (e.g., MDM or environment variables).",
-			FlagSet: (func() *flag.FlagSet {
-				fs := newFlagSet("syspolicy list")
-				fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format")
-				return fs
-			})(),
-		},
-		{
-			Name:       "reload",
-			ShortUsage: "tailscale syspolicy reload",
-			Exec:       runSysPolicyReload,
-			ShortHelp:  "Forces a reload of policy settings, even if no changes are detected, and prints the result",
-			LongHelp:   "The 'tailscale syspolicy reload' subcommand forces a reload of policy settings, even if no changes are detected, and prints the result.",
-			FlagSet: (func() *flag.FlagSet {
-				fs := newFlagSet("syspolicy reload")
-				fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format")
-				return fs
-			})(),
-		},
-	},
-}
-
-func runSysPolicyList(ctx context.Context, args []string) error {
-	policy, err := localClient.GetEffectivePolicy(ctx, setting.DefaultScope())
-	if err != nil {
-		return err
-	}
-	printPolicySettings(policy)
-	return nil
-
-}
-
-func runSysPolicyReload(ctx context.Context, args []string) error {
-	policy, err := localClient.ReloadEffectivePolicy(ctx, setting.DefaultScope())
-	if err != nil {
-		return err
-	}
-	printPolicySettings(policy)
-	return nil
-}
-
-func printPolicySettings(policy *setting.Snapshot) {
-	if syspolicyArgs.json {
-		json, err := json.MarshalIndent(policy, "", "\t")
-		if err != nil {
-			errf("syspolicy marshalling error: %v", err)
-		} else {
-			outln(string(json))
-		}
-		return
-	}
-	if policy.Len() == 0 {
-		outln("No policy settings")
-		return
-	}
-
-	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
-	fmt.Fprintln(w, "Name\tOrigin\tValue\tError")
-	fmt.Fprintln(w, "----\t------\t-----\t-----")
-	for _, k := range slices.Sorted(policy.Keys()) {
-		setting, _ := policy.GetSetting(k)
-		var origin string
-		if o := setting.Origin(); o != nil {
-			origin = o.String()
-		}
-		if err := setting.Error(); err != nil {
-			fmt.Fprintf(w, "%s\t%s\t\t{%s}\n", k, origin, err)
-		} else {
-			fmt.Fprintf(w, "%s\t%s\t%s\t\n", k, origin, setting.Value())
-		}
-	}
-	w.Flush()
-
-	fmt.Println()
-	return
-}

cmd/tailscale/cli/up.go

@@ -32,12 +32,10 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/netutil"
-	"tailscale.com/net/tsaddr"
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
-	"tailscale.com/types/views"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -116,6 +114,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet {
 	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
 	upf.BoolVar(&upArgs.advertiseConnector, "advertise-connector", false, "advertise this node as an app connector")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
+	upf.BoolVar(&upArgs.advertiseNatConnector, "advertise-nat-connector", false, "advertise this node as a nat connector")
 
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
@@ -164,9 +163,6 @@ func defaultNetfilterMode() string {
 	return "on"
 }
 
-// upArgsT is the type of upArgs, the argument struct for `tailscale up`.
-// As of 2024-10-08, upArgsT is frozen and no new arguments should be
-// added to it. Add new arguments to setArgsT instead.
 type upArgsT struct {
 	qr                     bool
 	reset                  bool
@@ -194,6 +190,7 @@ type upArgsT struct {
 	timeout                time.Duration
 	acceptedRisks          string
 	profileName            string
+	advertiseNatConnector  bool
 }
 
 func (a upArgsT) getAuthKey() (string, error) {
@@ -304,6 +301,7 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	prefs.OperatorUser = upArgs.opUser
 	prefs.ProfileName = upArgs.profileName
 	prefs.AppConnector.Advertise = upArgs.advertiseConnector
+	prefs.NatConnector.Advertise = upArgs.advertiseNatConnector
 
 	if goos == "linux" {
 		prefs.NoSNAT = !upArgs.snat
@@ -771,6 +769,7 @@ func init() {
 	addPrefFlagMapping("auto-update", "AutoUpdate.Apply")
 	addPrefFlagMapping("advertise-connector", "AppConnector")
 	addPrefFlagMapping("posture-checking", "PostureChecking")
+	addPrefFlagMapping("advertise-nat-connector", "NatConnector")
 }
 
 func addPrefFlagMapping(flagName string, prefNames ...string) {
@@ -1020,7 +1019,7 @@ func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]any) {
 			set(prefs.OperatorUser)
 		case "advertise-routes":
 			var sb strings.Builder
-			for i, r := range tsaddr.WithoutExitRoutes(views.SliceOf(prefs.AdvertiseRoutes)).All() {
+			for i, r := range withoutExitNodes(prefs.AdvertiseRoutes) {
 				if i > 0 {
 					sb.WriteByte(',')
 				}
@@ -1028,9 +1027,11 @@ func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]any) {
 			}
 			set(sb.String())
 		case "advertise-exit-node":
-			set(tsaddr.ContainsExitRoutes(views.SliceOf(prefs.AdvertiseRoutes)))
+			set(hasExitNodeRoutes(prefs.AdvertiseRoutes))
 		case "advertise-connector":
 			set(prefs.AppConnector.Advertise)
+		case "advertise-nat-connector":
+			set(prefs.NatConnector.Advertise)
 		case "snat-subnet-routes":
 			set(!prefs.NoSNAT)
 		case "stateful-filtering":
@@ -1062,6 +1063,36 @@ func fmtFlagValueArg(flagName string, val any) string {
 	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(fmt.Sprint(val)))
 }
 
+func hasExitNodeRoutes(rr []netip.Prefix) bool {
+	var v4, v6 bool
+	for _, r := range rr {
+		if r.Bits() == 0 {
+			if r.Addr().Is4() {
+				v4 = true
+			} else if r.Addr().Is6() {
+				v6 = true
+			}
+		}
+	}
+	return v4 && v6
+}
+
+// withoutExitNodes returns rr unchanged if it has only 1 or 0 /0
+// routes. If it has both IPv4 and IPv6 /0 routes, then it returns
+// a copy with all /0 routes removed.
+func withoutExitNodes(rr []netip.Prefix) []netip.Prefix {
+	if !hasExitNodeRoutes(rr) {
+		return rr
+	}
+	var out []netip.Prefix
+	for _, r := range rr {
+		if r.Bits() > 0 {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
 // exitNodeIP returns the exit node IP from p, using st to map
 // it from its ID form to an IP address if needed.
 func exitNodeIP(p *ipn.Prefs, st *ipnstate.Status) (ip netip.Addr) {
@@ -1155,7 +1186,6 @@ func resolveAuthKey(ctx context.Context, v, tags string) (string, error) {
 	}
 
 	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-cli"
 	tsClient.HTTPClient = credentials.Client(ctx)
 	tsClient.BaseURL = baseURL
 
@@ -1178,7 +1208,7 @@ func resolveAuthKey(ctx context.Context, v, tags string) (string, error) {
 }
 
 func warnOnAdvertiseRouts(ctx context.Context, prefs *ipn.Prefs) {
-	if len(prefs.AdvertiseRoutes) > 0 || prefs.AppConnector.Advertise {
+	if len(prefs.AdvertiseRoutes) > 0 || prefs.AppConnector.Advertise || prefs.NatConnector.Advertise {
 		// TODO(jwhited): compress CheckIPForwarding and CheckUDPGROForwarding
 		//  into a single HTTP req.
 		if err := localClient.CheckIPForwarding(ctx); err != nil {

cmd/tailscale/depaware.txt

@@ -5,6 +5,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
@@ -22,7 +26,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-  DW    github.com/google/uuid                                       from tailscale.com/clientupdate+
+        github.com/google/uuid                                       from tailscale.com/clientupdate+
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
@@ -76,20 +80,18 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/cmd/tailscale/cli
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
-  LW    tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/cmd/tailscale/cli/ffcomplete                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/cmd/tailscale/cli/ffcomplete/internal          from tailscale.com/cmd/tailscale/cli/ffcomplete
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlhttp                            from tailscale.com/cmd/tailscale/cli
-        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/envknob/featureknob                            from tailscale.com/client/web
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/health/healthmsg                               from tailscale.com/cmd/tailscale/cli
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -118,9 +120,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
+        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
         tailscale.com/syncs                                          from tailscale.com/cmd/tailscale/cli+
@@ -132,7 +134,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
         tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
-        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
+        tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
@@ -144,7 +146,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
-        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/types/key+
         tailscale.com/types/views                                    from tailscale.com/tailcfg+
@@ -152,14 +153,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck+
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dnscache+
         tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy+
-        tailscale.com/util/ctxkey                                    from tailscale.com/types/logger+
+        tailscale.com/util/ctxkey                                    from tailscale.com/types/logger
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/groupmember                               from tailscale.com/client/web
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlhttp+
@@ -170,19 +171,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
-        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
-        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
-        tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
-   W 💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
+     💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
@@ -261,7 +257,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         crypto/tls                                                   from github.com/miekg/dns+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-  DW    database/sql/driver                                          from github.com/google/uuid
+        database/sql/driver                                          from github.com/google/uuid
    W    debug/dwarf                                                  from debug/pe
    W    debug/pe                                                     from github.com/dblohm7/wingoes/pe
         embed                                                        from crypto/internal/nistec+
@@ -321,7 +317,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
-        runtime/debug                                                from tailscale.com+
+        runtime/debug                                                from github.com/coder/websocket/internal/xsync+
         slices                                                       from tailscale.com/client/web+
         sort                                                         from compress/flate+
         strconv                                                      from archive/tar+

cmd/tailscaled/depaware.txt

@@ -79,6 +79,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
+        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -107,7 +111,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-  DW    github.com/google/uuid                                       from tailscale.com/clientupdate+
+        github.com/google/uuid                                       from tailscale.com/clientupdate+
         github.com/gorilla/csrf                                      from tailscale.com/client/web
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
@@ -217,7 +221,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
@@ -240,12 +244,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/client/web                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
-  LW    tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/cmd/tailscaled/childproc                       from tailscale.com/cmd/tailscaled+
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/cmd/tailscaled+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
-        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/tailscaled+
@@ -260,7 +263,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/drive/driveimpl/dirfs                          from tailscale.com/drive/driveimpl+
         tailscale.com/drive/driveimpl/shared                         from tailscale.com/drive/driveimpl+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -319,11 +321,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
-        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
+        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -360,7 +362,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/control/controlclient+
-        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/tka+
         tailscale.com/types/views                                    from tailscale.com/ipn/ipnlocal+
@@ -378,7 +379,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/control/controlclient+
         tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
@@ -397,12 +398,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/cmd/tailscaled+
-        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
-        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
-        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
-        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
@@ -412,7 +409,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+
@@ -510,7 +507,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         crypto/tls                                                   from github.com/aws/aws-sdk-go-v2/aws/transport/http+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-  DW    database/sql/driver                                          from github.com/google/uuid
+        database/sql/driver                                          from github.com/google/uuid
    W    debug/dwarf                                                  from debug/pe
    W    debug/pe                                                     from github.com/dblohm7/wingoes/pe
         embed                                                        from crypto/internal/nistec+

cmd/tailscaled/deps_test.go

@@ -1,30 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package main
-
-import (
-	"testing"
-
-	"tailscale.com/tstest/deptest"
-)
-
-func TestOmitSSH(t *testing.T) {
-	const msg = "unexpected with ts_omit_ssh"
-	deptest.DepChecker{
-		GOOS:   "linux",
-		GOARCH: "amd64",
-		Tags:   "ts_omit_ssh",
-		BadDeps: map[string]string{
-			"tailscale.com/ssh/tailssh":            msg,
-			"golang.org/x/crypto/ssh":              msg,
-			"tailscale.com/sessionrecording":       msg,
-			"github.com/anmitsu/go-shlex":          msg,
-			"github.com/creack/pty":                msg,
-			"github.com/kr/fs":                     msg,
-			"github.com/pkg/sftp":                  msg,
-			"github.com/u-root/u-root/pkg/termios": msg,
-			"tempfork/gliderlabs/ssh":              msg,
-		},
-	}.Check(t)
-}

cmd/tailscaled/ssh.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build (linux || darwin || freebsd || openbsd) && !ts_omit_ssh
+//go:build linux || darwin || freebsd || openbsd
 
 package main
 

cmd/tailscaled/tailscaled.go

@@ -625,6 +625,11 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 		UseSocketOnly: args.socketpath != paths.DefaultTailscaledSocket(),
 	})
 	configureTaildrop(logf, lb)
+	if f, err := lb.NatcHandlerForFlow(); err == nil && f != nil {
+		ns.GetTCPHandlerForFlow = f
+	} else if err != nil {
+		return nil, fmt.Errorf("lb.NatcHandler: %w", err)
+	}
 	if err := ns.Start(lb); err != nil {
 		log.Fatalf("failed to start netstack: %v", err)
 	}
@@ -680,15 +685,12 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 		ListenPort:    args.port,
 		NetMon:        sys.NetMon.Get(),
 		HealthTracker: sys.HealthTracker(),
-		Metrics:       sys.UserMetricsRegistry(),
 		Dialer:        sys.Dialer.Get(),
 		SetSubsystem:  sys.Set,
 		ControlKnobs:  sys.ControlKnobs(),
 		DriveForLocal: driveimpl.NewFileSystemForLocal(logf),
 	}
 
-	sys.HealthTracker().SetMetricsRegistry(sys.UserMetricsRegistry())
-
 	onlyNetstack = name == "userspace-networking"
 	netstackSubnetRouter := onlyNetstack // but mutated later on some platforms
 	netns.SetEnabled(!onlyNetstack)
@@ -788,6 +790,7 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 }
 
 func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
+	tfs, _ := sys.DriveForLocal.GetOK()
 	ret, err := netstack.Create(logf,
 		sys.Tun.Get(),
 		sys.Engine.Get(),
@@ -795,6 +798,7 @@ func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
 		sys.Dialer.Get(),
 		sys.DNSManager.Get(),
 		sys.ProxyMapper(),
+		tfs,
 	)
 	if err != nil {
 		return nil, err

cmd/testwrapper/testwrapper.go

@@ -42,7 +42,6 @@ type testAttempt struct {
 	testName      string // "TestFoo"
 	outcome       string // "pass", "fail", "skip"
 	logs          bytes.Buffer
-	start, end    time.Time
 	isMarkedFlaky bool   // set if the test is marked as flaky
 	issueURL      string // set if the test is marked as flaky
 
@@ -133,17 +132,11 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 		}
 		pkg := goOutput.Package
 		pkgTests := resultMap[pkg]
-		if pkgTests == nil {
-			pkgTests = make(map[string]*testAttempt)
-			resultMap[pkg] = pkgTests
-		}
 		if goOutput.Test == "" {
 			switch goOutput.Action {
-			case "start":
-				pkgTests[""] = &testAttempt{start: goOutput.Time}
 			case "fail", "pass", "skip":
 				for _, test := range pkgTests {
-					if test.testName != "" && test.outcome == "" {
+					if test.outcome == "" {
 						test.outcome = "fail"
 						ch <- test
 					}
@@ -151,13 +144,15 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 				ch <- &testAttempt{
 					pkg:         goOutput.Package,
 					outcome:     goOutput.Action,
-					start:       pkgTests[""].start,
-					end:         goOutput.Time,
 					pkgFinished: true,
 				}
 			}
 			continue
 		}
+		if pkgTests == nil {
+			pkgTests = make(map[string]*testAttempt)
+			resultMap[pkg] = pkgTests
+		}
 		testName := goOutput.Test
 		if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
 			testName = test
@@ -173,10 +168,8 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 			pkgTests[testName] = &testAttempt{
 				pkg:      pkg,
 				testName: testName,
-				start:    goOutput.Time,
 			}
 		case "skip", "pass", "fail":
-			pkgTests[testName].end = goOutput.Time
 			pkgTests[testName].outcome = goOutput.Action
 			ch <- pkgTests[testName]
 		case "output":
@@ -220,7 +213,7 @@ func main() {
 		firstRun.tests = append(firstRun.tests, &packageTests{Pattern: pkg})
 	}
 	toRun := []*nextRun{firstRun}
-	printPkgOutcome := func(pkg, outcome string, attempt int, runtime time.Duration) {
+	printPkgOutcome := func(pkg, outcome string, attempt int) {
 		if outcome == "skip" {
 			fmt.Printf("?\t%s [skipped/no tests] \n", pkg)
 			return
@@ -232,10 +225,10 @@ func main() {
 			outcome = "FAIL"
 		}
 		if attempt > 1 {
-			fmt.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt)
+			fmt.Printf("%s\t%s [attempt=%d]\n", outcome, pkg, attempt)
 			return
 		}
-		fmt.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds())
+		fmt.Printf("%s\t%s\n", outcome, pkg)
 	}
 
 	// Check for -coverprofile argument and filter it out
@@ -314,7 +307,7 @@ func main() {
 						// when a package times out.
 						failed = true
 					}
-					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt, tr.end.Sub(tr.start))
+					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt)
 					continue
 				}
 				if testingVerbose || tr.outcome == "fail" {

cmd/testwrapper/testwrapper_test.go

@@ -10,7 +10,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"sync"
 	"testing"
 )
@@ -77,10 +76,7 @@ func TestFlakeRun(t *testing.T) {
 		t.Fatalf("go run . %s: %s with output:\n%s", testfile, err, out)
 	}
 
-	// Replace the unpredictable timestamp with "0.00s".
-	out = regexp.MustCompile(`\t\d+\.\d\d\ds\t`).ReplaceAll(out, []byte("\t0.00s\t"))
-
-	want := []byte("ok\t" + testfile + "\t0.00s\t[attempt=2]")
+	want := []byte("ok\t" + testfile + " [attempt=2]")
 	if !bytes.Contains(out, want) {
 		t.Fatalf("wanted output containing %q but got:\n%s", want, out)
 	}

cmd/tsconnect/common.go

@@ -150,7 +150,6 @@ func runEsbuildServe(buildOptions esbuild.BuildOptions) {
 		log.Fatalf("Cannot start esbuild server: %v", err)
 	}
 	log.Printf("Listening on http://%s:%d\n", result.Host, result.Port)
-	select {}
 }
 
 func runEsbuild(buildOptions esbuild.BuildOptions) esbuild.BuildResult {

cmd/tsconnect/wasm/wasm_js.go

@@ -108,14 +108,13 @@ func newIPN(jsConfig js.Value) map[string]any {
 		SetSubsystem:  sys.Set,
 		ControlKnobs:  sys.ControlKnobs(),
 		HealthTracker: sys.HealthTracker(),
-		Metrics:       sys.UserMetricsRegistry(),
 	})
 	if err != nil {
 		log.Fatal(err)
 	}
 	sys.Set(eng)
 
-	ns, err := netstack.Create(logf, sys.Tun.Get(), eng, sys.MagicSock.Get(), dialer, sys.DNSManager.Get(), sys.ProxyMapper())
+	ns, err := netstack.Create(logf, sys.Tun.Get(), eng, sys.MagicSock.Get(), dialer, sys.DNSManager.Get(), sys.ProxyMapper(), nil)
 	if err != nil {
 		log.Fatalf("netstack.Create: %v", err)
 	}
@@ -129,9 +128,6 @@ func newIPN(jsConfig js.Value) map[string]any {
 	dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
 		return ns.DialContextTCP(ctx, dst)
 	}
-	dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
-		return ns.DialContextUDP(ctx, dst)
-	}
 	sys.NetstackRouter.Set(true)
 	sys.Tun.Get().Start()
 
@@ -272,8 +268,8 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 						name = p.Hostinfo().Hostname()
 					}
 					addrs := make([]string, p.Addresses().Len())
-					for i, ap := range p.Addresses().All() {
-						addrs[i] = ap.Addr().String()
+					for i := range p.Addresses().Len() {
+						addrs[i] = p.Addresses().At(i).Addr().String()
 					}
 					return jsNetMapPeerNode{
 						jsNetMapNode: jsNetMapNode{
@@ -589,8 +585,8 @@ func mapSlice[T any, M any](a []T, f func(T) M) []M {
 
 func mapSliceView[T any, M any](a views.Slice[T], f func(T) M) []M {
 	n := make([]M, a.Len())
-	for i, v := range a.All() {
-		n[i] = f(v)
+	for i := range a.Len() {
+		n[i] = f(a.At(i))
 	}
 	return n
 }

cmd/tsidp/tsidp.go

@@ -64,7 +64,6 @@ var (
 	flagLocalPort          = flag.Int("local-port", -1, "allow requests from localhost")
 	flagUseLocalTailscaled = flag.Bool("use-local-tailscaled", false, "use local tailscaled instead of tsnet")
 	flagFunnel             = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet")
-	flagDir                = flag.String("dir", "", "tsnet state directory; a default one will be created if not provided")
 )
 
 func main() {
@@ -121,7 +120,6 @@ func main() {
 	} else {
 		ts := &tsnet.Server{
 			Hostname: "idp",
-			Dir:      *flagDir,
 		}
 		if *flagVerbose {
 			ts.Logf = log.Printf