cmd/tailscale: add tailnet info to status output

Updates #14375

Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d
Signed-off-by: Will Norris <will@tailscale.com>

.github/workflows/codeql-analysis.yml

@@ -49,13 +49,13 @@ jobs:
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6

.github/workflows/golangci-lint.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
         with:
           go-version-file: go.mod
           cache: false

.github/workflows/test.yml

@@ -80,7 +80,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -153,13 +153,13 @@ jobs:
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -260,7 +260,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -319,7 +319,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -367,7 +367,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -461,7 +461,7 @@ jobs:
       run: |
         echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts

Makefile

@@ -100,7 +100,7 @@ publishdevoperator: ## Build and publish k8s-operator image to location specifie
 	@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
 	@test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1)
 	@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
-	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=operator ./build_docker.sh
+	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh
 
 publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO}
 	@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)

VERSION.txt

@@ -1 +1 @@
-1.77.0
+1.79.0

build_docker.sh

@@ -17,12 +17,20 @@ eval "$(./build_dist.sh shellvars)"
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
 DEFAULT_BASE="tailscale/alpine-base:3.18"
+# Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
+# Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
+# annotations defined here will be overriden by release scripts that call this script.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+DEFAULT_ANNOTATIONS="org.opencontainers.image.source=https://github.com/tailscale/tailscale/blob/main/build_docker.sh,org.opencontainers.image.vendor=Tailscale"
 
 PUSH="${PUSH:-false}"
 TARGET="${TARGET:-${DEFAULT_TARGET}}"
 TAGS="${TAGS:-${DEFAULT_TAGS}}"
 BASE="${BASE:-${DEFAULT_BASE}}"
 PLATFORM="${PLATFORM:-}" # default to all platforms
+# OCI annotations that will be added to the image.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
+ANNOTATIONS="${ANNOTATIONS:-${DEFAULT_ANNOTATIONS}}"
 
 case "$TARGET" in
   client)
@@ -43,9 +51,10 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/containerboot
     ;;
-  operator)
+  k8s-operator)
     DEFAULT_REPOS="tailscale/k8s-operator"
     REPOS="${REPOS:-${DEFAULT_REPOS}}"
     go run github.com/tailscale/mkctr \
@@ -56,9 +65,11 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/operator
     ;;
   k8s-nameserver)
@@ -72,9 +83,11 @@ case "$TARGET" in
         -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
       --base="${BASE}" \
       --tags="${TAGS}" \
+      --gotags="ts_kube,ts_package_container" \
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/k8s-nameserver
     ;;
   *)

client/tailscale/localclient.go

@@ -40,6 +40,7 @@ import (
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
+	"tailscale.com/util/syspolicy/setting"
 )
 
 // defaultLocalClient is the default LocalClient when using the legacy
@@ -492,6 +493,17 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
 	return nil
 }
 
+// DebugActionBody invokes a debug action with a body parameter, such as
+// "debug-force-prefer-derp".
+// These are development tools and subject to change or removal over time.
+func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
+	body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, rbody)
+	if err != nil {
+		return fmt.Errorf("error %w: %s", err, body)
+	}
+	return nil
+}
+
 // DebugResultJSON invokes a debug action and returns its result as something JSON-able.
 // These are development tools and subject to change or removal over time.
 func (lc *LocalClient) DebugResultJSON(ctx context.Context, action string) (any, error) {
@@ -814,6 +826,33 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
+// GetEffectivePolicy returns the effective policy for the specified scope.
+func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+	scopeID, err := scope.MarshalText()
+	if err != nil {
+		return nil, err
+	}
+	body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID))
+	if err != nil {
+		return nil, err
+	}
+	return decodeJSON[*setting.Snapshot](body)
+}
+
+// ReloadEffectivePolicy reloads the effective policy for the specified scope
+// by reading and merging policy settings from all applicable policy sources.
+func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+	scopeID, err := scope.MarshalText()
+	if err != nil {
+		return nil, err
+	}
+	body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody)
+	if err != nil {
+		return nil, err
+	}
+	return decodeJSON[*setting.Snapshot](body)
+}
+
 // GetDNSOSConfig returns the system DNS configuration for the current device.
 // That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
 func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
@@ -1299,6 +1338,17 @@ func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConf
 	return nil
 }
 
+// DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be
+// run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over
+// to another replica whilst there is still some grace period for the existing connections to terminate.
+func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
+	_, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil)
+	if err != nil {
+		return fmt.Errorf("error disconnecting control: %w", err)
+	}
+	return nil
+}
+
 // NetworkLockDisable shuts down network-lock across the tailnet.
 func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) error {
 	if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil {

client/web/web.go

@@ -26,6 +26,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/envknob"
+	"tailscale.com/envknob/featureknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -960,37 +961,16 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 }
 
 func availableFeatures() map[string]bool {
-	env := hostinfo.GetEnvType()
 	features := map[string]bool{
 		"advertise-exit-node": true, // available on all platforms
 		"advertise-routes":    true, // available on all platforms
-		"use-exit-node":       canUseExitNode(env) == nil,
-		"ssh":                 envknob.CanRunTailscaleSSH() == nil,
+		"use-exit-node":       featureknob.CanUseExitNode() == nil,
+		"ssh":                 featureknob.CanRunTailscaleSSH() == nil,
 		"auto-update":         version.IsUnstableBuild() && clientupdate.CanAutoUpdate(),
 	}
-	if env == hostinfo.HomeAssistantAddOn {
-		// Setting SSH on Home Assistant causes trouble on startup
-		// (since the flag is not being passed to `tailscale up`).
-		// Although Tailscale SSH does work here,
-		// it's not terribly useful since it's running in a separate container.
-		features["ssh"] = false
-	}
 	return features
 }
 
-func canUseExitNode(env hostinfo.EnvType) error {
-	switch dist := distro.Get(); dist {
-	case distro.Synology, // see https://github.com/tailscale/tailscale/issues/1995
-		distro.QNAP,
-		distro.Unraid:
-		return fmt.Errorf("Tailscale exit nodes cannot be used on %s.", dist)
-	}
-	if env == hostinfo.HomeAssistantAddOn {
-		return errors.New("Tailscale exit nodes cannot be used on Home Assistant.")
-	}
-	return nil
-}
-
 // aclsAllowAccess returns whether tailnet ACLs (as expressed in the provided filter rules)
 // permit any devices to access the local web client.
 // This does not currently check whether a specific device can connect, just any device.

cmd/checkmetrics/checkmetrics.go

@@ -0,0 +1,131 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// checkmetrics validates that all metrics in the tailscale client-metrics
+// are documented in a given path or URL.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"time"
+
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tsnet"
+	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/util/httpm"
+)
+
+var (
+	kbPath = flag.String("kb-path", "", "filepath to the client-metrics knowledge base")
+	kbUrl  = flag.String("kb-url", "", "URL to the client-metrics knowledge base page")
+)
+
+func main() {
+	flag.Parse()
+	if *kbPath == "" && *kbUrl == "" {
+		log.Fatalf("either -kb-path or -kb-url must be set")
+	}
+
+	var control testcontrol.Server
+	ts := httptest.NewServer(&control)
+	defer ts.Close()
+
+	td, err := os.MkdirTemp("", "testcontrol")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	// tsnet is used not used as a Tailscale client, but as a way to
+	// boot up Tailscale, have all the metrics registered, and then
+	// verifiy that all the metrics are documented.
+	tsn := &tsnet.Server{
+		Dir:        td,
+		Store:      new(mem.Store),
+		UserLogf:   log.Printf,
+		Ephemeral:  true,
+		ControlURL: ts.URL,
+	}
+	if err := tsn.Start(); err != nil {
+		log.Fatal(err)
+	}
+	defer tsn.Close()
+
+	log.Printf("checking that all metrics are documented, looking for: %s", tsn.Sys().UserMetricsRegistry().MetricNames())
+
+	if *kbPath != "" {
+		kb, err := readKB(*kbPath)
+		if err != nil {
+			log.Fatalf("reading kb: %v", err)
+		}
+		missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
+
+		if len(missing) > 0 {
+			log.Fatalf("found undocumented metrics in %q: %v", *kbPath, missing)
+		}
+	}
+
+	if *kbUrl != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		kb, err := getKB(ctx, *kbUrl)
+		if err != nil {
+			log.Fatalf("getting kb: %v", err)
+		}
+		missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
+
+		if len(missing) > 0 {
+			log.Fatalf("found undocumented metrics in %q: %v", *kbUrl, missing)
+		}
+	}
+}
+
+func readKB(path string) (string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("reading file: %w", err)
+	}
+
+	return string(b), nil
+}
+
+func getKB(ctx context.Context, url string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, url, nil)
+	if err != nil {
+		return "", fmt.Errorf("creating request: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("getting kb page: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("reading body: %w", err)
+	}
+	return string(b), nil
+}
+
+func undocumentedMetrics(b string, metrics []string) []string {
+	var missing []string
+	for _, metric := range metrics {
+		if !strings.Contains(b, metric) {
+			missing = append(missing, metric)
+		}
+	}
+	return missing
+}

cmd/containerboot/healthz.go

@@ -7,7 +7,6 @@ package main
 
 import (
 	"log"
-	"net"
 	"net/http"
 	"sync"
 )
@@ -23,29 +22,29 @@ type healthz struct {
 func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.Lock()
 	defer h.Unlock()
+
 	if h.hasAddrs {
 		w.Write([]byte("ok"))
 	} else {
-		http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
+		http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable)
 	}
 }
 
-// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
-// provided address. A containerized tailscale instance is considered healthy if
+func (h *healthz) update(healthy bool) {
+	h.Lock()
+	defer h.Unlock()
+
+	if h.hasAddrs != healthy {
+		log.Println("Setting healthy", healthy)
+	}
+	h.hasAddrs = healthy
+}
+
+// healthHandlers registers a simple health handler at /healthz.
+// A containerized tailscale instance is considered healthy if
 // it has at least one tailnet IP address.
-func runHealthz(addr string, h *healthz) {
-	lis, err := net.Listen("tcp", addr)
-	if err != nil {
-		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
-	}
-	mux := http.NewServeMux()
-	mux.Handle("/healthz", h)
-	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
-	hs := &http.Server{Handler: mux}
-
-	go func() {
-		if err := hs.Serve(lis); err != nil {
-			log.Fatalf("failed running health endpoint: %v", err)
-		}
-	}()
+func healthHandlers(mux *http.ServeMux) *healthz {
+	h := &healthz{}
+	mux.Handle("GET /healthz", h)
+	return h
 }

cmd/containerboot/kube.go

@@ -9,30 +9,56 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log"
 	"net/http"
 	"net/netip"
 	"os"
 
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 )
 
-// storeDeviceID writes deviceID to 'device_id' data field of the named
-// Kubernetes Secret.
-func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
-	s := &kubeapi.Secret{
-		Data: map[string][]byte{
-			"device_id": []byte(deviceID),
-		},
-	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+// kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
+// this rather than any of the upstream Kubernetes client libaries to avoid extra imports.
+type kubeClient struct {
+	kubeclient.Client
+	stateSecret string
+	canPatch    bool // whether the client has permissions to patch Kubernetes Secrets
 }
 
-// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields
-// 'device_ips', 'device_fqdn' of the named Kubernetes Secret.
-func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, addresses []netip.Prefix) error {
+func newKubeClient(root string, stateSecret string) (*kubeClient, error) {
+	if root != "/" {
+		// If we are running in a test, we need to set the root path to the fake
+		// service account directory.
+		kubeclient.SetRootPathForTesting(root)
+	}
+	var err error
+	kc, err := kubeclient.New("tailscale-container")
+	if err != nil {
+		return nil, fmt.Errorf("Error creating kube client: %w", err)
+	}
+	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		// Used to set http server in tests, or optionally enabled by flag
+		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	}
+	return &kubeClient{Client: kc, stateSecret: stateSecret}, nil
+}
+
+// storeDeviceID writes deviceID to 'device_id' data field of the client's state Secret.
+func (kc *kubeClient) storeDeviceID(ctx context.Context, deviceID tailcfg.StableNodeID) error {
+	s := &kubeapi.Secret{
+		Data: map[string][]byte{
+			kubetypes.KeyDeviceID: []byte(deviceID),
+		},
+	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
+}
+
+// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields 'device_ips', 'device_fqdn' of client's
+// state Secret.
+func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, addresses []netip.Prefix) error {
 	var ips []string
 	for _, addr := range addresses {
 		ips = append(ips, addr.Addr().String())
@@ -44,16 +70,28 @@ func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, a
 
 	s := &kubeapi.Secret{
 		Data: map[string][]byte{
-			"device_fqdn": []byte(fqdn),
-			"device_ips":  deviceIPs,
+			kubetypes.KeyDeviceFQDN: []byte(fqdn),
+			kubetypes.KeyDeviceIPs:  deviceIPs,
 		},
 	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
+}
+
+// storeHTTPSEndpoint writes an HTTPS endpoint exposed by this device via 'tailscale serve' to the client's state
+// Secret. In practice this will be the same value that gets written to 'device_fqdn', but this should only be called
+// when the serve config has been successfully set up.
+func (kc *kubeClient) storeHTTPSEndpoint(ctx context.Context, ep string) error {
+	s := &kubeapi.Secret{
+		Data: map[string][]byte{
+			kubetypes.KeyHTTPSEndpoint: []byte(ep),
+		},
+	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }
 
 // deleteAuthKey deletes the 'authkey' field of the given kube
 // secret. No-op if there is no authkey in the secret.
-func deleteAuthKey(ctx context.Context, secretName string) error {
+func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
 	m := []kubeclient.JSONPatch{
 		{
@@ -61,7 +99,7 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 			Path: "/data/authkey",
 		},
 	}
-	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
+	if err := kc.JSONPatchResource(ctx, kc.stateSecret, kubeclient.TypeSecrets, m); err != nil {
 		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
@@ -72,22 +110,19 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc kubeclient.Client
-
-func initKubeClient(root string) {
-	if root != "/" {
-		// If we are running in a test, we need to set the root path to the fake
-		// service account directory.
-		kubeclient.SetRootPathForTesting(root)
+// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale
+// state Secret.
+// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container.
+func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error {
+	capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
+	d := map[string][]byte{
+		kubetypes.KeyCapVer: []byte(capVerS),
 	}
-	var err error
-	kc, err = kubeclient.New()
-	if err != nil {
-		log.Fatalf("Error creating kube client: %v", err)
+	if podUID != "" {
+		d[kubetypes.KeyPodUID] = []byte(podUID)
 	}
-	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
-		// Derive the API server address from the environment variables
-		// Used to set http server in tests, or optionally enabled by flag
-		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	s := &kubeapi.Secret{
+		Data: d,
 	}
+	return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
 }

cmd/containerboot/kube_test.go

@@ -21,7 +21,7 @@ func TestSetupKube(t *testing.T) {
 		cfg     *settings
 		wantErr bool
 		wantCfg *settings
-		kc      kubeclient.Client
+		kc      *kubeClient
 	}{
 		{
 			name: "TS_AUTHKEY set, state Secret exists",
@@ -29,14 +29,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -48,14 +48,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -67,14 +67,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -87,14 +87,14 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 403}
 				},
-			},
+			}},
 			wantCfg: &settings{
 				AuthKey:    "foo",
 				KubeSecret: "foo",
@@ -111,11 +111,11 @@ func TestSetupKube(t *testing.T) {
 				AuthKey:    "foo",
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, errors.New("broken")
 				},
-			},
+			}},
 			wantErr: true,
 		},
 		{
@@ -127,14 +127,14 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, true, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return nil, &kubeapi.Status{Code: 404}
 				},
-			},
+			}},
 		},
 		{
 			// Interactive login using URL in Pod logs
@@ -145,28 +145,28 @@ func TestSetupKube(t *testing.T) {
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return &kubeapi.Secret{}, nil
 				},
-			},
+			}},
 		},
 		{
 			name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return false, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				KubeSecret: "foo",
 			},
@@ -177,14 +177,14 @@ func TestSetupKube(t *testing.T) {
 			cfg: &settings{
 				KubeSecret: "foo",
 			},
-			kc: &kubeclient.FakeClient{
+			kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
 				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
 					return true, false, nil
 				},
 				GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
 					return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
 				},
-			},
+			}},
 			wantCfg: &settings{
 				KubeSecret:         "foo",
 				AuthKey:            "foo",
@@ -194,9 +194,9 @@ func TestSetupKube(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		kc = tt.kc
+		kc := tt.kc
 		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
+			if err := tt.cfg.setupKube(context.Background(), kc); (err != nil) != tt.wantErr {
 				t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
 			}
 			if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {

cmd/containerboot/main.go

@@ -52,11 +52,17 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
-//   - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
-//     served at /healthz at the provided address, which should be in form [<address>]:<port>.
-//     If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
-//     The health endpoint will return 200 OK if this node has at least one tailnet IP address,
-//     otherwise returns 503.
+//   - TS_HEALTHCHECK_ADDR_PORT: deprecated, use TS_ENABLE_HEALTH_CHECK instead and optionally
+//     set TS_LOCAL_ADDR_PORT. Will be removed in 1.82.0.
+//   - TS_LOCAL_ADDR_PORT: the address and port to serve local metrics and health
+//     check endpoints if enabled via TS_ENABLE_METRICS and/or TS_ENABLE_HEALTH_CHECK.
+//     Defaults to [::]:9002, serving on all available interfaces.
+//   - TS_ENABLE_METRICS: if true, a metrics endpoint will be served at /metrics on
+//     the address specified by TS_LOCAL_ADDR_PORT. See https://tailscale.com/kb/1482/client-metrics
+//     for more information on the metrics exposed.
+//   - TS_ENABLE_HEALTH_CHECK: if true, a health check endpoint will be served at /healthz on
+//     the address specified by TS_LOCAL_ADDR_PORT. The health endpoint will return 200
+//     OK if this node has at least one tailnet IP address, otherwise returns 503.
 //     NB: the health criteria might change in the future.
 //   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
 //     directory that containers tailscaled config in file. The config file needs to be
@@ -99,10 +105,10 @@ import (
 	"log"
 	"math"
 	"net"
+	"net/http"
 	"net/netip"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -115,6 +121,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	kubeutils "tailscale.com/k8s-operator"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/ptr"
@@ -161,9 +168,13 @@ func main() {
 	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
+	var kc *kubeClient
 	if cfg.InKubernetes {
-		initKubeClient(cfg.Root)
-		if err := cfg.setupKube(bootCtx); err != nil {
+		kc, err = newKubeClient(cfg.Root, cfg.KubeSecret)
+		if err != nil {
+			log.Fatalf("error initializing kube client: %v", err)
+		}
+		if err := cfg.setupKube(bootCtx, kc); err != nil {
 			log.Fatalf("error setting up for running on Kubernetes: %v", err)
 		}
 	}
@@ -179,6 +190,34 @@ func main() {
 	}
 	defer killTailscaled()
 
+	var healthCheck *healthz
+	if cfg.HealthCheckAddrPort != "" {
+		mux := http.NewServeMux()
+
+		log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
+		healthCheck = healthHandlers(mux)
+
+		close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
+		defer close()
+	}
+
+	if cfg.localMetricsEnabled() || cfg.localHealthEnabled() {
+		mux := http.NewServeMux()
+
+		if cfg.localMetricsEnabled() {
+			log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort)
+			metricsHandlers(mux, client, cfg.DebugAddrPort)
+		}
+
+		if cfg.localHealthEnabled() {
+			log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
+			healthCheck = healthHandlers(mux)
+		}
+
+		close := runHTTPServer(mux, cfg.LocalAddrPort)
+		defer close()
+	}
+
 	if cfg.EnableForwardingOptimizations {
 		if err := client.SetUDPGROForwarding(bootCtx); err != nil {
 			log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
@@ -285,12 +324,18 @@ authLoop:
 		}
 	}
 
+	// Remove any serve config and advertised HTTPS endpoint that may have been set by a previous run of
+	// containerboot, but only if we're providing a new one.
 	if cfg.ServeConfigPath != "" {
-		// Remove any serve config that may have been set by a previous run of
-		// containerboot, but only if we're providing a new one.
+		log.Printf("serve proxy: unsetting previous config")
 		if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil {
 			log.Fatalf("failed to unset serve config: %v", err)
 		}
+		if hasKubeStateStore(cfg) {
+			if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil {
+				log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err)
+			}
+		}
 	}
 
 	if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) {
@@ -298,11 +343,17 @@ authLoop:
 		// authkey is no longer needed. We don't strictly need to
 		// wipe it, but it's good hygiene.
 		log.Printf("Deleting authkey from kube secret")
-		if err := deleteAuthKey(ctx, cfg.KubeSecret); err != nil {
+		if err := kc.deleteAuthKey(ctx); err != nil {
 			log.Fatalf("deleting authkey from kube secret: %v", err)
 		}
 	}
 
+	if hasKubeStateStore(cfg) {
+		if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil {
+			log.Fatalf("storing capability version and UID: %v", err)
+		}
+	}
+
 	w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 	if err != nil {
 		log.Fatalf("rewatching tailscaled for updates after auth: %v", err)
@@ -322,12 +373,9 @@ authLoop:
 		certDomain        = new(atomic.Pointer[string])
 		certDomainChanged = make(chan bool, 1)
 
-		h             = &healthz{} // http server for the healthz endpoint
-		healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
+		triggerWatchServeConfigChanges sync.Once
 	)
-	if cfg.ServeConfigPath != "" {
-		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
-	}
+
 	var nfr linuxfw.NetfilterRunner
 	if isL3Proxy(cfg) {
 		nfr, err = newNetfilterRunner(log.Printf)
@@ -428,7 +476,7 @@ runLoop:
 				// fails.
 				deviceID := n.NetMap.SelfNode.StableID()
 				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceID, &deviceID) {
-					if err := storeDeviceID(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID()); err != nil {
+					if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil {
 						log.Fatalf("storing device ID in Kubernetes Secret: %v", err)
 					}
 				}
@@ -501,8 +549,11 @@ runLoop:
 					resetTimer(false)
 					backendAddrs = newBackendAddrs
 				}
-				if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) != 0 {
-					cd := n.NetMap.DNS.CertDomains[0]
+				if cfg.ServeConfigPath != "" {
+					cd := certDomainFromNetmap(n.NetMap)
+					if cd == "" {
+						cd = kubetypes.ValueNoHTTPS
+					}
 					prev := certDomain.Swap(ptr.To(cd))
 					if prev == nil || *prev != cd {
 						select {
@@ -544,17 +595,21 @@ runLoop:
 				// TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'.
 				deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()}
 				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceEndpoints, &deviceEndpoints) {
-					if err := storeDeviceEndpoints(ctx, cfg.KubeSecret, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
+					if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
 						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
 
-				if cfg.HealthCheckAddrPort != "" {
-					h.Lock()
-					h.hasAddrs = len(addrs) != 0
-					h.Unlock()
-					healthzRunner()
+				if healthCheck != nil {
+					healthCheck.update(len(addrs) != 0)
 				}
+
+				if cfg.ServeConfigPath != "" {
+					triggerWatchServeConfigChanges.Do(func() {
+						go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc)
+					})
+				}
+
 				if egressSvcsNotify != nil {
 					egressSvcsNotify <- n
 				}
@@ -731,7 +786,6 @@ func tailscaledConfigFilePath() string {
 		}
 		cv, err := kubeutils.CapVerFromFileName(e.Name())
 		if err != nil {
-			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
 			continue
 		}
 		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
@@ -739,8 +793,28 @@ func tailscaledConfigFilePath() string {
 		}
 	}
 	if maxCompatVer == -1 {
-		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
+		log.Fatalf("no tailscaled config file found in %q for current capability version %d", dir, tailcfg.CurrentCapabilityVersion)
+	}
+	filePath := filepath.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	log.Printf("Using tailscaled config file %q to match current capability version %d", filePath, tailcfg.CurrentCapabilityVersion)
+	return filePath
+}
+
+func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) {
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("failed to listen on addr %q: %v", addr, err)
+	}
+	srv := &http.Server{Handler: mux}
+
+	go func() {
+		if err := srv.Serve(ln); err != nil {
+			log.Fatalf("failed running server: %v", err)
+		}
+	}()
+
+	return func() error {
+		err := srv.Shutdown(context.Background())
+		return errors.Join(err, ln.Close())
 	}
-	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
 }

cmd/containerboot/main_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn"
+	"tailscale.com/kube/egressservices"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/netmap"
@@ -57,6 +58,16 @@ func TestContainerBoot(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error unmarshaling tailscaled config: %v", err)
 	}
+	serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}}
+	serveConfBytes, err := json.Marshal(serveConf)
+	if err != nil {
+		t.Fatalf("error unmarshaling serve config: %v", err)
+	}
+	egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}}
+	egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg)
+	if err != nil {
+		t.Fatalf("error unmarshaling egress services config: %v", err)
+	}
 
 	dirs := []string{
 		"var/lib",
@@ -73,14 +84,16 @@ func TestContainerBoot(t *testing.T) {
 		}
 	}
 	files := map[string][]byte{
-		"usr/bin/tailscaled":                    fakeTailscaled,
-		"usr/bin/tailscale":                     fakeTailscale,
-		"usr/bin/iptables":                      fakeTailscale,
-		"usr/bin/ip6tables":                     fakeTailscale,
-		"dev/net/tun":                           []byte(""),
-		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
-		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
-		"etc/tailscaled/cap-95.hujson":          tailscaledConfBytes,
+		"usr/bin/tailscaled":                         fakeTailscaled,
+		"usr/bin/tailscale":                          fakeTailscale,
+		"usr/bin/iptables":                           fakeTailscale,
+		"usr/bin/ip6tables":                          fakeTailscale,
+		"dev/net/tun":                                []byte(""),
+		"proc/sys/net/ipv4/ip_forward":               []byte("0"),
+		"proc/sys/net/ipv6/conf/all/forwarding":      []byte("0"),
+		"etc/tailscaled/cap-95.hujson":               tailscaledConfBytes,
+		"etc/tailscaled/serve-config.json":           serveConfBytes,
+		"etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes,
 	}
 	resetFiles := func() {
 		for path, content := range files {
@@ -101,6 +114,26 @@ func TestContainerBoot(t *testing.T) {
 
 	argFile := filepath.Join(d, "args")
 	runningSockPath := filepath.Join(d, "tmp/tailscaled.sock")
+	var localAddrPort, healthAddrPort int
+	for _, p := range []*int{&localAddrPort, &healthAddrPort} {
+		ln, err := net.Listen("tcp", ":0")
+		if err != nil {
+			t.Fatalf("Failed to open listener: %v", err)
+		}
+		if err := ln.Close(); err != nil {
+			t.Fatalf("Failed to close listener: %v", err)
+		}
+		port := ln.Addr().(*net.TCPAddr).Port
+		*p = port
+	}
+	metricsURL := func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d/metrics", port)
+	}
+	healthURL := func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
+	}
+
+	capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
 
 	type phase struct {
 		// If non-nil, send this IPN bus notification (and remember it as the
@@ -119,6 +152,8 @@ func TestContainerBoot(t *testing.T) {
 		// WantFatalLog is the fatal log message we expect from containerboot.
 		// If set for a phase, the test will finish on that phase.
 		WantFatalLog string
+
+		EndpointStatuses map[string]int
 	}
 	runningNotify := &ipn.Notify{
 		State: ptr.To(ipn.Running),
@@ -147,6 +182,11 @@ func TestContainerBoot(t *testing.T) {
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
 					},
+					// No metrics or health by default.
+					EndpointStatuses: map[string]int{
+						metricsURL(9002): -1,
+						healthURL(9002):  -1,
+					},
 				},
 				{
 					Notify: runningNotify,
@@ -453,10 +493,11 @@ func TestContainerBoot(t *testing.T) {
 				{
 					Notify: runningNotify,
 					WantKubeSecret: map[string]string{
-						"authkey":     "tskey-key",
-						"device_fqdn": "test-node.test.ts.net",
-						"device_id":   "myID",
-						"device_ips":  `["100.64.0.1"]`,
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 			},
@@ -546,9 +587,10 @@ func TestContainerBoot(t *testing.T) {
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
 					},
 					WantKubeSecret: map[string]string{
-						"device_fqdn": "test-node.test.ts.net",
-						"device_id":   "myID",
-						"device_ips":  `["100.64.0.1"]`,
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 			},
@@ -575,10 +617,11 @@ func TestContainerBoot(t *testing.T) {
 				{
 					Notify: runningNotify,
 					WantKubeSecret: map[string]string{
-						"authkey":     "tskey-key",
-						"device_fqdn": "test-node.test.ts.net",
-						"device_id":   "myID",
-						"device_ips":  `["100.64.0.1"]`,
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 				{
@@ -593,10 +636,11 @@ func TestContainerBoot(t *testing.T) {
 						},
 					},
 					WantKubeSecret: map[string]string{
-						"authkey":     "tskey-key",
-						"device_fqdn": "new-name.test.ts.net",
-						"device_id":   "newID",
-						"device_ips":  `["100.64.0.1"]`,
+						"authkey":          "tskey-key",
+						"device_fqdn":      "new-name.test.ts.net",
+						"device_id":        "newID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
 					},
 				},
 			},
@@ -700,6 +744,199 @@ func TestContainerBoot(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "metrics_enabled",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_METRICS":  "true",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(localAddrPort):  -1,
+					},
+				}, {
+					Notify: runningNotify,
+				},
+			},
+		},
+		{
+			Name: "health_enabled",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT":     fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_HEALTH_CHECK": "true",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): -1,
+						healthURL(localAddrPort):  503, // Doesn't start passing until the next phase.
+					},
+				}, {
+					Notify: runningNotify,
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): -1,
+						healthURL(localAddrPort):  200,
+					},
+				},
+			},
+		},
+		{
+			Name: "metrics_and_health_on_same_port",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT":     fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_METRICS":      "true",
+				"TS_ENABLE_HEALTH_CHECK": "true",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(localAddrPort):  503, // Doesn't start passing until the next phase.
+					},
+				}, {
+					Notify: runningNotify,
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(localAddrPort):  200,
+					},
+				},
+			},
+		},
+		{
+			Name: "local_metrics_and_deprecated_health",
+			Env: map[string]string{
+				"TS_LOCAL_ADDR_PORT":       fmt.Sprintf("[::]:%d", localAddrPort),
+				"TS_ENABLE_METRICS":        "true",
+				"TS_HEALTHCHECK_ADDR_PORT": fmt.Sprintf("[::]:%d", healthAddrPort),
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(healthAddrPort): 503, // Doesn't start passing until the next phase.
+					},
+				}, {
+					Notify: runningNotify,
+					EndpointStatuses: map[string]int{
+						metricsURL(localAddrPort): 200,
+						healthURL(healthAddrPort): 200,
+					},
+				},
+			},
+		},
+		{
+			Name: "serve_config_no_kube",
+			Env: map[string]string{
+				"TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"),
+				"TS_AUTHKEY":      "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+				},
+			},
+		},
+		{
+			Name: "serve_config_kube",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_SERVE_CONFIG":               filepath.Join(d, "etc/tailscaled/serve-config.json"),
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+					WantKubeSecret: map[string]string{
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"https_endpoint":   "no-https",
+						"tailscale_capver": capver,
+					},
+				},
+			},
+		},
+		{
+			Name: "egress_svcs_config_kube",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":        kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS":  kube.Port,
+				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+					WantKubeSecret: map[string]string{
+						"authkey":          "tskey-key",
+						"device_fqdn":      "test-node.test.ts.net",
+						"device_id":        "myID",
+						"device_ips":       `["100.64.0.1"]`,
+						"tailscale_capver": capver,
+					},
+				},
+			},
+		},
+		{
+			Name: "egress_svcs_config_no_kube",
+			Env: map[string]string{
+				"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
+				"TS_AUTHKEY":                     "tskey-key",
+			},
+			Phases: []phase{
+				{
+					WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -796,7 +1033,26 @@ func TestContainerBoot(t *testing.T) {
 					return nil
 				})
 				if err != nil {
-					t.Fatal(err)
+					t.Fatalf("phase %d: %v", i, err)
+				}
+
+				for url, want := range p.EndpointStatuses {
+					err := tstest.WaitFor(2*time.Second, func() error {
+						resp, err := http.Get(url)
+						if err != nil && want != -1 {
+							return fmt.Errorf("GET %s: %v", url, err)
+						}
+						if want > 0 && resp.StatusCode != want {
+							defer resp.Body.Close()
+							body, _ := io.ReadAll(resp.Body)
+							return fmt.Errorf("GET %s, want %d, got %d\n%s", url, want, resp.StatusCode, string(body))
+						}
+
+						return nil
+					})
+					if err != nil {
+						t.Fatalf("phase %d: %v", i, err)
+					}
 				}
 			}
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
@@ -955,6 +1211,12 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if r.Method != "GET" {
 			panic(fmt.Sprintf("unsupported method %q", r.Method))
 		}
+	case "/localapi/v0/usermetrics":
+		if r.Method != "GET" {
+			panic(fmt.Sprintf("unsupported method %q", r.Method))
+		}
+		w.Write([]byte("fake metrics"))
+		return
 	default:
 		panic(fmt.Sprintf("unsupported path %q", r.URL.Path))
 	}

cmd/containerboot/metrics.go

@@ -0,0 +1,79 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
+)
+
+// metrics is a simple metrics HTTP server, if enabled it forwards requests to
+// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
+type metrics struct {
+	debugEndpoint string
+	lc            *tailscale.LocalClient
+}
+
+func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+
+	resp, err := do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	for key, val := range resp.Header {
+		for _, v := range val {
+			w.Header().Add(key, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
+	localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
+	proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
+}
+
+func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
+	if m.debugEndpoint == "" {
+		http.Error(w, "debug endpoint not configured", http.StatusNotFound)
+		return
+	}
+
+	debugURL := "http://" + m.debugEndpoint + r.URL.Path
+	proxy(w, r, debugURL, http.DefaultClient.Do)
+}
+
+// metricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding
+// requests to tailscaled's /localapi/v0/usermetrics API.
+//
+// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
+// endpoint if configured to ease migration for a breaking change serving user
+// metrics instead of debug metrics on the "metrics" port.
+func metricsHandlers(mux *http.ServeMux, lc *tailscale.LocalClient, debugAddrPort string) {
+	m := &metrics{
+		lc:            lc,
+		debugEndpoint: debugAddrPort,
+	}
+
+	mux.HandleFunc("GET /metrics", m.handleMetrics)
+	mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
+}

cmd/containerboot/serve.go

@@ -19,6 +19,8 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/types/netmap"
 )
 
 // watchServeConfigChanges watches path for changes, and when it sees one, reads
@@ -26,21 +28,21 @@ import (
 // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
 // is written to when the certDomain changes, causing the serve config to be
 // re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient, kc *kubeClient) {
 	if certDomainAtomic == nil {
-		panic("cd must not be nil")
+		panic("certDomainAtomic must not be nil")
 	}
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	if w, err := fsnotify.NewWatcher(); err != nil {
-		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		log.Printf("serve proxy: failed to create fsnotify watcher, timer-only mode: %v", err)
 		ticker := time.NewTicker(5 * time.Second)
 		defer ticker.Stop()
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
 		if err := w.Add(filepath.Dir(path)); err != nil {
-			log.Fatalf("failed to add fsnotify watch: %v", err)
+			log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
 		}
 		eventChan = w.Events
 	}
@@ -59,24 +61,62 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			// k8s handles these mounts. So just re-read the file and apply it
 			// if it's changed.
 		}
-		if certDomain == "" {
-			continue
-		}
 		sc, err := readServeConfig(path, certDomain)
 		if err != nil {
-			log.Fatalf("failed to read serve config: %v", err)
+			log.Fatalf("serve proxy: failed to read serve config: %v", err)
 		}
 		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
 			continue
 		}
-		log.Printf("Applying serve config")
-		if err := lc.SetServeConfig(ctx, sc); err != nil {
-			log.Fatalf("failed to set serve config: %v", err)
+		validateHTTPSServe(certDomain, sc)
+		if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil {
+			log.Fatalf("serve proxy: error updating serve config: %v", err)
+		}
+		if kc != nil && kc.canPatch {
+			if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil {
+				log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err)
+			}
 		}
 		prevServeConfig = sc
 	}
 }
 
+func certDomainFromNetmap(nm *netmap.NetworkMap) string {
+	if len(nm.DNS.CertDomains) == 0 {
+		return ""
+	}
+	return nm.DNS.CertDomains[0]
+}
+
+func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc *tailscale.LocalClient) error {
+	// TODO(irbekrm): This means that serve config that does not expose HTTPS endpoint will not be set for a tailnet
+	// that does not have HTTPS enabled. We probably want to fix this.
+	if certDomain == kubetypes.ValueNoHTTPS {
+		return nil
+	}
+	log.Printf("serve proxy: applying serve config")
+	return lc.SetServeConfig(ctx, sc)
+}
+
+func validateHTTPSServe(certDomain string, sc *ipn.ServeConfig) {
+	if certDomain != kubetypes.ValueNoHTTPS || !hasHTTPSEndpoint(sc) {
+		return
+	}
+	log.Printf(
+		`serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet,
+		(perhaps a Kubernetes operator Ingress proxy) but it is not able to issue TLS certs, so this will likely not work.
+		To make it work, ensure that HTTPS is enabled for your tailnet, see https://tailscale.com/kb/1153/enabling-https for more details.`)
+}
+
+func hasHTTPSEndpoint(cfg *ipn.ServeConfig) bool {
+	for _, tcpCfg := range cfg.TCP {
+		if tcpCfg.HTTPS {
+			return true
+		}
+	}
+	return false
+}
+
 // readServeConfig reads the ipn.ServeConfig from path, replacing
 // ${TS_CERT_DOMAIN} with certDomain.
 func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {

cmd/containerboot/services.go

@@ -389,7 +389,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta
 		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
 		Value: bs,
 	}
-	if err := ep.kc.JSONPatchSecret(ctx, ep.stateSecret, []kubeclient.JSONPatch{patch}); err != nil {
+	if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil {
 		return fmt.Errorf("error patching state Secret: %w", err)
 	}
 	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()

cmd/containerboot/settings.go

@@ -67,7 +67,12 @@ type settings struct {
 	PodIP               string
 	PodIPv4             string
 	PodIPv6             string
+	PodUID              string
 	HealthCheckAddrPort string
+	LocalAddrPort       string
+	MetricsEnabled      bool
+	HealthCheckEnabled  bool
+	DebugAddrPort       string
 	EgressSvcsCfgPath   string
 }
 
@@ -98,7 +103,12 @@ func configFromEnv() (*settings, error) {
 		PodIP:                                 defaultEnv("POD_IP", ""),
 		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
 		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
+		LocalAddrPort:                         defaultEnv("TS_LOCAL_ADDR_PORT", "[::]:9002"),
+		MetricsEnabled:                        defaultBool("TS_ENABLE_METRICS", false),
+		HealthCheckEnabled:                    defaultBool("TS_ENABLE_HEALTH_CHECK", false),
+		DebugAddrPort:                         defaultEnv("TS_DEBUG_ADDR_PORT", ""),
 		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
+		PodUID:                                defaultEnv("POD_UID", ""),
 	}
 	podIPs, ok := os.LookupEnv("POD_IPS")
 	if ok {
@@ -171,17 +181,34 @@ func (s *settings) validate() error {
 		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
 	}
 	if s.HealthCheckAddrPort != "" {
+		log.Printf("[warning] TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0. Please use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT instead.")
 		if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
-			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
+			return fmt.Errorf("error parsing TS_HEALTHCHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
 		}
 	}
+	if s.localMetricsEnabled() || s.localHealthEnabled() {
+		if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
+		}
+	}
+	if s.DebugAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.DebugAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_DEBUG_ADDR_PORT value %q: %w", s.DebugAddrPort, err)
+		}
+	}
+	if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" {
+		return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT")
+	}
+	if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
+		return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
+	}
 	return nil
 }
 
 // setupKube is responsible for doing any necessary configuration and checks to
 // ensure that tailscale state storage and authentication mechanism will work on
 // Kubernetes.
-func (cfg *settings) setupKube(ctx context.Context) error {
+func (cfg *settings) setupKube(ctx context.Context, kc *kubeClient) error {
 	if cfg.KubeSecret == "" {
 		return nil
 	}
@@ -190,6 +217,7 @@ func (cfg *settings) setupKube(ctx context.Context) error {
 		return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
 	}
 	cfg.KubernetesCanPatch = canPatch
+	kc.canPatch = canPatch
 
 	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
 	if err != nil {
@@ -272,6 +300,14 @@ func hasKubeStateStore(cfg *settings) bool {
 	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
 }
 
+func (cfg *settings) localMetricsEnabled() bool {
+	return cfg.LocalAddrPort != "" && cfg.MetricsEnabled
+}
+
+func (cfg *settings) localHealthEnabled() bool {
+	return cfg.LocalAddrPort != "" && cfg.HealthCheckEnabled
+}
+
 // defaultEnv returns the value of the given envvar name, or defVal if
 // unset.
 func defaultEnv(name, defVal string) string {

cmd/containerboot/tailscaled.go

@@ -90,6 +90,12 @@ func tailscaledArgs(cfg *settings) []string {
 	if cfg.TailscaledConfigFilePath != "" {
 		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
 	}
+	// Once enough proxy versions have been released for all the supported
+	// versions to understand this cfg setting, the operator can stop
+	// setting TS_TAILSCALED_EXTRA_ARGS for the debug flag.
+	if cfg.DebugAddrPort != "" && !strings.Contains(cfg.DaemonExtraArgs, cfg.DebugAddrPort) {
+		args = append(args, "--debug="+cfg.DebugAddrPort)
+	}
 	if cfg.DaemonExtraArgs != "" {
 		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
 	}

cmd/derper/cert.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"path/filepath"
 	"regexp"
@@ -53,8 +54,9 @@ func certProviderByCertMode(mode, dir, hostname string) (certProvider, error) {
 }
 
 type manualCertManager struct {
-	cert     *tls.Certificate
-	hostname string
+	cert       *tls.Certificate
+	hostname   string // hostname or IP address of server
+	noHostname bool   // whether hostname is an IP address
 }
 
 // NewManualCertManager returns a cert provider which read certificate by given hostname on create.
@@ -74,7 +76,11 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	if err := x509Cert.VerifyHostname(hostname); err != nil {
 		return nil, fmt.Errorf("cert invalid for hostname %q: %w", hostname, err)
 	}
-	return &manualCertManager{cert: &cert, hostname: hostname}, nil
+	return &manualCertManager{
+		cert:       &cert,
+		hostname:   hostname,
+		noHostname: net.ParseIP(hostname) != nil,
+	}, nil
 }
 
 func (m *manualCertManager) TLSConfig() *tls.Config {
@@ -88,7 +94,7 @@ func (m *manualCertManager) TLSConfig() *tls.Config {
 }
 
 func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	if hi.ServerName != m.hostname {
+	if hi.ServerName != m.hostname && !m.noHostname {
 		return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
 	}
 

cmd/derper/cert_test.go

@@ -0,0 +1,97 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// Verify that in --certmode=manual mode, we can use a bare IP address
+// as the --hostname and that GetCertificate will return it.
+func TestCertIP(t *testing.T) {
+	dir := t.TempDir()
+	const hostname = "1.2.3.4"
+
+	priv, err := ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ip := net.ParseIP(hostname)
+	if ip == nil {
+		t.Fatalf("invalid IP address %q", hostname)
+	}
+	template := &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"Tailscale Test Corp"},
+		},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(30 * 24 * time.Hour),
+
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IPAddresses:           []net.IP{ip},
+	}
+	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	certOut, err := os.Create(filepath.Join(dir, hostname+".crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		t.Fatalf("Failed to write data to cert.pem: %v", err)
+	}
+	if err := certOut.Close(); err != nil {
+		t.Fatalf("Error closing cert.pem: %v", err)
+	}
+
+	keyOut, err := os.OpenFile(filepath.Join(dir, hostname+".key"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	privBytes, err := x509.MarshalPKCS8PrivateKey(priv)
+	if err != nil {
+		t.Fatalf("Unable to marshal private key: %v", err)
+	}
+	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
+		t.Fatalf("Failed to write data to key.pem: %v", err)
+	}
+	if err := keyOut.Close(); err != nil {
+		t.Fatalf("Error closing key.pem: %v", err)
+	}
+
+	cp, err := certProviderByCertMode("manual", dir, hostname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	back, err := cp.TLSConfig().GetCertificate(&tls.ClientHelloInfo{
+		ServerName: "", // no SNI
+	})
+	if err != nil {
+		t.Fatalf("GetCertificate: %v", err)
+	}
+	if back == nil {
+		t.Fatalf("GetCertificate returned nil")
+	}
+}

cmd/derper/depaware.txt

@@ -27,7 +27,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
@@ -113,9 +112,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
-        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
+        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper
         tailscale.com/paths                                          from tailscale.com/client/tailscale
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
@@ -139,6 +139,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/ipn
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/ipn+
@@ -150,24 +151,29 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/set                                       from tailscale.com/derp+
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/testenv                                   from tailscale.com/util/syspolicy+
         tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
@@ -188,7 +194,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
-        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -237,7 +243,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         crypto/tls                                                   from golang.org/x/crypto/acme+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -249,7 +254,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper
+        flag                                                         from tailscale.com/cmd/derper+
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -257,6 +262,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
+        html/template                                                from tailscale.com/cmd/derper
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
@@ -268,7 +274,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -283,7 +289,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
-   W    os/user                                                      from tailscale.com/util/winutil
+   W    os/user                                                      from tailscale.com/util/winutil+
         path                                                         from github.com/prometheus/client_golang/prometheus/internal+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
@@ -301,6 +307,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
         text/tabwriter                                               from runtime/pprof
+        text/template                                                from html/template
+        text/template/parse                                          from html/template+
         time                                                         from compress/gzip+
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+

cmd/derper/derper.go

@@ -19,6 +19,7 @@ import (
 	"expvar"
 	"flag"
 	"fmt"
+	"html/template"
 	"io"
 	"log"
 	"math"
@@ -57,7 +58,7 @@ var (
 	configPath  = flag.String("c", "", "config file path")
 	certMode    = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
 	certDir     = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
-	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
+	hostname    = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks")
 	runSTUN     = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
 	runDERP     = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
 
@@ -212,25 +213,16 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
 		w.WriteHeader(200)
-		io.WriteString(w, `<html><body>
-<h1>DERP</h1>
-<p>
-  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
-</p>
-<p>
-  Documentation:
-</p>
-<ul>
-  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
-  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
-  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
-</ul>
-`)
-		if !*runDERP {
-			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
-		}
-		if tsweb.AllowDebugAccess(r) {
-			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
+		err := homePageTemplate.Execute(w, templateData{
+			ShowAbuseInfo: validProdHostname.MatchString(*hostname),
+			Disabled:      !*runDERP,
+			AllowDebug:    tsweb.AllowDebugAccess(r),
+		})
+		if err != nil {
+			if r.Context().Err() == nil {
+				log.Printf("homePageTemplate.Execute: %v", err)
+			}
+			return
 		}
 	}))
 	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -468,3 +460,52 @@ func init() {
 		return 0
 	}))
 }
+
+type templateData struct {
+	ShowAbuseInfo bool
+	Disabled      bool
+	AllowDebug    bool
+}
+
+// homePageTemplate renders the home page using [templateData].
+var homePageTemplate = template.Must(template.New("home").Parse(`<html><body>
+<h1>DERP</h1>
+<p>
+  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
+</p>
+
+<p>
+  It provides STUN, interactive connectivity establishment, and relaying of end-to-end encrypted traffic
+  for Tailscale clients.
+</p>
+
+{{if .ShowAbuseInfo }}
+<p>
+  If you suspect abuse, please contact <a href="mailto:security@tailscale.com">security@tailscale.com</a>.
+</p>
+{{end}}
+
+<p>
+  Documentation:
+</p>
+
+<ul>
+{{if .ShowAbuseInfo }}
+  <li><a href="https://tailscale.com/security-policies">Tailscale Security Policies</a></li>
+  <li><a href="https://tailscale.com/tailscale-aup">Tailscale Acceptable Use Policies</a></li>
+{{end}}
+  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
+  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
+  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
+</ul>
+
+{{if .Disabled}}
+<p>Status: <b>disabled</b></p>
+{{end}}
+
+{{if .AllowDebug}}
+<p>Debug info at <a href='/debug/'>/debug/</a>.</p>
+{{end}}
+</body>
+</html>
+`))

cmd/derper/derper_test.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"net/http/httptest"
@@ -107,6 +108,33 @@ func TestDeps(t *testing.T) {
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
 			"tailscale.com/net/packet":           "not needed in derper",
 			"github.com/gaissmai/bart":           "not needed in derper",
+			"database/sql/driver":                "not needed in derper", // previously came in via github.com/google/uuid
 		},
 	}.Check(t)
 }
+
+func TestTemplate(t *testing.T) {
+	buf := &bytes.Buffer{}
+	err := homePageTemplate.Execute(buf, templateData{
+		ShowAbuseInfo: true,
+		Disabled:      true,
+		AllowDebug:    true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	str := buf.String()
+	if !strings.Contains(str, "If you suspect abuse") {
+		t.Error("Output is missing abuse mailto")
+	}
+	if !strings.Contains(str, "Tailscale Security Policies") {
+		t.Error("Output is missing Tailscale Security Policies link")
+	}
+	if !strings.Contains(str, "Status:") {
+		t.Error("Output is missing disabled status")
+	}
+	if !strings.Contains(str, "Debug info") {
+		t.Error("Output is missing debug info")
+	}
+}

cmd/derpprobe/derpprobe.go

@@ -29,6 +29,7 @@ var (
 	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
 	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
 	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	regionCode   = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -47,6 +48,9 @@ func main() {
 	if *bwInterval > 0 {
 		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
 	}
+	if *regionCode != "" {
+		opts = append(opts, prober.WithRegion(*regionCode))
+	}
 	dp, err := prober.DERP(p, *derpMapURL, opts...)
 	if err != nil {
 		log.Fatal(err)
@@ -75,6 +79,11 @@ func main() {
 		prober.WithPageLink("Prober metrics", "/debug/varz"),
 		prober.WithProbeLink("Run Probe", "/debug/probe-run?name={{.Name}}"),
 	), tsweb.HandlerOptions{Logf: log.Printf}))
+	mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok\n"))
+	}))
 	log.Printf("Listening on %s", *listen)
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }

cmd/get-authkey/main.go

@@ -46,7 +46,6 @@ func main() {
 		ClientID:     clientID,
 		ClientSecret: clientSecret,
 		TokenURL:     baseURL + "/api/v2/oauth/token",
-		Scopes:       []string{"device"},
 	}
 
 	ctx := context.Background()

cmd/gitops-pusher/gitops-pusher.go

@@ -58,8 +58,8 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		}
 
 		if cache.PrevETag == "" {
-			log.Println("no previous etag found, assuming local file is correct and recording that")
-			cache.PrevETag = localEtag
+			log.Println("no previous etag found, assuming the latest control etag")
+			cache.PrevETag = controlEtag
 		}
 
 		log.Printf("control: %s", controlEtag)
@@ -105,8 +105,8 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		}
 
 		if cache.PrevETag == "" {
-			log.Println("no previous etag found, assuming local file is correct and recording that")
-			cache.PrevETag = localEtag
+			log.Println("no previous etag found, assuming the latest control etag")
+			cache.PrevETag = controlEtag
 		}
 
 		log.Printf("control: %s", controlEtag)
@@ -148,8 +148,8 @@ func getChecksums(cache *Cache, client *http.Client, tailnet, apiKey string) fun
 		}
 
 		if cache.PrevETag == "" {
-			log.Println("no previous etag found, assuming local file is correct and recording that")
-			cache.PrevETag = Shuck(localEtag)
+			log.Println("no previous etag found, assuming control etag")
+			cache.PrevETag = Shuck(controlEtag)
 		}
 
 		log.Printf("control: %s", controlEtag)

cmd/k8s-operator/connector.go

@@ -10,10 +10,12 @@ import (
 	"fmt"
 	"net/netip"
 	"slices"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/pkg/errors"
+	"errors"
+
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
@@ -34,6 +36,7 @@ import (
 
 const (
 	reasonConnectorCreationFailed = "ConnectorCreationFailed"
+	reasonConnectorCreating       = "ConnectorCreating"
 	reasonConnectorCreated        = "ConnectorCreated"
 	reasonConnectorInvalid        = "ConnectorInvalid"
 
@@ -58,6 +61,7 @@ type ConnectorReconciler struct {
 
 	subnetRouters set.Slice[types.UID] // for subnet routers gauge
 	exitNodes     set.Slice[types.UID] // for exit nodes gauge
+	appConnectors set.Slice[types.UID] // for app connectors gauge
 }
 
 var (
@@ -67,6 +71,8 @@ var (
 	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
 	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
+	// gaugeConnectorAppConnectorResources tracks the number of Connectors currently managed by this operator instance that are app connectors.
+	gaugeConnectorAppConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithAppConnectorCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -108,13 +114,12 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	oldCnStatus := cn.Status.DeepCopy()
 	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
+		var updateErr error
+		if !apiequality.Semantic.DeepEqual(oldCnStatus, &cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
+			updateErr = a.Client.Status().Update(ctx, cn)
 		}
-		return res, err
+		return res, errors.Join(err, updateErr)
 	}
 
 	if !slices.Contains(cn.Finalizers, FinalizerName) {
@@ -131,17 +136,24 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	}
 
 	if err := a.validate(cn); err != nil {
-		logger.Errorf("error validating Connector spec: %w", err)
 		message := fmt.Sprintf(messageConnectorInvalid, err)
 		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorInvalid, message)
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reasonConnectorInvalid, message)
 	}
 
 	if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil {
-		logger.Errorf("error creating Connector resources: %w", err)
+		reason := reasonConnectorCreationFailed
 		message := fmt.Sprintf(messageConnectorCreationFailed, err)
-		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorCreationFailed, message)
-		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reasonConnectorCreationFailed, message)
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			reason = reasonConnectorCreating
+			message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
+			err = nil
+			logger.Info(message)
+		} else {
+			a.recorder.Eventf(cn, corev1.EventTypeWarning, reason, message)
+		}
+
+		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reason, message)
 	}
 
 	logger.Info("Connector resources synced")
@@ -150,6 +162,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 	}
+	if cn.Spec.AppConnector != nil {
+		cn.Status.IsAppConnector = true
+	}
 	cn.Status.SubnetRoutes = ""
 	return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 }
@@ -183,29 +198,44 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 			isExitNode: cn.Spec.ExitNode,
 		},
 		ProxyClassName: proxyClass,
+		proxyType:      proxyTypeConnector,
 	}
 
 	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
 		sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 	}
 
+	if cn.Spec.AppConnector != nil {
+		sts.Connector.isAppConnector = true
+		if len(cn.Spec.AppConnector.Routes) != 0 {
+			sts.Connector.routes = cn.Spec.AppConnector.Routes.Stringify()
+		}
+	}
+
 	a.mu.Lock()
-	if sts.Connector.isExitNode {
+	if cn.Spec.ExitNode {
 		a.exitNodes.Add(cn.UID)
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
-	if sts.Connector.routes != "" {
+	if cn.Spec.SubnetRouter != nil {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
+	if cn.Spec.AppConnector != nil {
+		a.appConnectors.Add(cn.GetUID())
+	} else {
+		a.appConnectors.Remove(cn.GetUID())
+	}
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
@@ -213,27 +243,27 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		return err
 	}
 
-	_, tsHost, ips, err := a.ssr.DeviceInfo(ctx, crl)
+	dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
 	if err != nil {
 		return err
 	}
 
-	if tsHost == "" {
-		logger.Debugf("no Tailscale hostname known yet, waiting for connector pod to finish auth")
+	if dev == nil || dev.hostname == "" {
+		logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth")
 		// No hostname yet. Wait for the connector pod to auth.
 		cn.Status.TailnetIPs = nil
 		cn.Status.Hostname = ""
 		return nil
 	}
 
-	cn.Status.TailnetIPs = ips
-	cn.Status.Hostname = tsHost
+	cn.Status.TailnetIPs = dev.ips
+	cn.Status.Hostname = dev.hostname
 
 	return nil
 }
 
 func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
-	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector")); err != nil {
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil {
 		return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
 	} else if !done {
 		logger.Debugf("Connector cleanup not done yet, waiting for next reconcile")
@@ -248,12 +278,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
 	a.mu.Lock()
 	a.subnetRouters.Remove(cn.UID)
 	a.exitNodes.Remove(cn.UID)
+	a.appConnectors.Remove(cn.UID)
 	a.mu.Unlock()
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 	return true, nil
 }
@@ -262,8 +295,14 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 	// Connector fields are already validated at apply time with CEL validation
 	// on custom resource fields. The checks here are a backup in case the
 	// CEL validation breaks without us noticing.
-	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
-		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
+	if cn.Spec.SubnetRouter == nil && !cn.Spec.ExitNode && cn.Spec.AppConnector == nil {
+		return errors.New("invalid spec: a Connector must be configured as at least one of subnet router, exit node or app connector")
+	}
+	if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
+		return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
+	}
+	if cn.Spec.AppConnector != nil {
+		return validateAppConnector(cn.Spec.AppConnector)
 	}
 	if cn.Spec.SubnetRouter == nil {
 		return nil
@@ -272,19 +311,27 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 }
 
 func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
-	if len(sb.AdvertiseRoutes) < 1 {
+	if len(sb.AdvertiseRoutes) == 0 {
 		return errors.New("invalid subnet router spec: no routes defined")
 	}
-	var err error
-	for _, route := range sb.AdvertiseRoutes {
+	return validateRoutes(sb.AdvertiseRoutes)
+}
+
+func validateAppConnector(ac *tsapi.AppConnector) error {
+	return validateRoutes(ac.Routes)
+}
+
+func validateRoutes(routes tsapi.Routes) error {
+	var errs []error
+	for _, route := range routes {
 		pfx, e := netip.ParsePrefix(string(route))
 		if e != nil {
-			err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err))
+			errs = append(errs, fmt.Errorf("route %v is invalid: %v", route, e))
 			continue
 		}
 		if pfx.Masked() != pfx {
-			err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
+			errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
 		}
 	}
-	return err
+	return errors.Join(errs...)
 }

cmd/k8s-operator/connector_test.go

@@ -8,12 +8,14 @@ package main
 import (
 	"context"
 	"testing"
+	"time"
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -296,3 +298,100 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	expectReconciled(t, cr, "", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 }
+
+func TestConnectorWithAppConnector(t *testing.T) {
+	// Setup
+	cn := &tsapi.Connector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  types.UID("1234-UID"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       tsapi.ConnectorKind,
+			APIVersion: "tailscale.io/v1alpha1",
+		},
+		Spec: tsapi.ConnectorSpec{
+			AppConnector: &tsapi.AppConnector{},
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(cn).
+		WithStatusSubresource(cn).
+		Build()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	fr := record.NewFakeRecorder(1)
+	cr := &ConnectorReconciler{
+		Client: fc,
+		clock:  cl,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		recorder: fr,
+	}
+
+	// 1. Connector with app connnector is created and becomes ready
+	expectReconciled(t, cr, "", "test")
+	fullName, shortName := findGenName(t, fc, "", "test", "connector")
+	opts := configOpts{
+		stsName:        shortName,
+		secretName:     fullName,
+		parentType:     "connector",
+		hostname:       "test-connector",
+		app:            kubetypes.AppConnector,
+		isAppConnector: true,
+	}
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	// Connector's ready condition should be set to true
+
+	cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
+	cn.Status.IsAppConnector = true
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectEqual(t, fc, cn, nil)
+
+	// 2. Connector with invalid app connector routes has status set to invalid
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	expectReconciled(t, cr, "", "test")
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionFalse,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorInvalid,
+		Message:            "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
+	}}
+	expectEqual(t, fc, cn, nil)
+
+	// 3. Connector with valid app connnector routes becomes ready
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectReconciled(t, cr, "", "test")
+}

cmd/k8s-operator/depaware.txt

@@ -80,10 +80,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -310,7 +306,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
@@ -382,7 +378,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         k8s.io/api/storage/v1beta1                                   from k8s.io/client-go/applyconfigurations/storage/v1beta1+
         k8s.io/api/storagemigration/v1alpha1                         from k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1+
         k8s.io/apiextensions-apiserver/pkg/apis/apiextensions        from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
-     💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1     from sigs.k8s.io/controller-runtime/pkg/webhook/conversion
+     💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1     from sigs.k8s.io/controller-runtime/pkg/webhook/conversion+
         k8s.io/apimachinery/pkg/api/equality                         from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
         k8s.io/apimachinery/pkg/api/errors                           from k8s.io/apimachinery/pkg/util/managedfields/internal+
         k8s.io/apimachinery/pkg/api/meta                             from k8s.io/apimachinery/pkg/api/validation+
@@ -658,6 +654,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/ipn/localapi+
@@ -668,6 +665,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/doctor/routetable                              from tailscale.com/ipn/ipnlocal
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -734,11 +732,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/tsd+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -773,6 +771,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/appc+
@@ -790,7 +789,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/appc+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlclient+
@@ -810,8 +809,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
@@ -821,7 +823,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -35,9 +35,13 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      - name: oauth
-        secret:
-          secretName: operator-oauth
+        - name: oauth
+          {{- with .Values.oauthSecretVolume }}
+          {{- toYaml . | nindent 10 }}
+          {{- else }}
+          secret:
+            secretName: operator-oauth
+          {{- end }}
       containers:
         - name: operator
           {{- with .Values.operatorConfig.securityContext }}
@@ -81,6 +85,14 @@ spec:
             - name: PROXY_DEFAULT_CLASS
               value: {{ .Values.proxyConfig.defaultProxyClass }}
             {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.ingressClass.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: IngressClass
 metadata:
@@ -6,3 +7,4 @@ metadata:
 spec:
   controller: tailscale.com/ts-ingress # controller name currently can not be changed
   # parameters: {} # currently no parameters are supported
+{{- end }}

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -6,6 +6,10 @@ kind: ServiceAccount
 metadata:
   name: operator
   namespace: {{ .Release.Namespace }}
+  {{- with .Values.operatorConfig.serviceAccountAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -30,6 +34,10 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["recorders", "recorders/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["get", "list", "watch"]
+  resourceNames: ["servicemonitors.monitoring.coreos.com"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -65,6 +73,9 @@ rules:
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
   verbs: ["get", "create", "patch", "update", "list", "watch"]
+- apiGroups: ["monitoring.coreos.com"]
+  resources: ["servicemonitors"]
+  verbs: ["get", "list", "update", "create", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -16,6 +16,9 @@ rules:
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -3,11 +3,26 @@
 
 # Operator oauth credentials. If set a Kubernetes Secret with the provided
 # values will be created in the operator namespace. If unset a Secret named
-# operator-oauth must be precreated.
+# operator-oauth must be precreated or oauthSecretVolume needs to be adjusted.
+# This block will be overridden by oauthSecretVolume, if set.
 oauth: {}
   # clientId: ""
   # clientSecret: ""
 
+# Secret volume.
+# If set it defines the volume the oauth secrets will be mounted from.
+# The volume needs to contain two files named `client_id` and `client_secret`.
+# If unset the volume will reference the Secret named operator-oauth.
+# This block will override the oauth block.
+oauthSecretVolume: {}
+  # csi:
+  #   driver: secrets-store.csi.k8s.io
+  #   readOnly: true
+  #   volumeAttributes:
+  #     secretProviderClass: tailscale-oauth
+  #
+  ## NAME is pre-defined!
+
 # installCRDs determines whether tailscale.com CRDs should be installed as part
 # of chart installation. We do not use Helm's CRD installation mechanism as that
 # does not allow for upgrading CRDs.
@@ -40,6 +55,9 @@ operatorConfig:
   podAnnotations: {}
   podLabels: {}
 
+  serviceAccountAnnotations: {}
+  # eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/tailscale-operator-role
+
   tolerations: []
 
   affinity: {}
@@ -54,6 +72,9 @@ operatorConfig:
   # - name: EXTRA_VAR2
   #   value: "value2"
 
+# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
+ingressClass:
+  enabled: true
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -24,6 +24,10 @@ spec:
           jsonPath: .status.isExitNode
           name: IsExitNode
           type: string
+        - description: Whether this Connector instance is an app connector.
+          jsonPath: .status.isAppConnector
+          name: IsAppConnector
+          type: string
         - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
@@ -66,10 +70,40 @@ spec:
                 https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
+                appConnector:
+                  description: |-
+                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                    Connector does not act as an app connector.
+                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                    Admin panel.
+                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                    tested or optimised for.
+                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                    device with a static IP address.
+                    https://tailscale.com/kb/1281/app-connectors
+                  type: object
+                  properties:
+                    routes:
+                      description: |-
+                        Routes are optional preconfigured routes for the domains routed via the app connector.
+                        If not set, routes for the domains will be discovered dynamically.
+                        If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                        also dynamically discover other routes.
+                        https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                      type: array
+                      minItems: 1
+                      items:
+                        type: string
+                        format: cidr
                 exitNode:
                   description: |-
-                    ExitNode defines whether the Connector node should act as a
-                    Tailscale exit node. Defaults to false.
+                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                    This field is mutually exclusive with the appConnector field.
                     https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
@@ -90,9 +124,11 @@ spec:
                   type: string
                 subnetRouter:
                   description: |-
-                    SubnetRouter defines subnet routes that the Connector node should
-                    expose to tailnet. If unset, none are exposed.
+                    SubnetRouter defines subnet routes that the Connector device should
+                    expose to tailnet as a Tailscale subnet router.
                     https://tailscale.com/kb/1019/subnets/
+                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                    This field is mutually exclusive with the appConnector field.
                   type: object
                   required:
                     - advertiseRoutes
@@ -125,8 +161,10 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
               x-kubernetes-validations:
-                - rule: has(self.subnetRouter) || self.exitNode == true
-                  message: A Connector needs to be either an exit node or a subnet router, or both.
+                - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                  message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                - rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
+                  message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
             status:
               description: |-
                 ConnectorStatus describes the status of the Connector. This is set
@@ -200,6 +238,9 @@ spec:
                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                     node.
                   type: string
+                isAppConnector:
+                  description: IsAppConnector is set to true if the Connector acts as an app connector.
+                  type: boolean
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -73,9 +73,35 @@ spec:
                     enable:
                       description: |-
                         Setting enable to true will make the proxy serve Tailscale metrics
-                        at <pod-ip>:9001/debug/metrics.
+                        at <pod-ip>:9002/metrics.
+                        A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
+                        serve the metrics at <service-ip>:9002/metrics.
+
+                        In 1.78.x and 1.80.x, this field also serves as the default value for
+                        .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                        fields will independently default to false.
+
                         Defaults to false.
                       type: boolean
+                    serviceMonitor:
+                      description: |-
+                        Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
+                        The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
+                        The ingested metrics for each Service monitor will have labels to identify the proxy:
+                        ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
+                        ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
+                        ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
+                        job: ts_<proxy type>_[<parent namespace>]_<parent_name>
+                      type: object
+                      required:
+                        - enable
+                      properties:
+                        enable:
+                          description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
+                          type: boolean
+                  x-kubernetes-validations:
+                    - rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable  && !self.enable)'
+                      message: ServiceMonitor can only be enabled if metrics are enabled
                 statefulSet:
                   description: |-
                     Configuration parameters for the proxy's StatefulSet. Tailscale
@@ -1249,6 +1275,25 @@ spec:
                           description: Configuration for the proxy container running tailscale.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1360,11 +1405,12 @@ spec:
                             securityContext:
                               description: |-
                                 Container security context.
-                                Security context specified here will override the security context by the operator.
-                                By default the operator:
-                                - sets 'privileged: true' for the init container
-                                - set NET_ADMIN capability for tailscale container for proxies that
-                                are created for Services or Connector.
+                                Security context specified here will override the security context set by the operator.
+                                By default the operator sets the Tailscale container and the Tailscale init container to privileged
+                                for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
+                                You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
+                                installing device plugin in your cluster and configuring the proxies tun device to be created
+                                by the device plugin, see  https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
                                 https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
                               type: object
                               properties:
@@ -1553,6 +1599,25 @@ spec:
                           description: Configuration for the proxy init container that enables forwarding.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1664,11 +1729,12 @@ spec:
                             securityContext:
                               description: |-
                                 Container security context.
-                                Security context specified here will override the security context by the operator.
-                                By default the operator:
-                                - sets 'privileged: true' for the init container
-                                - set NET_ADMIN capability for tailscale container for proxies that
-                                are created for Services or Connector.
+                                Security context specified here will override the security context set by the operator.
+                                By default the operator sets the Tailscale container and the Tailscale init container to privileged
+                                for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
+                                You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
+                                installing device plugin in your cluster and configuring the proxies tun device to be created
+                                by the device plugin, see  https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
                                 https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
                               type: object
                               properties:
@@ -1896,6 +1962,182 @@ spec:
                                   Value is the taint value the toleration matches to.
                                   If the operator is Exists, the value should be empty, otherwise just a regular string.
                                 type: string
+                        topologySpreadConstraints:
+                          description: |-
+                            Proxy Pod's topology spread constraints.
+                            By default Tailscale Kubernetes operator does not apply any topology spread constraints.
+                            https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
+                          type: array
+                          items:
+                            description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                            type: object
+                            required:
+                              - maxSkew
+                              - topologyKey
+                              - whenUnsatisfiable
+                            properties:
+                              labelSelector:
+                                description: |-
+                                  LabelSelector is used to find matching pods.
+                                  Pods that match this label selector are counted to determine the number of pods
+                                  in their corresponding topology domain.
+                                type: object
+                                properties:
+                                  matchExpressions:
+                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                    type: array
+                                    items:
+                                      description: |-
+                                        A label selector requirement is a selector that contains values, a key, and an operator that
+                                        relates the key and values.
+                                      type: object
+                                      required:
+                                        - key
+                                        - operator
+                                      properties:
+                                        key:
+                                          description: key is the label key that the selector applies to.
+                                          type: string
+                                        operator:
+                                          description: |-
+                                            operator represents a key's relationship to a set of values.
+                                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                                          type: string
+                                        values:
+                                          description: |-
+                                            values is an array of string values. If the operator is In or NotIn,
+                                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                            the values array must be empty. This array is replaced during a strategic
+                                            merge patch.
+                                          type: array
+                                          items:
+                                            type: string
+                                          x-kubernetes-list-type: atomic
+                                    x-kubernetes-list-type: atomic
+                                  matchLabels:
+                                    description: |-
+                                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                    type: object
+                                    additionalProperties:
+                                      type: string
+                                x-kubernetes-map-type: atomic
+                              matchLabelKeys:
+                                description: |-
+                                  MatchLabelKeys is a set of pod label keys to select the pods over which
+                                  spreading will be calculated. The keys are used to lookup values from the
+                                  incoming pod labels, those key-value labels are ANDed with labelSelector
+                                  to select the group of existing pods over which spreading will be calculated
+                                  for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
+                                  MatchLabelKeys cannot be set when LabelSelector isn't set.
+                                  Keys that don't exist in the incoming pod labels will
+                                  be ignored. A null or empty list means only match against labelSelector.
+
+                                  This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+                                type: array
+                                items:
+                                  type: string
+                                x-kubernetes-list-type: atomic
+                              maxSkew:
+                                description: |-
+                                  MaxSkew describes the degree to which pods may be unevenly distributed.
+                                  When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
+                                  between the number of matching pods in the target topology and the global minimum.
+                                  The global minimum is the minimum number of matching pods in an eligible domain
+                                  or zero if the number of eligible domains is less than MinDomains.
+                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                  labelSelector spread as 2/2/1:
+                                  In this case, the global minimum is 1.
+                                  | zone1 | zone2 | zone3 |
+                                  |  P P  |  P P  |   P   |
+                                  - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
+                                  scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
+                                  violate MaxSkew(1).
+                                  - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
+                                  When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
+                                  to topologies that satisfy it.
+                                  It's a required field. Default value is 1 and 0 is not allowed.
+                                type: integer
+                                format: int32
+                              minDomains:
+                                description: |-
+                                  MinDomains indicates a minimum number of eligible domains.
+                                  When the number of eligible domains with matching topology keys is less than minDomains,
+                                  Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
+                                  And when the number of eligible domains with matching topology keys equals or greater than minDomains,
+                                  this value has no effect on scheduling.
+                                  As a result, when the number of eligible domains is less than minDomains,
+                                  scheduler won't schedule more than maxSkew Pods to those domains.
+                                  If value is nil, the constraint behaves as if MinDomains is equal to 1.
+                                  Valid values are integers greater than 0.
+                                  When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
+
+                                  For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
+                                  labelSelector spread as 2/2/2:
+                                  | zone1 | zone2 | zone3 |
+                                  |  P P  |  P P  |  P P  |
+                                  The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
+                                  In this situation, new pod with the same labelSelector cannot be scheduled,
+                                  because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
+                                  it will violate MaxSkew.
+                                type: integer
+                                format: int32
+                              nodeAffinityPolicy:
+                                description: |-
+                                  NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
+                                  when calculating pod topology spread skew. Options are:
+                                  - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
+                                  - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
+
+                                  If this value is nil, the behavior is equivalent to the Honor policy.
+                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                type: string
+                              nodeTaintsPolicy:
+                                description: |-
+                                  NodeTaintsPolicy indicates how we will treat node taints when calculating
+                                  pod topology spread skew. Options are:
+                                  - Honor: nodes without taints, along with tainted nodes for which the incoming pod
+                                  has a toleration, are included.
+                                  - Ignore: node taints are ignored. All nodes are included.
+
+                                  If this value is nil, the behavior is equivalent to the Ignore policy.
+                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                type: string
+                              topologyKey:
+                                description: |-
+                                  TopologyKey is the key of node labels. Nodes that have a label with this key
+                                  and identical values are considered to be in the same topology.
+                                  We consider each <key, value> as a "bucket", and try to put balanced number
+                                  of pods into each bucket.
+                                  We define a domain as a particular instance of a topology.
+                                  Also, we define an eligible domain as a domain whose nodes meet the requirements of
+                                  nodeAffinityPolicy and nodeTaintsPolicy.
+                                  e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
+                                  And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
+                                  It's a required field.
+                                type: string
+                              whenUnsatisfiable:
+                                description: |-
+                                  WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
+                                  the spread constraint.
+                                  - DoNotSchedule (default) tells the scheduler not to schedule it.
+                                  - ScheduleAnyway tells the scheduler to schedule the pod in any location,
+                                    but giving higher precedence to topologies that would help reduce the
+                                    skew.
+                                  A constraint is considered "Unsatisfiable" for an incoming pod
+                                  if and only if every possible node assignment for that pod would violate
+                                  "MaxSkew" on some topology.
+                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                  labelSelector spread as 3/1/1:
+                                  | zone1 | zone2 | zone3 |
+                                  | P P P |   P   |   P   |
+                                  If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
+                                  to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
+                                  MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
+                                  won't make it *more* imbalanced.
+                                  It's a required field.
+                                type: string
                 tailscale:
                   description: |-
                     TailscaleConfig contains options to configure the tailscale-specific

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -53,6 +53,10 @@ spec:
               jsonPath: .status.isExitNode
               name: IsExitNode
               type: string
+            - description: Whether this Connector instance is an app connector.
+              jsonPath: .status.isAppConnector
+              name: IsAppConnector
+              type: string
             - description: Status of the deployed Connector resources.
               jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
               name: Status
@@ -91,10 +95,40 @@ spec:
                             More info:
                             https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
+                            appConnector:
+                                description: |-
+                                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                                    Connector does not act as an app connector.
+                                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                                    Admin panel.
+                                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                                    tested or optimised for.
+                                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                                    device with a static IP address.
+                                    https://tailscale.com/kb/1281/app-connectors
+                                properties:
+                                    routes:
+                                        description: |-
+                                            Routes are optional preconfigured routes for the domains routed via the app connector.
+                                            If not set, routes for the domains will be discovered dynamically.
+                                            If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                                            also dynamically discover other routes.
+                                            https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                                        items:
+                                            format: cidr
+                                            type: string
+                                        minItems: 1
+                                        type: array
+                                type: object
                             exitNode:
                                 description: |-
-                                    ExitNode defines whether the Connector node should act as a
-                                    Tailscale exit node. Defaults to false.
+                                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                                    This field is mutually exclusive with the appConnector field.
                                     https://tailscale.com/kb/1103/exit-nodes
                                 type: boolean
                             hostname:
@@ -115,9 +149,11 @@ spec:
                                 type: string
                             subnetRouter:
                                 description: |-
-                                    SubnetRouter defines subnet routes that the Connector node should
-                                    expose to tailnet. If unset, none are exposed.
+                                    SubnetRouter defines subnet routes that the Connector device should
+                                    expose to tailnet as a Tailscale subnet router.
                                     https://tailscale.com/kb/1019/subnets/
+                                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                                    This field is mutually exclusive with the appConnector field.
                                 properties:
                                     advertiseRoutes:
                                         description: |-
@@ -151,8 +187,10 @@ spec:
                                 type: array
                         type: object
                         x-kubernetes-validations:
-                            - message: A Connector needs to be either an exit node or a subnet router, or both.
-                              rule: has(self.subnetRouter) || self.exitNode == true
+                            - message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                              rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                            - message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
+                              rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
                     status:
                         description: |-
                             ConnectorStatus describes the status of the Connector. This is set
@@ -225,6 +263,9 @@ spec:
                                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                                     node.
                                 type: string
+                            isAppConnector:
+                                description: IsAppConnector is set to true if the Connector acts as an app connector.
+                                type: boolean
                             isExitNode:
                                 description: IsExitNode is set to true if the Connector acts as an exit node.
                                 type: boolean
@@ -499,12 +540,38 @@ spec:
                                     enable:
                                         description: |-
                                             Setting enable to true will make the proxy serve Tailscale metrics
-                                            at <pod-ip>:9001/debug/metrics.
+                                            at <pod-ip>:9002/metrics.
+                                            A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
+                                            serve the metrics at <service-ip>:9002/metrics.
+
+                                            In 1.78.x and 1.80.x, this field also serves as the default value for
+                                            .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                                            fields will independently default to false.
+
                                             Defaults to false.
                                         type: boolean
+                                    serviceMonitor:
+                                        description: |-
+                                            Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
+                                            The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
+                                            The ingested metrics for each Service monitor will have labels to identify the proxy:
+                                            ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
+                                            ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
+                                            ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
+                                            job: ts_<proxy type>_[<parent namespace>]_<parent_name>
+                                        properties:
+                                            enable:
+                                                description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
+                                                type: boolean
+                                        required:
+                                            - enable
+                                        type: object
                                 required:
                                     - enable
                                 type: object
+                                x-kubernetes-validations:
+                                    - message: ServiceMonitor can only be enabled if metrics are enabled
+                                      rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable  && !self.enable)'
                             statefulSet:
                                 description: |-
                                     Configuration parameters for the proxy's StatefulSet. Tailscale
@@ -1675,6 +1742,25 @@ spec:
                                             tailscaleContainer:
                                                 description: Configuration for the proxy container running tailscale.
                                                 properties:
+                                                    debug:
+                                                        description: |-
+                                                            Configuration for enabling extra debug information in the container.
+                                                            Not recommended for production use.
+                                                        properties:
+                                                            enable:
+                                                                description: |-
+                                                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                                                    9001 is a container port named "debug". The endpoints and their responses
+                                                                    may change in backwards incompatible ways in the future, and should not
+                                                                    be considered stable.
+
+                                                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                                                    this setting will default to false, and no requests will be proxied.
+                                                                type: boolean
+                                                        type: object
                                                     env:
                                                         description: |-
                                                             List of environment variables to set in the container.
@@ -1786,11 +1872,12 @@ spec:
                                                     securityContext:
                                                         description: |-
                                                             Container security context.
-                                                            Security context specified here will override the security context by the operator.
-                                                            By default the operator:
-                                                            - sets 'privileged: true' for the init container
-                                                            - set NET_ADMIN capability for tailscale container for proxies that
-                                                            are created for Services or Connector.
+                                                            Security context specified here will override the security context set by the operator.
+                                                            By default the operator sets the Tailscale container and the Tailscale init container to privileged
+                                                            for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
+                                                            You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
+                                                            installing device plugin in your cluster and configuring the proxies tun device to be created
+                                                            by the device plugin, see  https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
                                                             https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
                                                         properties:
                                                             allowPrivilegeEscalation:
@@ -1979,6 +2066,25 @@ spec:
                                             tailscaleInitContainer:
                                                 description: Configuration for the proxy init container that enables forwarding.
                                                 properties:
+                                                    debug:
+                                                        description: |-
+                                                            Configuration for enabling extra debug information in the container.
+                                                            Not recommended for production use.
+                                                        properties:
+                                                            enable:
+                                                                description: |-
+                                                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                                                    9001 is a container port named "debug". The endpoints and their responses
+                                                                    may change in backwards incompatible ways in the future, and should not
+                                                                    be considered stable.
+
+                                                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                                                    this setting will default to false, and no requests will be proxied.
+                                                                type: boolean
+                                                        type: object
                                                     env:
                                                         description: |-
                                                             List of environment variables to set in the container.
@@ -2090,11 +2196,12 @@ spec:
                                                     securityContext:
                                                         description: |-
                                                             Container security context.
-                                                            Security context specified here will override the security context by the operator.
-                                                            By default the operator:
-                                                            - sets 'privileged: true' for the init container
-                                                            - set NET_ADMIN capability for tailscale container for proxies that
-                                                            are created for Services or Connector.
+                                                            Security context specified here will override the security context set by the operator.
+                                                            By default the operator sets the Tailscale container and the Tailscale init container to privileged
+                                                            for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
+                                                            You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
+                                                            installing device plugin in your cluster and configuring the proxies tun device to be created
+                                                            by the device plugin, see  https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
                                                             https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
                                                         properties:
                                                             allowPrivilegeEscalation:
@@ -2323,6 +2430,182 @@ spec:
                                                             type: string
                                                     type: object
                                                 type: array
+                                            topologySpreadConstraints:
+                                                description: |-
+                                                    Proxy Pod's topology spread constraints.
+                                                    By default Tailscale Kubernetes operator does not apply any topology spread constraints.
+                                                    https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
+                                                items:
+                                                    description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                                                    properties:
+                                                        labelSelector:
+                                                            description: |-
+                                                                LabelSelector is used to find matching pods.
+                                                                Pods that match this label selector are counted to determine the number of pods
+                                                                in their corresponding topology domain.
+                                                            properties:
+                                                                matchExpressions:
+                                                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                                                    items:
+                                                                        description: |-
+                                                                            A label selector requirement is a selector that contains values, a key, and an operator that
+                                                                            relates the key and values.
+                                                                        properties:
+                                                                            key:
+                                                                                description: key is the label key that the selector applies to.
+                                                                                type: string
+                                                                            operator:
+                                                                                description: |-
+                                                                                    operator represents a key's relationship to a set of values.
+                                                                                    Valid operators are In, NotIn, Exists and DoesNotExist.
+                                                                                type: string
+                                                                            values:
+                                                                                description: |-
+                                                                                    values is an array of string values. If the operator is In or NotIn,
+                                                                                    the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                                                                    the values array must be empty. This array is replaced during a strategic
+                                                                                    merge patch.
+                                                                                items:
+                                                                                    type: string
+                                                                                type: array
+                                                                                x-kubernetes-list-type: atomic
+                                                                        required:
+                                                                            - key
+                                                                            - operator
+                                                                        type: object
+                                                                    type: array
+                                                                    x-kubernetes-list-type: atomic
+                                                                matchLabels:
+                                                                    additionalProperties:
+                                                                        type: string
+                                                                    description: |-
+                                                                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                                                                        map is equivalent to an element of matchExpressions, whose key field is "key", the
+                                                                        operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                                                    type: object
+                                                            type: object
+                                                            x-kubernetes-map-type: atomic
+                                                        matchLabelKeys:
+                                                            description: |-
+                                                                MatchLabelKeys is a set of pod label keys to select the pods over which
+                                                                spreading will be calculated. The keys are used to lookup values from the
+                                                                incoming pod labels, those key-value labels are ANDed with labelSelector
+                                                                to select the group of existing pods over which spreading will be calculated
+                                                                for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
+                                                                MatchLabelKeys cannot be set when LabelSelector isn't set.
+                                                                Keys that don't exist in the incoming pod labels will
+                                                                be ignored. A null or empty list means only match against labelSelector.
+
+                                                                This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+                                                            items:
+                                                                type: string
+                                                            type: array
+                                                            x-kubernetes-list-type: atomic
+                                                        maxSkew:
+                                                            description: |-
+                                                                MaxSkew describes the degree to which pods may be unevenly distributed.
+                                                                When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
+                                                                between the number of matching pods in the target topology and the global minimum.
+                                                                The global minimum is the minimum number of matching pods in an eligible domain
+                                                                or zero if the number of eligible domains is less than MinDomains.
+                                                                For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                                                labelSelector spread as 2/2/1:
+                                                                In this case, the global minimum is 1.
+                                                                | zone1 | zone2 | zone3 |
+                                                                |  P P  |  P P  |   P   |
+                                                                - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
+                                                                scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
+                                                                violate MaxSkew(1).
+                                                                - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
+                                                                When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
+                                                                to topologies that satisfy it.
+                                                                It's a required field. Default value is 1 and 0 is not allowed.
+                                                            format: int32
+                                                            type: integer
+                                                        minDomains:
+                                                            description: |-
+                                                                MinDomains indicates a minimum number of eligible domains.
+                                                                When the number of eligible domains with matching topology keys is less than minDomains,
+                                                                Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
+                                                                And when the number of eligible domains with matching topology keys equals or greater than minDomains,
+                                                                this value has no effect on scheduling.
+                                                                As a result, when the number of eligible domains is less than minDomains,
+                                                                scheduler won't schedule more than maxSkew Pods to those domains.
+                                                                If value is nil, the constraint behaves as if MinDomains is equal to 1.
+                                                                Valid values are integers greater than 0.
+                                                                When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
+
+                                                                For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
+                                                                labelSelector spread as 2/2/2:
+                                                                | zone1 | zone2 | zone3 |
+                                                                |  P P  |  P P  |  P P  |
+                                                                The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
+                                                                In this situation, new pod with the same labelSelector cannot be scheduled,
+                                                                because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
+                                                                it will violate MaxSkew.
+                                                            format: int32
+                                                            type: integer
+                                                        nodeAffinityPolicy:
+                                                            description: |-
+                                                                NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
+                                                                when calculating pod topology spread skew. Options are:
+                                                                - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
+                                                                - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
+
+                                                                If this value is nil, the behavior is equivalent to the Honor policy.
+                                                                This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                                            type: string
+                                                        nodeTaintsPolicy:
+                                                            description: |-
+                                                                NodeTaintsPolicy indicates how we will treat node taints when calculating
+                                                                pod topology spread skew. Options are:
+                                                                - Honor: nodes without taints, along with tainted nodes for which the incoming pod
+                                                                has a toleration, are included.
+                                                                - Ignore: node taints are ignored. All nodes are included.
+
+                                                                If this value is nil, the behavior is equivalent to the Ignore policy.
+                                                                This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                                            type: string
+                                                        topologyKey:
+                                                            description: |-
+                                                                TopologyKey is the key of node labels. Nodes that have a label with this key
+                                                                and identical values are considered to be in the same topology.
+                                                                We consider each <key, value> as a "bucket", and try to put balanced number
+                                                                of pods into each bucket.
+                                                                We define a domain as a particular instance of a topology.
+                                                                Also, we define an eligible domain as a domain whose nodes meet the requirements of
+                                                                nodeAffinityPolicy and nodeTaintsPolicy.
+                                                                e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
+                                                                And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
+                                                                It's a required field.
+                                                            type: string
+                                                        whenUnsatisfiable:
+                                                            description: |-
+                                                                WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
+                                                                the spread constraint.
+                                                                - DoNotSchedule (default) tells the scheduler not to schedule it.
+                                                                - ScheduleAnyway tells the scheduler to schedule the pod in any location,
+                                                                  but giving higher precedence to topologies that would help reduce the
+                                                                  skew.
+                                                                A constraint is considered "Unsatisfiable" for an incoming pod
+                                                                if and only if every possible node assignment for that pod would violate
+                                                                "MaxSkew" on some topology.
+                                                                For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                                                labelSelector spread as 3/1/1:
+                                                                | zone1 | zone2 | zone3 |
+                                                                | P P P |   P   |   P   |
+                                                                If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
+                                                                to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
+                                                                MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
+                                                                won't make it *more* imbalanced.
+                                                                It's a required field.
+                                                            type: string
+                                                    required:
+                                                        - maxSkew
+                                                        - topologyKey
+                                                        - whenUnsatisfiable
+                                                    type: object
+                                                type: array
                                         type: object
                                 type: object
                             tailscale:
@@ -4386,6 +4669,16 @@ rules:
         - list
         - watch
         - update
+    - apiGroups:
+        - apiextensions.k8s.io
+      resourceNames:
+        - servicemonitors.monitoring.coreos.com
+      resources:
+        - customresourcedefinitions
+      verbs:
+        - get
+        - list
+        - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -4466,6 +4759,16 @@ rules:
         - update
         - list
         - watch
+    - apiGroups:
+        - monitoring.coreos.com
+      resources:
+        - servicemonitors
+      verbs:
+        - get
+        - list
+        - update
+        - create
+        - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -4486,6 +4789,14 @@ rules:
         - patch
         - update
         - watch
+    - apiGroups:
+        - ""
+      resources:
+        - events
+      verbs:
+        - create
+        - patch
+        - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -4558,6 +4869,14 @@ spec:
                       value: "false"
                     - name: PROXY_FIREWALL_MODE
                       value: auto
+                    - name: POD_NAME
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.name
+                    - name: POD_UID
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.uid
                   image: tailscale/k8s-operator:unstable
                   imagePullPolicy: Always
                   name: operator

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -30,7 +30,13 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
           securityContext:
-            capabilities:
-              add:
-                - NET_ADMIN
+            privileged: true

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -24,3 +24,11 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid

cmd/k8s-operator/dnsrecords.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"slices"
+	"strings"
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -98,7 +99,15 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.
 		return reconcile.Result{}, nil
 	}
 
-	return reconcile.Result{}, dnsRR.maybeProvision(ctx, headlessSvc, logger)
+	if err := dnsRR.maybeProvision(ctx, headlessSvc, logger); err != nil {
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			logger.Infof("optimistic lock error, retrying: %s", err)
+		} else {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return reconcile.Result{}, nil
 }
 
 // maybeProvision ensures that dnsrecords ConfigMap contains a record for the

cmd/k8s-operator/e2e/ingress_test.go

@@ -0,0 +1,108 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	kube "tailscale.com/k8s-operator"
+	"tailscale.com/tstest"
+)
+
+// See [TestMain] for test requirements.
+func TestIngress(t *testing.T) {
+	if tsClient == nil {
+		t.Skip("TestIngress requires credentials for a tailscale client")
+	}
+
+	ctx := context.Background()
+	cfg := config.GetConfigOrDie()
+	cl, err := client.New(cfg, client.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Apply nginx
+	createAndCleanup(t, ctx, cl, &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nginx",
+			Namespace: "default",
+			Labels: map[string]string{
+				"app.kubernetes.io/name": "nginx",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "nginx",
+					Image: "nginx",
+				},
+			},
+		},
+	})
+	// Apply service to expose it as ingress
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			Annotations: map[string]string{
+				"tailscale.com/expose": "true",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"app.kubernetes.io/name": "nginx",
+			},
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+			},
+		},
+	}
+	createAndCleanup(t, ctx, cl, svc)
+
+	// TODO: instead of timing out only when test times out, cancel context after 60s or so.
+	if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) {
+		maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")}
+		if err := get(ctx, cl, maybeReadySvc); err != nil {
+			return false, err
+		}
+		isReady := kube.SvcIsReady(maybeReadySvc)
+		if isReady {
+			t.Log("Service is ready")
+		}
+		return isReady, nil
+	}); err != nil {
+		t.Fatalf("error waiting for the Service to become Ready: %v", err)
+	}
+
+	var resp *http.Response
+	if err := tstest.WaitFor(time.Second*60, func() error {
+		// TODO(tomhjp): Get the tailnet DNS name from the associated secret instead.
+		// If we are not the first tailnet node with the requested name, we'll get
+		// a -N suffix.
+		resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name))
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("error trying to reach service: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v; response body s", resp.StatusCode)
+	}
+}

cmd/k8s-operator/e2e/main_test.go

@@ -0,0 +1,194 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/go-logr/zapr"
+	"github.com/tailscale/hujson"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/oauth2/clientcredentials"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"tailscale.com/client/tailscale"
+)
+
+const (
+	e2eManagedComment = "// This is managed by the k8s-operator e2e tests"
+)
+
+var (
+	tsClient   *tailscale.Client
+	testGrants = map[string]string{
+		"test-proxy": `{
+			"src": ["tag:e2e-test-proxy"],
+			"dst": ["tag:k8s-operator"],
+			"app": {
+				"tailscale.com/cap/kubernetes": [{
+					"impersonate": {
+						"groups": ["ts:e2e-test-proxy"],
+					},
+				}],
+			},
+		}`,
+	}
+)
+
+// This test suite is currently not run in CI.
+// It requires some setup not handled by this code:
+// - Kubernetes cluster with tailscale operator installed
+// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy)
+// - Operator installed with --set apiServerProxyConfig.mode="true"
+// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key
+// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env
+// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag
+func TestMain(m *testing.M) {
+	code, err := runTests(m)
+	if err != nil {
+		log.Fatal(err)
+	}
+	os.Exit(code)
+}
+
+func runTests(m *testing.M) (int, error) {
+	zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar()
+	logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
+	tailscale.I_Acknowledge_This_API_Is_Unstable = true
+
+	if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" {
+		cleanup, err := setupClientAndACLs()
+		if err != nil {
+			return 0, err
+		}
+		defer func() {
+			err = errors.Join(err, cleanup())
+		}()
+	}
+
+	return m.Run(), nil
+}
+
+func setupClientAndACLs() (cleanup func() error, _ error) {
+	ctx := context.Background()
+	credentials := clientcredentials.Config{
+		ClientID:     os.Getenv("TS_API_CLIENT_ID"),
+		ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"),
+		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
+		Scopes:       []string{"auth_keys", "policy_file"},
+	}
+	tsClient = tailscale.NewClient("-", nil)
+	tsClient.HTTPClient = credentials.Client(ctx)
+
+	if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) {
+		for test, grant := range testGrants {
+			deleteTestGrants(test, acls)
+			addTestGrant(test, grant, acls)
+		}
+	}); err != nil {
+		return nil, err
+	}
+
+	return func() error {
+		return patchACLs(ctx, tsClient, func(acls *hujson.Value) {
+			for test := range testGrants {
+				deleteTestGrants(test, acls)
+			}
+		})
+	}, nil
+}
+
+func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error {
+	acls, err := tsClient.ACLHuJSON(ctx)
+	if err != nil {
+		return err
+	}
+	hj, err := hujson.Parse([]byte(acls.ACL))
+	if err != nil {
+		return err
+	}
+
+	patchFn(&hj)
+
+	hj.Format()
+	acls.ACL = hj.String()
+	if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func addTestGrant(test, grant string, acls *hujson.Value) error {
+	v, err := hujson.Parse([]byte(grant))
+	if err != nil {
+		return err
+	}
+
+	// Add the managed comment to the first line of the grant object contents.
+	v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test))
+
+	if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deleteTestGrants(test string, acls *hujson.Value) error {
+	grants := acls.Find("/grants")
+
+	var patches []string
+	for i, g := range grants.Value.(*hujson.Array).Elements {
+		members := g.Value.(*hujson.Object).Members
+		if len(members) == 0 {
+			continue
+		}
+		comment := strings.TrimSpace(string(members[0].Name.BeforeExtra))
+		if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test {
+			patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i))
+		}
+	}
+
+	// Remove in reverse order so we don't affect the found indices as we mutate.
+	slices.Reverse(patches)
+
+	if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func objectMeta(namespace, name string) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Namespace: namespace,
+		Name:      name,
+	}
+}
+
+func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) {
+	t.Helper()
+	if err := cl.Create(ctx, obj); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := cl.Delete(ctx, obj); err != nil {
+			t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
+		}
+	})
+}
+
+func get(ctx context.Context, cl client.Client, obj client.Object) error {
+	return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)
+}

cmd/k8s-operator/e2e/proxy_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/tsnet"
+	"tailscale.com/tstest"
+)
+
+// See [TestMain] for test requirements.
+func TestProxy(t *testing.T) {
+	if tsClient == nil {
+		t.Skip("TestProxy requires credentials for a tailscale client")
+	}
+
+	ctx := context.Background()
+	cfg := config.GetConfigOrDie()
+	cl, err := client.New(cfg, client.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create role and role binding to allow a group we'll impersonate to do stuff.
+	createAndCleanup(t, ctx, cl, &rbacv1.Role{
+		ObjectMeta: objectMeta("tailscale", "read-secrets"),
+		Rules: []rbacv1.PolicyRule{{
+			APIGroups: []string{""},
+			Verbs:     []string{"get"},
+			Resources: []string{"secrets"},
+		}},
+	})
+	createAndCleanup(t, ctx, cl, &rbacv1.RoleBinding{
+		ObjectMeta: objectMeta("tailscale", "read-secrets"),
+		Subjects: []rbacv1.Subject{{
+			Kind: "Group",
+			Name: "ts:e2e-test-proxy",
+		}},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: "read-secrets",
+		},
+	})
+
+	// Get operator host name from kube secret.
+	operatorSecret := corev1.Secret{
+		ObjectMeta: objectMeta("tailscale", "operator"),
+	}
+	if err := get(ctx, cl, &operatorSecret); err != nil {
+		t.Fatal(err)
+	}
+
+	// Connect to tailnet with test-specific tag so we can use the
+	// [testGrants] ACLs when connecting to the API server proxy
+	ts := tsnetServerWithTag(t, ctx, "tag:e2e-test-proxy")
+	proxyCfg := &rest.Config{
+		Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)),
+		Dial: ts.Dial,
+	}
+	proxyCl, err := client.New(proxyCfg, client.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Expect success.
+	allowedSecret := corev1.Secret{
+		ObjectMeta: objectMeta("tailscale", "operator"),
+	}
+	// Wait for up to a minute the first time we use the proxy, to give it time
+	// to provision the TLS certs.
+	if err := tstest.WaitFor(time.Second*60, func() error {
+		return get(ctx, proxyCl, &allowedSecret)
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Expect forbidden.
+	forbiddenSecret := corev1.Secret{
+		ObjectMeta: objectMeta("default", "operator"),
+	}
+	if err := get(ctx, proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) {
+		t.Fatalf("expected forbidden error fetching secret from default namespace: %s", err)
+	}
+}
+
+func tsnetServerWithTag(t *testing.T, ctx context.Context, tag string) *tsnet.Server {
+	caps := tailscale.KeyCapabilities{
+		Devices: tailscale.KeyDeviceCapabilities{
+			Create: tailscale.KeyDeviceCreateCapabilities{
+				Reusable:      false,
+				Preauthorized: true,
+				Ephemeral:     true,
+				Tags:          []string{tag},
+			},
+		},
+	}
+
+	authKey, authKeyMeta, err := tsClient.CreateKey(ctx, caps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := tsClient.DeleteKey(ctx, authKeyMeta.ID); err != nil {
+			t.Errorf("error deleting auth key: %s", err)
+		}
+	})
+
+	ts := &tsnet.Server{
+		Hostname:  "test-proxy",
+		Ephemeral: true,
+		Dir:       t.TempDir(),
+		AuthKey:   authKey,
+	}
+	_, err = ts.Up(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := ts.Close(); err != nil {
+			t.Errorf("error shutting down tsnet.Server: %s", err)
+		}
+	})
+
+	return ts
+}
+
+func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string {
+	profiles := map[string]any{}
+	if err := json.Unmarshal(s.Data["_profiles"], &profiles); err != nil {
+		t.Fatal(err)
+	}
+	key, ok := strings.CutPrefix(string(s.Data["_current-profile"]), "profile-")
+	if !ok {
+		t.Fatal(string(s.Data["_current-profile"]))
+	}
+	profile, ok := profiles[key]
+	if !ok {
+		t.Fatal(profiles)
+	}
+
+	return ((profile.(map[string]any))["Name"]).(string)
+}

cmd/k8s-operator/egress-services-readiness.go

@@ -64,7 +64,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
 	oldStatus := svc.Status.DeepCopy()
 	defer func() {
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
-		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
+		if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
 			err = errors.Join(err, esrr.Status().Update(ctx, svc))
 		}
 	}()

cmd/k8s-operator/egress-services.go

@@ -51,12 +51,12 @@ const (
 	labelSvcType = "tailscale.com/svc-type" // ingress or egress
 	typeEgress   = "egress"
 	// maxPorts is the maximum number of ports that can be exposed on a
-	// container. In practice this will be ports in range [3000 - 4000). The
+	// container. In practice this will be ports in range [10000 - 11000). The
 	// high range should make it easier to distinguish container ports from
 	// the tailnet target ports for debugging purposes (i.e when reading
-	// netfilter rules). The limit of 10000 is somewhat arbitrary, the
+	// netfilter rules). The limit of 1000 is somewhat arbitrary, the
 	// assumption is that this would not be hit in practice.
-	maxPorts = 10000
+	maxPorts = 1000
 
 	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
 )
@@ -123,7 +123,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
 
 	oldStatus := svc.Status.DeepCopy()
 	defer func() {
-		if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
+		if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
 			err = errors.Join(err, esr.Status().Update(ctx, svc))
 		}
 	}()
@@ -136,9 +136,8 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	}
 
 	if !slices.Contains(svc.Finalizers, FinalizerName) {
-		l.Infof("configuring tailnet service") // logged exactly once
 		svc.Finalizers = append(svc.Finalizers, FinalizerName)
-		if err := esr.Update(ctx, svc); err != nil {
+		if err := esr.updateSvcSpec(ctx, svc); err != nil {
 			err := fmt.Errorf("failed to add finalizer: %w", err)
 			r := svcConfiguredReason(svc, false, l)
 			tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
@@ -157,7 +156,15 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
 		return res, err
 	}
 
-	return res, esr.maybeProvision(ctx, svc, l)
+	if err := esr.maybeProvision(ctx, svc, l); err != nil {
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			l.Infof("optimistic lock error, retrying: %s", err)
+		} else {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return res, nil
 }
 
 func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) {
@@ -198,7 +205,7 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
 	if svc.Spec.ExternalName != clusterIPSvcFQDN {
 		l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
 		svc.Spec.ExternalName = clusterIPSvcFQDN
-		if err = esr.Update(ctx, svc); err != nil {
+		if err = esr.updateSvcSpec(ctx, svc); err != nil {
 			err = fmt.Errorf("error updating ExternalName Service: %w", err)
 			return err
 		}
@@ -222,6 +229,15 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 		found := false
 		for _, wantsPM := range svc.Spec.Ports {
 			if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
+				// We don't use the port name to distinguish this port internally, but Kubernetes
+				// require that, for Service ports with more than one name each port is uniquely named.
+				// So we can always pick the port name from the ExternalName Service as at this point we
+				// know that those are valid names because Kuberentes already validated it once. Note
+				// that users could have changed an unnamed port to a named port and might have changed
+				// port names- this should still work.
+				// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
+				// See also https://github.com/tailscale/tailscale/issues/13406#issuecomment-2507230388
+				clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
 				found = true
 				break
 			}
@@ -246,7 +262,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 		if !found {
 			// Calculate a free port to expose on container and add
 			// a new PortMap to the ClusterIP Service.
-			if usedPorts.Len() == maxPorts {
+			if usedPorts.Len() >= maxPorts {
 				// TODO(irbekrm): refactor to avoid extra reconciles here. Low priority as in practice,
 				// the limit should not be hit.
 				return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts)
@@ -540,13 +556,13 @@ func svcNameBase(s string) string {
 	}
 }
 
-// unusedPort returns a port in range [3000 - 4000). The caller must ensure that
-// usedPorts does not contain all ports in range [3000 - 4000).
+// unusedPort returns a port in range [10000 - 11000). The caller must ensure that
+// usedPorts does not contain all ports in range [10000 - 11000).
 func unusedPort(usedPorts sets.Set[int32]) int32 {
 	foundFreePort := false
 	var suggestPort int32
 	for !foundFreePort {
-		suggestPort = rand.Int32N(maxPorts) + 3000
+		suggestPort = rand.Int32N(maxPorts) + 10000
 		if !usedPorts.Has(suggestPort) {
 			foundFreePort = true
 		}
@@ -714,3 +730,13 @@ func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
 	}
 	return ep
 }
+
+// updateSvcSpec ensures that the given Service's spec is updated in cluster, but the local Service object still retains
+// the not-yet-applied status.
+// TODO(irbekrm): once we do SSA for these patch updates, this will no longer be needed.
+func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error {
+	st := svc.Status.DeepCopy()
+	err := esr.Update(ctx, svc)
+	svc.Status = *st
+	return err
+}

cmd/k8s-operator/egress-services_test.go

@@ -105,28 +105,40 @@ func TestTailscaleEgressServices(t *testing.T) {
 				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
 			}
 		})
-		// Quirks of the fake client.
-		mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
-			svc.Status.Conditions = []metav1.Condition{}
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_retain_one_unnamed_port", func(t *testing.T) {
+		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80}}
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
 		})
 		expectReconciled(t, esr, "default", "test")
-		// Verify that a ClusterIP Service has been created.
-		name := findGenNameForEgressSvcResources(t, fc, svc)
-		expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
-		clusterSvc := mustGetClusterIPSvc(t, fc, name)
-		// Verify that an EndpointSlice has been created.
-		expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
-		// Verify that ConfigMap contains configuration for the new egress service.
-		mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
-		r := svcConfiguredReason(svc, true, zl.Sugar())
-		// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
-		// CluterIP Service.
-		svc.Status.Conditions = []metav1.Condition{
-			condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
-		}
-		svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
-		svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
-		expectEqual(t, fc, svc, nil)
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_add_two_named_ports", func(t *testing.T) {
+		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}}
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_add_udp_port", func(t *testing.T) {
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Port: 53, Protocol: "UDP", Name: "dns"})
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+	t.Run("service_change_protocol", func(t *testing.T) {
+		svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}, {Port: 53, Protocol: "TCP", Name: "tcp_dns"}}
+		mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		expectReconciled(t, esr, "default", "test")
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
 	})
 
 	t.Run("delete_external_name_service", func(t *testing.T) {
@@ -143,6 +155,29 @@ func TestTailscaleEgressServices(t *testing.T) {
 	})
 }
 
+func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) {
+	expectReconciled(t, esr, "default", "test")
+	// Verify that a ClusterIP Service has been created.
+	name := findGenNameForEgressSvcResources(t, fc, svc)
+	expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
+	clusterSvc := mustGetClusterIPSvc(t, fc, name)
+	// Verify that an EndpointSlice has been created.
+	expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
+	// Verify that ConfigMap contains configuration for the new egress service.
+	mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
+	r := svcConfiguredReason(svc, true, zl.Sugar())
+	// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
+	// CluterIP Service.
+	svc.Status.Conditions = []metav1.Condition{
+		condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "EgressSvcValid", "EgressSvcValid", clock),
+		condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
+	}
+	svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
+	svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
+	expectEqual(t, fc, svc, nil)
+
+}
+
 func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
 	return metav1.Condition{
 		Type:               string(typ),

cmd/k8s-operator/ingress.go

@@ -76,7 +76,15 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, ing)
 	}
 
-	return reconcile.Result{}, a.maybeProvision(ctx, logger, ing)
+	if err := a.maybeProvision(ctx, logger, ing); err != nil {
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			logger.Infof("optimistic lock error, retrying: %s", err)
+		} else {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return reconcile.Result{}, nil
 }
 
 func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
@@ -90,7 +98,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 		return nil
 	}
 
-	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress")); err != nil {
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil {
 		return fmt.Errorf("failed to cleanup: %w", err)
 	} else if !done {
 		logger.Debugf("cleanup not done yet, waiting for next reconcile")
@@ -268,6 +276,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		Tags:                tags,
 		ChildResourceLabels: crl,
 		ProxyClassName:      proxyClass,
+		proxyType:           proxyTypeIngressResource,
 	}
 
 	if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {
@@ -278,12 +287,12 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return fmt.Errorf("failed to provision: %w", err)
 	}
 
-	_, tsHost, _, err := a.ssr.DeviceInfo(ctx, crl)
+	dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
 	if err != nil {
-		return fmt.Errorf("failed to get device ID: %w", err)
+		return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err)
 	}
-	if tsHost == "" {
-		logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
+	if dev == nil || dev.ingressDNSName == "" {
+		logger.Debugf("no Ingress DNS name known yet, waiting for proxy Pod initialize and start serving Ingress")
 		// No hostname yet. Wait for the proxy pod to auth.
 		ing.Status.LoadBalancer.Ingress = nil
 		if err := a.Status().Update(ctx, ing); err != nil {
@@ -292,10 +301,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	logger.Debugf("setting ingress hostname to %q", tsHost)
+	logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
 	ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
 		{
-			Hostname: tsHost,
+			Hostname: dev.ingressDNSName,
 			Ports: []networkingv1.IngressPortStatus{
 				{
 					Protocol: "TCP",

cmd/k8s-operator/ingress_test.go

@@ -12,6 +12,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -141,6 +142,154 @@ func TestTailscaleIngress(t *testing.T) {
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 }
 
+func TestTailscaleIngressHostname(t *testing.T) {
+	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+	fc := fake.NewFakeClient(tsIngressClass)
+	ft := &fakeTSClient{}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ingR := &IngressReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			tsnetServer:       fakeTsnetServer,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
+
+	// 1. Resources get created for regular Ingress
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"default-test"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "1.2.3.4",
+			Ports: []corev1.ServicePort{{
+				Port: 8080,
+				Name: "http"},
+			},
+		},
+	})
+
+	expectReconciled(t, ingR, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
+	mustCreate(t, fc, &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fullName,
+			Namespace: "operator-ns",
+			UID:       "test-uid",
+		},
+	})
+	opts := configOpts{
+		stsName:    shortName,
+		secretName: fullName,
+		namespace:  "default",
+		parentType: "ingress",
+		hostname:   "default-test",
+		app:        kubetypes.AppIngressResource,
+	}
+	serveConfig := &ipn.ServeConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
+	}
+	opts.serveConfig = serveConfig
+
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+
+	// 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
+		mak.Set(&secret.Data, "pod_uid", []byte("test-uid"))
+		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
+	})
+	expectReconciled(t, ingR, "default", "test")
+	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
+
+	expectEqual(t, fc, ing, nil)
+
+	// 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
+		mak.Set(&secret.Data, "pod_uid", []byte("test-uid"))
+		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
+		mak.Set(&secret.Data, "https_endpoint", []byte("foo.tailnetxyz.ts.net"))
+	})
+	expectReconciled(t, ingR, "default", "test")
+	ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
+		Ingress: []networkingv1.IngressLoadBalancerIngress{
+			{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
+		},
+	}
+	expectEqual(t, fc, ing, nil)
+
+	// 4. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint ready
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
+		mak.Set(&secret.Data, "pod_uid", []byte("test-uid"))
+		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
+		mak.Set(&secret.Data, "https_endpoint", []byte("no-https"))
+	})
+	expectReconciled(t, ingR, "default", "test")
+	ing.Status.LoadBalancer.Ingress = nil
+	expectEqual(t, fc, ing, nil)
+
+	// 5. Ingress proxy's state has https_endpoints set, but its capver is not matching Pod UID (downgrade)
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
+		mak.Set(&secret.Data, "pod_uid", []byte("not-the-right-uid"))
+		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
+		mak.Set(&secret.Data, "https_endpoint", []byte("bar.tailnetxyz.ts.net"))
+	})
+	ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
+		Ingress: []networkingv1.IngressLoadBalancerIngress{
+			{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
+		},
+	}
+	expectReconciled(t, ingR, "default", "test")
+	expectEqual(t, fc, ing, nil)
+}
+
 func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	// Setup
 	pc := &tsapi.ProxyClass{
@@ -271,3 +420,124 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	opts.proxyClass = ""
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
 }
+
+func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
+	pc := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
+		Spec: tsapi.ProxyClassSpec{
+			Metrics: &tsapi.Metrics{
+				Enable:         true,
+				ServiceMonitor: &tsapi.ServiceMonitor{Enable: true},
+			},
+		},
+		Status: tsapi.ProxyClassStatus{
+			Conditions: []metav1.Condition{{
+				Status:             metav1.ConditionTrue,
+				Type:               string(tsapi.ProxyClassReady),
+				ObservedGeneration: 1,
+			}}},
+	}
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pc, tsIngressClass).
+		WithStatusSubresource(pc).
+		Build()
+	ft := &fakeTSClient{}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ingR := &IngressReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			tsnetServer:       fakeTsnetServer,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
+	// 1. Enable metrics- expect metrics Service to be created
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Labels: map[string]string{
+				"tailscale.com/proxy-class": "metrics",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"default-test"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "1.2.3.4",
+			Ports: []corev1.ServicePort{{
+				Port: 8080,
+				Name: "http"},
+			},
+		},
+	})
+
+	expectReconciled(t, ingR, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
+	opts := configOpts{
+		stsName:            shortName,
+		secretName:         fullName,
+		namespace:          "default",
+		tailscaleNamespace: "operator-ns",
+		parentType:         "ingress",
+		hostname:           "default-test",
+		app:                kubetypes.AppIngressResource,
+		enableMetrics:      true,
+		namespaced:         true,
+		proxyType:          proxyTypeIngressResource,
+	}
+	serveConfig := &ipn.ServeConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
+	}
+	opts.serveConfig = serveConfig
+
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
+	expectEqual(t, fc, expectedMetricsService(opts), nil)
+	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+	// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
+	})
+	expectReconciled(t, ingR, "default", "test")
+	// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
+	mustCreate(t, fc, crd)
+	expectReconciled(t, ingR, "default", "test")
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+}

cmd/k8s-operator/metrics_resources.go

@@ -0,0 +1,272 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+)
+
+const (
+	labelMetricsTarget = "tailscale.com/metrics-target"
+
+	// These labels get transferred from the metrics Service to the ingested Prometheus metrics.
+	labelPromProxyType            = "ts_proxy_type"
+	labelPromProxyParentName      = "ts_proxy_parent_name"
+	labelPromProxyParentNamespace = "ts_proxy_parent_namespace"
+	labelPromJob                  = "ts_prom_job"
+
+	serviceMonitorCRD = "servicemonitors.monitoring.coreos.com"
+)
+
+// ServiceMonitor contains a subset of fields of servicemonitors.monitoring.coreos.com Custom Resource Definition.
+// Duplicating it here allows us to avoid importing prometheus-operator library.
+// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L40
+type ServiceMonitor struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata"`
+	Spec              ServiceMonitorSpec `json:"spec"`
+}
+
+// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L55
+type ServiceMonitorSpec struct {
+	// Endpoints defines the endpoints to be scraped on the selected Service(s).
+	// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L82
+	Endpoints []ServiceMonitorEndpoint `json:"endpoints"`
+	// JobLabel is the label on the Service whose value will become the value of the Prometheus job label for the metrics ingested via this ServiceMonitor.
+	// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L66
+	JobLabel string `json:"jobLabel"`
+	// NamespaceSelector selects the namespace of Service(s) that this ServiceMonitor allows to scrape.
+	// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L88
+	NamespaceSelector ServiceMonitorNamespaceSelector `json:"namespaceSelector,omitempty"`
+	// Selector is the label selector for Service(s) that this ServiceMonitor allows to scrape.
+	// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L85
+	Selector metav1.LabelSelector `json:"selector"`
+	// TargetLabels are labels on the selected Service that should be applied as Prometheus labels to the ingested metrics.
+	// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L72
+	TargetLabels []string `json:"targetLabels"`
+}
+
+// ServiceMonitorNamespaceSelector selects namespaces in which Prometheus operator will attempt to find Services for
+// this ServiceMonitor.
+// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L88
+type ServiceMonitorNamespaceSelector struct {
+	MatchNames []string `json:"matchNames,omitempty"`
+}
+
+// ServiceMonitorEndpoint defines an endpoint of Service to scrape. We only define port here. Prometheus by default
+// scrapes /metrics path, which is what we want.
+type ServiceMonitorEndpoint struct {
+	// Port is the name of the Service port that Prometheus will scrape.
+	Port string `json:"port,omitempty"`
+}
+
+func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, opts *metricsOpts, pc *tsapi.ProxyClass, cl client.Client) error {
+	if opts.proxyType == proxyTypeEgress {
+		// Metrics are currently not being enabled for standalone egress proxies.
+		return nil
+	}
+	if pc == nil || pc.Spec.Metrics == nil || !pc.Spec.Metrics.Enable {
+		return maybeCleanupMetricsResources(ctx, opts, cl)
+	}
+	metricsSvc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      metricsResourceName(opts.proxyStsName),
+			Namespace: opts.tsNamespace,
+			Labels:    metricsResourceLabels(opts),
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: opts.proxyLabels,
+			Type:     corev1.ServiceTypeClusterIP,
+			Ports:    []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}},
+		},
+	}
+	var err error
+	metricsSvc, err = createOrUpdate(ctx, cl, opts.tsNamespace, metricsSvc, func(svc *corev1.Service) {
+		svc.Spec.Ports = metricsSvc.Spec.Ports
+		svc.Spec.Selector = metricsSvc.Spec.Selector
+	})
+	if err != nil {
+		return fmt.Errorf("error ensuring metrics Service: %w", err)
+	}
+
+	crdExists, err := hasServiceMonitorCRD(ctx, cl)
+	if err != nil {
+		return fmt.Errorf("error verifying that %q CRD exists: %w", serviceMonitorCRD, err)
+	}
+	if !crdExists {
+		return nil
+	}
+
+	if pc.Spec.Metrics.ServiceMonitor == nil || !pc.Spec.Metrics.ServiceMonitor.Enable {
+		return maybeCleanupServiceMonitor(ctx, cl, opts.proxyStsName, opts.tsNamespace)
+	}
+
+	logger.Info("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
+	svcMonitor, err := newServiceMonitor(metricsSvc)
+	if err != nil {
+		return fmt.Errorf("error creating ServiceMonitor: %w", err)
+	}
+	// We don't use createOrUpdate here because that does not work with unstructured types. We also do not update
+	// the ServiceMonitor because it is not expected that any of its fields would change. Currently this is good
+	// enough, but in future we might want to add logic to create-or-update unstructured types.
+	err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), svcMonitor.DeepCopy())
+	if apierrors.IsNotFound(err) {
+		if err := cl.Create(ctx, svcMonitor); err != nil {
+			return fmt.Errorf("error creating ServiceMonitor: %w", err)
+		}
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("error getting ServiceMonitor: %w", err)
+	}
+	return nil
+}
+
+// maybeCleanupMetricsResources ensures that any metrics resources created for a proxy are deleted. Only metrics Service
+// gets deleted explicitly because the ServiceMonitor has Service's owner reference, so gets garbage collected
+// automatically.
+func maybeCleanupMetricsResources(ctx context.Context, opts *metricsOpts, cl client.Client) error {
+	sel := metricsSvcSelector(opts.proxyLabels, opts.proxyType)
+	return cl.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(opts.tsNamespace), client.MatchingLabels(sel))
+}
+
+// maybeCleanupServiceMonitor cleans up any ServiceMonitor created for the named proxy StatefulSet.
+func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName, ns string) error {
+	smName := metricsResourceName(stsName)
+	sm := serviceMonitorTemplate(smName, ns)
+	u, err := serviceMonitorToUnstructured(sm)
+	if err != nil {
+		return fmt.Errorf("error building ServiceMonitor: %w", err)
+	}
+	err = cl.Get(ctx, types.NamespacedName{Name: smName, Namespace: ns}, u)
+	if apierrors.IsNotFound(err) {
+		return nil // nothing to do
+	}
+	if err != nil {
+		return fmt.Errorf("error verifying if ServiceMonitor %s/%s exists: %w", ns, stsName, err)
+	}
+	return cl.Delete(ctx, u)
+}
+
+// newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that
+// proxy that can be applied to the kube API server.
+// The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema.
+func newServiceMonitor(metricsSvc *corev1.Service) (*unstructured.Unstructured, error) {
+	sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace)
+	sm.ObjectMeta.Labels = metricsSvc.Labels
+	sm.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(metricsSvc, corev1.SchemeGroupVersion.WithKind("Service"))}
+	sm.Spec = ServiceMonitorSpec{
+		Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels},
+		Endpoints: []ServiceMonitorEndpoint{{
+			Port: "metrics",
+		}},
+		NamespaceSelector: ServiceMonitorNamespaceSelector{
+			MatchNames: []string{metricsSvc.Namespace},
+		},
+		JobLabel: labelPromJob,
+		TargetLabels: []string{
+			labelPromProxyParentName,
+			labelPromProxyParentNamespace,
+			labelPromProxyType,
+		},
+	}
+	return serviceMonitorToUnstructured(sm)
+}
+
+// serviceMonitorToUnstructured takes a ServiceMonitor and converts it to Unstructured type that can be used by the c/r
+// client in Kubernetes API server calls.
+func serviceMonitorToUnstructured(sm *ServiceMonitor) (*unstructured.Unstructured, error) {
+	contents, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sm)
+	if err != nil {
+		return nil, fmt.Errorf("error converting ServiceMonitor to Unstructured: %w", err)
+	}
+	u := &unstructured.Unstructured{}
+	u.SetUnstructuredContent(contents)
+	u.SetGroupVersionKind(sm.GroupVersionKind())
+	return u, nil
+}
+
+// metricsResourceName returns name for metrics Service and ServiceMonitor for a proxy StatefulSet.
+func metricsResourceName(stsName string) string {
+	// Maximum length of StatefulSet name if 52 chars, so this is fine.
+	return fmt.Sprintf("%s-metrics", stsName)
+}
+
+// metricsResourceLabels constructs labels that will be applied to metrics Service and metrics ServiceMonitor for a
+// proxy.
+func metricsResourceLabels(opts *metricsOpts) map[string]string {
+	lbls := map[string]string{
+		LabelManaged:             "true",
+		labelMetricsTarget:       opts.proxyStsName,
+		labelPromProxyType:       opts.proxyType,
+		labelPromProxyParentName: opts.proxyLabels[LabelParentName],
+	}
+	// Include namespace label for proxies created for a namespaced type.
+	if isNamespacedProxyType(opts.proxyType) {
+		lbls[labelPromProxyParentNamespace] = opts.proxyLabels[LabelParentNamespace]
+	}
+	lbls[labelPromJob] = promJobName(opts)
+	return lbls
+}
+
+// promJobName constructs the value of the Prometheus job label that will apply to all metrics for a ServiceMonitor.
+func promJobName(opts *metricsOpts) string {
+	// Include parent resource namespace for proxies created for namespaced types.
+	if opts.proxyType == proxyTypeIngressResource || opts.proxyType == proxyTypeIngressService {
+		return fmt.Sprintf("ts_%s_%s_%s", opts.proxyType, opts.proxyLabels[LabelParentNamespace], opts.proxyLabels[LabelParentName])
+	}
+	return fmt.Sprintf("ts_%s_%s", opts.proxyType, opts.proxyLabels[LabelParentName])
+}
+
+// metricsSvcSelector returns the minimum label set to uniquely identify a metrics Service for a proxy.
+func metricsSvcSelector(proxyLabels map[string]string, proxyType string) map[string]string {
+	sel := map[string]string{
+		labelPromProxyType:       proxyType,
+		labelPromProxyParentName: proxyLabels[LabelParentName],
+	}
+	// Include namespace label for proxies created for a namespaced type.
+	if isNamespacedProxyType(proxyType) {
+		sel[labelPromProxyParentNamespace] = proxyLabels[LabelParentNamespace]
+	}
+	return sel
+}
+
+// serviceMonitorTemplate returns a base ServiceMonitor type that, when converted to Unstructured, is a valid type that
+// can be used in kube API server calls via the c/r client.
+func serviceMonitorTemplate(name, ns string) *ServiceMonitor {
+	return &ServiceMonitor{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ServiceMonitor",
+			APIVersion: "monitoring.coreos.com/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+	}
+}
+
+type metricsOpts struct {
+	proxyStsName string            // name of StatefulSet for proxy
+	tsNamespace  string            // namespace in which Tailscale is installed
+	proxyLabels  map[string]string // labels of the proxy StatefulSet
+	proxyType    string
+}
+
+func isNamespacedProxyType(typ string) bool {
+	return typ == proxyTypeIngressResource || typ == proxyTypeIngressService
+}

cmd/k8s-operator/nameserver.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"slices"
+	"strings"
 	"sync"
 
 	_ "embed"
@@ -86,7 +87,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 			return reconcile.Result{}, nil
 		}
 		logger.Info("Cleaning up DNSConfig resources")
-		if err := a.maybeCleanup(ctx, &dnsCfg, logger); err != nil {
+		if err := a.maybeCleanup(&dnsCfg); err != nil {
 			logger.Errorf("error cleaning up reconciler resource: %v", err)
 			return res, err
 		}
@@ -100,9 +101,9 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	}
 
 	oldCnStatus := dnsCfg.Status.DeepCopy()
-	setStatus := func(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+	setStatus := func(dnsCfg *tsapi.DNSConfig, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, a.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldCnStatus, dnsCfg.Status) {
+		if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) {
 			// An error encountered here should get returned by the Reconcile function.
 			if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil {
 				err = errors.Wrap(err, updateErr.Error())
@@ -118,7 +119,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		msg := "invalid cluster configuration: more than one tailscale.com/dnsconfigs found. Please ensure that no more than one is created."
 		logger.Error(msg)
 		a.recorder.Event(&dnsCfg, corev1.EventTypeWarning, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
-		setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
+		setStatus(&dnsCfg, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
 	}
 
 	if !slices.Contains(dnsCfg.Finalizers, FinalizerName) {
@@ -127,11 +128,16 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		if err := a.Update(ctx, &dnsCfg); err != nil {
 			msg := fmt.Sprintf(messageNameserverCreationFailed, err)
 			logger.Error(msg)
-			return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
+			return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
 		}
 	}
 	if err := a.maybeProvision(ctx, &dnsCfg, logger); err != nil {
-		return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			logger.Infof("optimistic lock error, retrying: %s", err)
+			return reconcile.Result{}, nil
+		} else {
+			return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
+		}
 	}
 
 	a.mu.Lock()
@@ -149,7 +155,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
 			IP: ip,
 		}
-		return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated)
+		return setStatus(&dnsCfg, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated)
 	}
 	logger.Info("nameserver Service does not have an IP address allocated, waiting...")
 	return reconcile.Result{}, nil
@@ -188,7 +194,7 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa
 // maybeCleanup removes DNSConfig from being tracked. The cluster resources
 // created, will be automatically garbage collected as they are owned by the
 // DNSConfig.
-func (a *NameserverReconciler) maybeCleanup(ctx context.Context, dnsCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error {
+func (a *NameserverReconciler) maybeCleanup(dnsCfg *tsapi.DNSConfig) error {
 	a.mu.Lock()
 	a.managedNameservers.Remove(dnsCfg.UID)
 	a.mu.Unlock()

cmd/k8s-operator/operator.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -23,8 +24,11 @@ import (
 	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
+	toolscache "k8s.io/client-go/tools/cache"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -150,6 +154,13 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 		Hostname: hostname,
 		Logf:     zlog.Named("tailscaled").Debugf,
 	}
+	if p := os.Getenv("TS_PORT"); p != "" {
+		port, err := strconv.ParseUint(p, 10, 16)
+		if err != nil {
+			startlog.Fatalf("TS_PORT %q cannot be parsed as uint16: %v", p, err)
+		}
+		s.Port = uint16(port)
+	}
 	if kubeSecret != "" {
 		st, err := kubestore.New(logger.Discard, kubeSecret)
 		if err != nil {
@@ -231,21 +242,29 @@ func runReconcilers(opts reconcilerOpts) {
 	nsFilter := cache.ByObject{
 		Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(),
 	}
+	// We watch the ServiceMonitor CRD to ensure that reconcilers are re-triggered if user's workflows result in the
+	// ServiceMonitor CRD applied after some of our resources that define ServiceMonitor creation. This selector
+	// ensures that we only watch the ServiceMonitor CRD and that we don't cache full contents of it.
+	serviceMonitorSelector := cache.ByObject{
+		Field:     fields.SelectorFromSet(fields.Set{"metadata.name": serviceMonitorCRD}),
+		Transform: crdTransformer(startlog),
+	}
 	mgrOpts := manager.Options{
 		// TODO (irbekrm): stricter filtering what we watch/cache/call
 		// reconcilers on. c/r by default starts a watch on any
 		// resources that we GET via the controller manager's client.
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
-				&corev1.Secret{}:             nsFilter,
-				&corev1.ServiceAccount{}:     nsFilter,
-				&corev1.Pod{}:                nsFilter,
-				&corev1.ConfigMap{}:          nsFilter,
-				&appsv1.StatefulSet{}:        nsFilter,
-				&appsv1.Deployment{}:         nsFilter,
-				&discoveryv1.EndpointSlice{}: nsFilter,
-				&rbacv1.Role{}:               nsFilter,
-				&rbacv1.RoleBinding{}:        nsFilter,
+				&corev1.Secret{}:                            nsFilter,
+				&corev1.ServiceAccount{}:                    nsFilter,
+				&corev1.Pod{}:                               nsFilter,
+				&corev1.ConfigMap{}:                         nsFilter,
+				&appsv1.StatefulSet{}:                       nsFilter,
+				&appsv1.Deployment{}:                        nsFilter,
+				&discoveryv1.EndpointSlice{}:                nsFilter,
+				&rbacv1.Role{}:                              nsFilter,
+				&rbacv1.RoleBinding{}:                       nsFilter,
+				&apiextensionsv1.CustomResourceDefinition{}: serviceMonitorSelector,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
@@ -414,8 +433,13 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
 	}
 
+	// ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that
+	// define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get
+	// reconciled if the CRD is applied at a later point.
+	serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
+		Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter).
 		Complete(&ProxyClassReconciler{
 			Client:   mgr.GetClient(),
 			recorder: eventRecorder,
@@ -1010,6 +1034,49 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns
 	}
 }
 
+// proxyClassesWithServiceMonitor returns an event handler that, given that the event is for the Prometheus
+// ServiceMonitor CRD, returns all ProxyClasses that define that a ServiceMonitor should be created.
+func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		crd, ok := o.(*apiextensionsv1.CustomResourceDefinition)
+		if !ok {
+			logger.Debugf("[unexpected] ServiceMonitor CRD handler received an object that is not a CustomResourceDefinition")
+			return nil
+		}
+		if crd.Name != serviceMonitorCRD {
+			logger.Debugf("[unexpected] ServiceMonitor CRD handler received an unexpected CRD %q", crd.Name)
+			return nil
+		}
+		pcl := &tsapi.ProxyClassList{}
+		if err := cl.List(ctx, pcl); err != nil {
+			logger.Debugf("[unexpected] error listing ProxyClasses: %v", err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, pc := range pcl.Items {
+			if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable {
+				reqs = append(reqs, reconcile.Request{
+					NamespacedName: types.NamespacedName{Namespace: pc.Namespace, Name: pc.Name},
+				})
+			}
+		}
+		return reqs
+	}
+}
+
+// crdTransformer gets called before a CRD is stored to c/r cache, it removes the CRD spec to reduce memory consumption.
+func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc {
+	return func(o any) (any, error) {
+		crd, ok := o.(*apiextensionsv1.CustomResourceDefinition)
+		if !ok {
+			log.Infof("[unexpected] CRD transformer called for a non-CRD type")
+			return crd, nil
+		}
+		crd.Spec = apiextensionsv1.CustomResourceDefinitionSpec{}
+		return crd, nil
+	}
+}
+
 // indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
 // index is used a list filter.
 func indexEgressServices(o client.Object) []string {

cmd/k8s-operator/operator_test.go

@@ -432,6 +432,148 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 }
 
+func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		clock:    clock,
+		recorder: record.NewFakeRecorder(100),
+	}
+	tailnetTargetIP := "invalid-ip"
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	t0 := conditionTime(clock)
+
+	want := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyReady),
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: t0,
+				Reason:             reasonProxyInvalid,
+				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "invalid-ip" could not be parsed as a valid IP Address, error: ParseAddr("invalid-ip"): unable to parse IP`,
+			}},
+		},
+	}
+
+	expectEqual(t, fc, want, nil)
+}
+
+func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		clock:    clock,
+		recorder: record.NewFakeRecorder(100),
+	}
+	tailnetTargetIP := "999.999.999.999"
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	t0 := conditionTime(clock)
+
+	want := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyReady),
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: t0,
+				Reason:             reasonProxyInvalid,
+				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "999.999.999.999" could not be parsed as a valid IP Address, error: ParseAddr("999.999.999.999"): IPv4 field has value >255`,
+			}},
+		},
+	}
+
+	expectEqual(t, fc, want, nil)
+}
+
 func TestAnnotations(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
@@ -1246,7 +1388,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
+		confFileHash:    "acf3467364b0a3ba9b8ee0dd772cb7c2f0bf585e288fa99b7fe4566009ed6041",
 		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
@@ -1257,7 +1399,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684"
+	o.confFileHash = "d4cc13f09f55f4f6775689004f9a466723325b84d2b590692796bfe22aeaa389"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }

cmd/k8s-operator/proxyclass.go

@@ -15,6 +15,7 @@ import (
 	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
@@ -95,14 +96,14 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	pcr.mu.Unlock()
 
 	oldPCStatus := pc.Status.DeepCopy()
-	if errs := pcr.validate(pc); errs != nil {
+	if errs := pcr.validate(ctx, pc); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
 		pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
 		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
 	} else {
 		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
 	}
-	if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) {
+	if !apiequality.Semantic.DeepEqual(oldPCStatus, &pc.Status) {
 		if err := pcr.Client.Status().Update(ctx, pc); err != nil {
 			logger.Errorf("error updating ProxyClass status: %v", err)
 			return reconcile.Result{}, err
@@ -111,7 +112,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	return reconcile.Result{}, nil
 }
 
-func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
+func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) {
 	if sts := pc.Spec.StatefulSet; sts != nil {
 		if len(sts.Labels) > 0 {
 			if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
@@ -160,9 +161,23 @@ func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations fiel
 						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
 					}
 				}
+
+				if tc.Debug != nil {
+					violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "debug"), tc.Debug, "debug settings cannot be configured on the init container"))
+				}
 			}
 		}
 	}
+	if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable {
+		found, err := hasServiceMonitorCRD(ctx, pcr.Client)
+		if err != nil {
+			pcr.logger.Infof("[unexpected]: error retrieving %q CRD: %v", serviceMonitorCRD, err)
+			// best effort validation - don't error out here
+		} else if !found {
+			msg := fmt.Sprintf("ProxyClass defines that a ServiceMonitor custom resource should be created, but %q CRD was not found", serviceMonitorCRD)
+			violations = append(violations, field.TypeInvalid(field.NewPath("spec", "metrics", "serviceMonitor"), "enable", msg))
+		}
+	}
 	// We do not validate embedded fields (security context, resource
 	// requirements etc) as we inherit upstream validation for those fields.
 	// Invalid values would get rejected by upstream validations at apply
@@ -170,6 +185,16 @@ func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations fiel
 	return violations
 }
 
+func hasServiceMonitorCRD(ctx context.Context, cl client.Client) (bool, error) {
+	sm := &apiextensionsv1.CustomResourceDefinition{}
+	if err := cl.Get(ctx, types.NamespacedName{Name: serviceMonitorCRD}, sm); apierrors.IsNotFound(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 // maybeCleanup removes tailscale.com finalizer and ensures that the ProxyClass
 // is no longer counted towards k8s_proxyclass_resources.
 func (pcr *ProxyClassReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, pc *tsapi.ProxyClass) error {

cmd/k8s-operator/proxyclass_test.go

@@ -8,10 +8,12 @@
 package main
 
 import (
+	"context"
 	"testing"
 	"time"
 
 	"go.uber.org/zap"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -134,4 +136,76 @@ func TestProxyClass(t *testing.T) {
 		"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."}
 	expectReconciled(t, pcr, "", "test")
 	expectEvents(t, fr, expectedEvents)
+
+	// 6. A ProxyClass with ServiceMonitor enabled and in a cluster that has not ServiceMonitor CRD is invalid
+	pc.Spec.Metrics = &tsapi.Metrics{Enable: true, ServiceMonitor: &tsapi.ServiceMonitor{Enable: true}}
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec = pc.Spec
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.metrics.serviceMonitor: Invalid value: "enable": ProxyClass defines that a ServiceMonitor custom resource should be created, but "servicemonitors.monitoring.coreos.com" CRD was not found`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = "Warning ProxyClassInvalid " + msg
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 7. A ProxyClass with ServiceMonitor enabled and in a cluster that does have the ServiceMonitor CRD is valid
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	mustCreate(t, fc, crd)
+	expectReconciled(t, pcr, "", "test")
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+}
+
+func TestValidateProxyClass(t *testing.T) {
+	for name, tc := range map[string]struct {
+		pc    *tsapi.ProxyClass
+		valid bool
+	}{
+		"empty": {
+			valid: true,
+			pc:    &tsapi.ProxyClass{},
+		},
+		"debug_enabled_for_main_container": {
+			valid: true,
+			pc: &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{
+									Enable: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"debug_enabled_for_init_container": {
+			valid: false,
+			pc: &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleInitContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{
+									Enable: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			pcr := &ProxyClassReconciler{}
+			err := pcr.validate(context.Background(), tc.pc)
+			valid := err == nil
+			if valid != tc.valid {
+				t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
+			}
+		})
+	}
 }

cmd/k8s-operator/proxygroup.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"net/http"
 	"slices"
+	"strings"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -45,9 +46,12 @@ const (
 	reasonProxyGroupReady          = "ProxyGroupReady"
 	reasonProxyGroupCreating       = "ProxyGroupCreating"
 	reasonProxyGroupInvalid        = "ProxyGroupInvalid"
+
+	// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
+	optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
 )
 
-var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupCount)
+var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
 
 // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
 type ProxyGroupReconciler struct {
@@ -110,7 +114,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	oldPGStatus := pg.Status.DeepCopy()
 	setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldPGStatus, pg.Status) {
+		if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
 			// An error encountered here should get returned by the Reconcile function.
 			if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
 				err = errors.Wrap(err, updateErr.Error())
@@ -166,9 +170,17 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	}
 
 	if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
-		err = fmt.Errorf("error provisioning ProxyGroup resources: %w", err)
-		r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
-		return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
+		reason := reasonProxyGroupCreationFailed
+		msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err)
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			reason = reasonProxyGroupCreating
+			msg = fmt.Sprintf("optimistic lock error, retrying: %s", err)
+			err = nil
+			logger.Info(msg)
+		} else {
+			r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg)
+		}
+		return setStatusReady(pg, metav1.ConditionFalse, reason, msg)
 	}
 
 	desiredReplicas := int(pgReplicas(pg))
@@ -259,6 +271,15 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	}); err != nil {
 		return fmt.Errorf("error provisioning StatefulSet: %w", err)
 	}
+	mo := &metricsOpts{
+		tsNamespace:  r.tsNamespace,
+		proxyStsName: pg.Name,
+		proxyLabels:  pgLabels(pg.Name, nil),
+		proxyType:    "proxygroup",
+	}
+	if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil {
+		return fmt.Errorf("error reconciling metrics resources: %w", err)
+	}
 
 	if err := r.cleanupDanglingResources(ctx, pg); err != nil {
 		return fmt.Errorf("error cleaning up dangling resources: %w", err)
@@ -327,6 +348,14 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
 		}
 	}
 
+	mo := &metricsOpts{
+		proxyLabels: pgLabels(pg.Name, nil),
+		tsNamespace: r.tsNamespace,
+		proxyType:   "proxygroup"}
+	if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil {
+		return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
+	}
+
 	logger.Infof("cleaned up ProxyGroup resources")
 	r.mu.Lock()
 	r.proxyGroups.Remove(pg.UID)
@@ -353,7 +382,7 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc
 
 func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) {
 	logger := r.logger(pg.Name)
-	var allConfigs []tailscaledConfigs
+	var configSHA256Sum string
 	for i := range pgReplicas(pg) {
 		cfgSecret := &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
@@ -389,7 +418,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		if err != nil {
 			return "", fmt.Errorf("error creating tailscaled config: %w", err)
 		}
-		allConfigs = append(allConfigs, configs)
 
 		for cap, cfg := range configs {
 			cfgJSON, err := json.Marshal(cfg)
@@ -399,6 +427,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
 		}
 
+		// The config sha256 sum is a value for a hash annotation used to trigger
+		// pod restarts when tailscaled config changes. Any config changes apply
+		// to all replicas, so it is sufficient to only hash the config for the
+		// first replica.
+		//
+		// In future, we're aiming to eliminate restarts altogether and have
+		// pods dynamically reload their config when it changes.
+		if i == 0 {
+			sum := sha256.New()
+			for _, cfg := range configs {
+				// Zero out the auth key so it doesn't affect the sha256 hash when we
+				// remove it from the config after the pods have all authed. Otherwise
+				// all the pods will need to restart immediately after authing.
+				cfg.AuthKey = nil
+				b, err := json.Marshal(cfg)
+				if err != nil {
+					return "", err
+				}
+				if _, err := sum.Write(b); err != nil {
+					return "", err
+				}
+			}
+
+			configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil))
+		}
+
 		if existingCfgSecret != nil {
 			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
 			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
@@ -412,16 +466,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		}
 	}
 
-	sum := sha256.New()
-	b, err := json.Marshal(allConfigs)
-	if err != nil {
-		return "", err
-	}
-	if _, err := sum.Write(b); err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%x", sum.Sum(nil)), nil
+	return configSHA256Sum, nil
 }
 
 func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {

cmd/k8s-operator/proxygroup_specs.go

@@ -15,6 +15,7 @@ import (
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
 
@@ -92,6 +93,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 	c.Image = image
 	c.VolumeMounts = func() []corev1.VolumeMount {
 		var mounts []corev1.VolumeMount
+
+		// TODO(tomhjp): Read config directly from the secret instead. The
+		// mounts change on scaling up/down which causes unnecessary restarts
+		// for pods that haven't meaningfully changed.
 		for i := range pgReplicas(pg) {
 			mounts = append(mounts, corev1.VolumeMount{
 				Name:      fmt.Sprintf("tailscaledconfig-%d", i),
@@ -121,15 +126,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 					},
 				},
 			},
-			{
-				Name: "POD_NAME",
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						// Secret is named after the pod.
-						FieldPath: "metadata.name",
-					},
-				},
-			},
 			{
 				Name:  "TS_KUBE_SECRET",
 				Value: "$(POD_NAME)",
@@ -143,8 +139,8 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				Value: "/etc/tsconfig/$(POD_NAME)",
 			},
 			{
-				Name:  "TS_USERSPACE",
-				Value: "false",
+				Name:  "TS_INTERNAL_APP",
+				Value: kubetypes.AppProxyGroupEgress,
 			},
 		}
 
@@ -162,7 +158,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 			})
 		}
 
-		return envs
+		return append(c.Env, envs...)
 	}()
 
 	return ss, nil
@@ -206,6 +202,15 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 					return secrets
 				}(),
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"create",
+					"patch",
+					"get",
+				},
+			},
 		},
 	}
 }

cmd/k8s-operator/proxygroup_test.go

@@ -17,6 +17,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -35,6 +36,8 @@ var defaultProxyClassAnnotations = map[string]string{
 }
 
 func TestProxyGroup(t *testing.T) {
+	const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196"
+
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "default-pc",
@@ -74,12 +77,20 @@ func TestProxyGroup(t *testing.T) {
 		l:        zl.Sugar(),
 		clock:    cl,
 	}
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	opts := configOpts{
+		proxyType:          "proxygroup",
+		stsName:            pg.Name,
+		parentType:         "proxygroup",
+		tailscaleNamespace: "tailscale",
+	}
 
 	t.Run("proxyclass_not_ready", func(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, false, "")
 	})
 
 	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
@@ -100,10 +111,11 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		if expected := 1; reconciler.proxyGroups.Len() != expected {
 			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
 		}
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		keyReq := tailscale.KeyCapabilities{
 			Devices: tailscale.KeyDeviceCapabilities{
 				Create: tailscale.KeyDeviceCreateCapabilities{
@@ -135,7 +147,7 @@ func TestProxyGroup(t *testing.T) {
 		}
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 	})
 
 	t.Run("scale_up_to_3", func(t *testing.T) {
@@ -146,6 +158,7 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 
 		addNodeIDToStateSecrets(t, fc, pg)
 		expectReconciled(t, reconciler, "", pg.Name)
@@ -155,7 +168,7 @@ func TestProxyGroup(t *testing.T) {
 			TailnetIPs: []string{"1.2.3.4", "::1"},
 		})
 		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 	})
 
 	t.Run("scale_down_to_1", func(t *testing.T) {
@@ -163,11 +176,47 @@ func TestProxyGroup(t *testing.T) {
 		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
 			p.Spec = pg.Spec
 		})
+
 		expectReconciled(t, reconciler, "", pg.Name)
+
 		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+	})
 
-		expectProxyGroupResources(t, fc, pg, true)
+	t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) {
+		pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{
+			AcceptRoutes: true,
+		}
+		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+			p.Spec = pc.Spec
+		})
+
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74")
+	})
+
+	t.Run("enable_metrics", func(t *testing.T) {
+		pc.Spec.Metrics = &tsapi.Metrics{Enable: true}
+		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+			p.Spec = pc.Spec
+		})
+		expectReconciled(t, reconciler, "", pg.Name)
+		expectEqual(t, fc, expectedMetricsService(opts), nil)
+	})
+	t.Run("enable_service_monitor_no_crd", func(t *testing.T) {
+		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
+		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+			p.Spec.Metrics = pc.Spec.Metrics
+		})
+		expectReconciled(t, reconciler, "", pg.Name)
+	})
+	t.Run("create_crd_expect_service_monitor", func(t *testing.T) {
+		mustCreate(t, fc, crd)
+		expectReconciled(t, reconciler, "", pg.Name)
+		expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
 	})
 
 	t.Run("delete_and_cleanup", func(t *testing.T) {
@@ -177,7 +226,7 @@ func TestProxyGroup(t *testing.T) {
 
 		expectReconciled(t, reconciler, "", pg.Name)
 
-		expectMissing[tsapi.Recorder](t, fc, "", pg.Name)
+		expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
 		if expected := 0; reconciler.proxyGroups.Len() != expected {
 			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
 		}
@@ -186,18 +235,19 @@ func TestProxyGroup(t *testing.T) {
 		if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" {
 			t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
 		}
+		expectMissing[corev1.Service](t, reconciler, "tailscale", metricsResourceName(pg.Name))
 		// The fake client does not clean up objects whose owner has been
 		// deleted, so we can't test for the owned resources getting deleted.
 	})
 }
 
-func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool) {
+func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
 	t.Helper()
 
 	role := pgRole(pg, tsNamespace)
 	roleBinding := pgRoleBinding(pg, tsNamespace)
 	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", cfgHash)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -207,9 +257,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 		expectEqual(t, fc, role, nil)
 		expectEqual(t, fc, roleBinding, nil)
 		expectEqual(t, fc, serviceAccount, nil)
-		expectEqual(t, fc, statefulSet, func(ss *appsv1.StatefulSet) {
-			ss.Spec.Template.Annotations[podAnnotationLastSetConfigFileHash] = ""
-		})
+		expectEqual(t, fc, statefulSet, nil)
 	} else {
 		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
 		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
@@ -218,11 +266,13 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 	}
 
 	var expectedSecrets []string
-	for i := range pgReplicas(pg) {
-		expectedSecrets = append(expectedSecrets,
-			fmt.Sprintf("%s-%d", pg.Name, i),
-			fmt.Sprintf("%s-%d-config", pg.Name, i),
-		)
+	if shouldExist {
+		for i := range pgReplicas(pg) {
+			expectedSecrets = append(expectedSecrets,
+				fmt.Sprintf("%s-%d", pg.Name, i),
+				fmt.Sprintf("%s-%d-config", pg.Name, i),
+			)
+		}
 	}
 	expectSecrets(t, fc, expectedSecrets)
 }

cmd/k8s-operator/sts.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"os"
 	"slices"
+	"strconv"
 	"strings"
 
 	"go.uber.org/zap"
@@ -94,6 +95,12 @@ const (
 	podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
 	// podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents.
 	podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash"
+
+	proxyTypeEgress          = "egress_service"
+	proxyTypeIngressService  = "ingress_service"
+	proxyTypeIngressResource = "ingress_resource"
+	proxyTypeConnector       = "connector"
+	proxyTypeProxyGroup      = "proxygroup"
 )
 
 var (
@@ -122,6 +129,8 @@ type tailscaleSTSConfig struct {
 	Hostname string
 	Tags     []string // if empty, use defaultTags
 
+	proxyType string
+
 	// Connector specifies a configuration of a Connector instance if that's
 	// what this StatefulSet should be created for.
 	Connector *connector
@@ -132,10 +141,13 @@ type tailscaleSTSConfig struct {
 }
 
 type connector struct {
-	// routes is a list of subnet routes that this Connector should expose.
+	// routes is a list of routes that this Connector should advertise either as a subnet router or as an app
+	// connector.
 	routes string
 	// isExitNode defines whether this Connector should act as an exit node.
 	isExitNode bool
+	// isAppConnector defines whether this Connector should act as an app connector.
+	isAppConnector bool
 }
 type tsnetServer interface {
 	CertDomains() []string
@@ -186,22 +198,30 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
 	}
 	sts.ProxyClass = proxyClass
 
-	secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
+	secretName, tsConfigHash, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
 	}
-	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash, configs)
+	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash)
 	if err != nil {
 		return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
 	}
-
+	mo := &metricsOpts{
+		proxyStsName: hsvc.Name,
+		tsNamespace:  hsvc.Namespace,
+		proxyLabels:  hsvc.Labels,
+		proxyType:    sts.proxyType,
+	}
+	if err = reconcileMetricsResources(ctx, logger, mo, sts.ProxyClass, a.Client); err != nil {
+		return nil, fmt.Errorf("failed to ensure metrics resources: %w", err)
+	}
 	return hsvc, nil
 }
 
 // Cleanup removes all resources associated that were created by Provision with
 // the given labels. It returns true when all resources have been removed,
 // otherwise it returns false and the caller should retry later.
-func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string) (done bool, _ error) {
+func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) {
 	// Need to delete the StatefulSet first, and delete it with foreground
 	// cascading deletion. That way, the pod that's writing to the Secret will
 	// stop running before we start looking at the Secret's contents, and
@@ -227,21 +247,21 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
 		return false, nil
 	}
 
-	id, _, _, err := a.DeviceInfo(ctx, labels)
+	dev, err := a.DeviceInfo(ctx, labels, logger)
 	if err != nil {
 		return false, fmt.Errorf("getting device info: %w", err)
 	}
-	if id != "" {
-		logger.Debugf("deleting device %s from control", string(id))
-		if err := a.tsClient.DeleteDevice(ctx, string(id)); err != nil {
+	if dev != nil && dev.id != "" {
+		logger.Debugf("deleting device %s from control", string(dev.id))
+		if err := a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
 			errResp := &tailscale.ErrResponse{}
 			if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
-				logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
+				logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
 			} else {
 				return false, fmt.Errorf("deleting device: %w", err)
 			}
 		} else {
-			logger.Debugf("device %s deleted from control", string(id))
+			logger.Debugf("device %s deleted from control", string(dev.id))
 		}
 	}
 
@@ -254,6 +274,14 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
 			return false, err
 		}
 	}
+	mo := &metricsOpts{
+		proxyLabels: labels,
+		tsNamespace: a.operatorNamespace,
+		proxyType:   typ,
+	}
+	if err := maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
+		return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
+	}
 	return true, nil
 }
 
@@ -413,40 +441,66 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
 // that acts as an operator proxy. It retrieves info from a Kubernetes Secret
 // labeled with the provided labels.
 // Either of device ID, hostname and IPs can be empty string if not found in the Secret.
-func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
+func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) {
 	sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
 	if err != nil {
-		return "", "", nil, err
+		return dev, err
 	}
 	if sec == nil {
-		return "", "", nil, nil
+		return dev, nil
+	}
+	pod := new(corev1.Pod)
+	if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) {
+		return dev, nil
 	}
 
-	return deviceInfo(sec)
+	return deviceInfo(sec, pod, logger)
 }
 
-func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
-	id = tailcfg.StableNodeID(sec.Data["device_id"])
+// device contains tailscale state of a proxy device as gathered from its tailscale state Secret.
+type device struct {
+	id       tailcfg.StableNodeID // device's stable ID
+	hostname string               // MagicDNS name of the device
+	ips      []string             // Tailscale IPs of the device
+	// ingressDNSName is the L7 Ingress DNS name. In practice this will be the same value as hostname, but only set
+	// when the device has been configured to serve traffic on it via 'tailscale serve'.
+	ingressDNSName string
+}
+
+func deviceInfo(sec *corev1.Secret, pod *corev1.Pod, log *zap.SugaredLogger) (dev *device, err error) {
+	id := tailcfg.StableNodeID(sec.Data[kubetypes.KeyDeviceID])
 	if id == "" {
-		return "", "", nil, nil
+		return dev, nil
 	}
+	dev = &device{id: id}
 	// Kubernetes chokes on well-formed FQDNs with the trailing dot, so we have
 	// to remove it.
-	hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
-	if hostname == "" {
+	dev.hostname = strings.TrimSuffix(string(sec.Data[kubetypes.KeyDeviceFQDN]), ".")
+	if dev.hostname == "" {
 		// Device ID gets stored and retrieved in a different flow than
 		// FQDN and IPs. A device that acts as Kubernetes operator
-		// proxy, but whose route setup has failed might have an device
+		// proxy, but whose route setup has failed might have a device
 		// ID, but no FQDN/IPs. If so, return the ID, to allow the
 		// operator to clean up such devices.
-		return id, "", nil, nil
+		return dev, nil
 	}
-	if rawDeviceIPs, ok := sec.Data["device_ips"]; ok {
-		if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
-			return "", "", nil, err
+	// TODO(irbekrm): we fall back to using the hostname field to determine Ingress's hostname to ensure backwards
+	// compatibility. In 1.82 we can remove this fallback mechanism.
+	dev.ingressDNSName = dev.hostname
+	if proxyCapVer(sec, pod, log) >= 109 {
+		dev.ingressDNSName = strings.TrimSuffix(string(sec.Data[kubetypes.KeyHTTPSEndpoint]), ".")
+		if strings.EqualFold(dev.ingressDNSName, kubetypes.ValueNoHTTPS) {
+			dev.ingressDNSName = ""
 		}
 	}
-	return id, hostname, ips, nil
+	if rawDeviceIPs, ok := sec.Data[kubetypes.KeyDeviceIPs]; ok {
+		ips := make([]string, 0)
+		if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
+			return nil, err
+		}
+		dev.ips = ips
+	}
+	return dev, nil
 }
 
 func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) {
@@ -473,7 +527,7 @@ var proxyYaml []byte
 //go:embed deploy/manifests/userspace-proxy.yaml
 var userspaceProxyYaml []byte
 
-func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -518,11 +572,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_KUBE_SECRET",
 			Value: proxySecret,
 		},
-		corev1.EnvVar{
-			// Old tailscaled config key is still used for backwards compatibility.
-			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
-			Value: "/etc/tsconfig/tailscaled",
-		},
 		corev1.EnvVar{
 			// New style is in the form of cap-<capability-version>.hujson.
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
@@ -668,24 +717,42 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [
 	return custom
 }
 
+func debugSetting(pc *tsapi.ProxyClass) bool {
+	if pc == nil ||
+		pc.Spec.StatefulSet == nil ||
+		pc.Spec.StatefulSet.Pod == nil ||
+		pc.Spec.StatefulSet.Pod.TailscaleContainer == nil ||
+		pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug == nil {
+		// This default will change to false in 1.82.0.
+		return pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
+	}
+
+	return pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug.Enable
+}
+
 func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet {
 	if pc == nil || ss == nil {
 		return ss
 	}
-	if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
-		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
-			enableMetrics(ss, pc)
-		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+
+	metricsEnabled := pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
+	debugEnabled := debugSetting(pc)
+	if metricsEnabled || debugEnabled {
+		isEgress := stsCfg != nil && (stsCfg.TailnetTargetFQDN != "" || stsCfg.TailnetTargetIP != "")
+		isForwardingL7Ingress := stsCfg != nil && stsCfg.ForwardClusterTrafficViaL7IngressProxy
+		if isEgress {
 			// TODO (irbekrm): fix this
 			// For Ingress proxies that have been configured with
 			// tailscale.com/experimental-forward-cluster-traffic-via-ingress
 			// annotation, all cluster traffic is forwarded to the
 			// Ingress backend(s).
-			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
-		} else {
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for egress proxies.")
+		} else if isForwardingL7Ingress {
 			// TODO (irbekrm): fix this
 			// For egress proxies, currently all cluster traffic is forwarded to the tailnet target.
 			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		} else {
+			enableEndpoints(ss, metricsEnabled, debugEnabled)
 		}
 	}
 
@@ -718,6 +785,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
 	ss.Spec.Template.Spec.Affinity = wantsPod.Affinity
 	ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
+	ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints
 
 	// Update containers.
 	updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container {
@@ -762,16 +830,58 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	return ss
 }
 
-func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) {
+func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
 	for i, c := range ss.Spec.Template.Spec.Containers {
 		if c.Name == "tailscale" {
-			// Serve metrics on on <pod-ip>:9001/debug/metrics. If
-			// we didn't specify Pod IP here, the proxy would, in
-			// some cases, also listen to its Tailscale IP- we don't
-			// want folks to start relying on this side-effect as a
-			// feature.
-			ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
-			ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001})
+			if debug {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
+					// Serve tailscaled's debug metrics on on
+					// <pod-ip>:9001/debug/metrics. If we didn't specify Pod IP
+					// here, the proxy would, in some cases, also listen to its
+					// Tailscale IP- we don't want folks to start relying on this
+					// side-effect as a feature.
+					corev1.EnvVar{
+						Name:  "TS_DEBUG_ADDR_PORT",
+						Value: "$(POD_IP):9001",
+					},
+					// TODO(tomhjp): Can remove this env var once 1.76.x is no
+					// longer supported.
+					corev1.EnvVar{
+						Name:  "TS_TAILSCALED_EXTRA_ARGS",
+						Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
+					},
+				)
+
+				ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
+					corev1.ContainerPort{
+						Name:          "debug",
+						Protocol:      "TCP",
+						ContainerPort: 9001,
+					},
+				)
+			}
+
+			if metrics {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
+					// Serve client metrics on <pod-ip>:9002/metrics.
+					corev1.EnvVar{
+						Name:  "TS_LOCAL_ADDR_PORT",
+						Value: "$(POD_IP):9002",
+					},
+					corev1.EnvVar{
+						Name:  "TS_ENABLE_METRICS",
+						Value: "true",
+					},
+				)
+				ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
+					corev1.ContainerPort{
+						Name:          "metrics",
+						Protocol:      "TCP",
+						ContainerPort: 9002,
+					},
+				)
+			}
+
 			break
 		}
 	}
@@ -785,15 +895,9 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 	return origConf.AuthKey, nil
 }
 
-// tailscaledConfig takes a proxy config, a newly generated auth key if
-// generated and a Secret with the previous proxy state and auth key and
-// returns tailscaled configuration and a hash of that configuration.
-//
-// As of 2024-05-09 it also returns legacy tailscaled config without the
-// later added NoStatefulFilter field to support proxies older than cap95.
-// TODO (irbekrm): remove the legacy config once we no longer need to support
-// versions older than cap94,
-// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
+// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that
+// configuration.
 func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
@@ -801,21 +905,19 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		AcceptRoutes:        "false", // AcceptRoutes defaults to true
 		Locked:              "false",
 		Hostname:            &stsC.Hostname,
-		NoStatefulFiltering: "false",
+		NoStatefulFiltering: "true", // Explicitly enforce default value, see #14216
+		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
 	}
 
-	// For egress proxies only, we need to ensure that stateful filtering is
-	// not in place so that traffic from cluster can be forwarded via
-	// Tailscale IPs.
-	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
-		conf.NoStatefulFiltering = "true"
-	}
 	if stsC.Connector != nil {
 		routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
 		if err != nil {
 			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
 		conf.AdvertiseRoutes = routes
+		if stsC.Connector.isAppConnector {
+			conf.AppConnector.Advertise = true
+		}
 	}
 	if shouldAcceptRoutes(stsC.ProxyClass) {
 		conf.AcceptRoutes = "true"
@@ -830,11 +932,13 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		}
 		conf.AuthKey = key
 	}
+
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[107] = *conf
+
+	// AppConnector config option is only understood by clients of capver 107 and newer.
+	conf.AppConnector = nil
 	capVerConfigs[95] = *conf
-	// legacy config should not contain NoStatefulFiltering field.
-	conf.NoStatefulFiltering.Clear()
-	capVerConfigs[94] = *conf
 	return capVerConfigs, nil
 }
 
@@ -1007,3 +1111,23 @@ func nameForService(svc *corev1.Service) string {
 func isValidFirewallMode(m string) bool {
 	return m == "auto" || m == "nftables" || m == "iptables"
 }
+
+// proxyCapVer accepts a proxy state Secret and a proxy Pod returns the capability version of a proxy Pod.
+// This is best effort - if the capability version can not (currently) be determined, it returns -1.
+func proxyCapVer(sec *corev1.Secret, pod *corev1.Pod, log *zap.SugaredLogger) tailcfg.CapabilityVersion {
+	if sec == nil || pod == nil {
+		return tailcfg.CapabilityVersion(-1)
+	}
+	if len(sec.Data[kubetypes.KeyCapVer]) == 0 || len(sec.Data[kubetypes.KeyPodUID]) == 0 {
+		return tailcfg.CapabilityVersion(-1)
+	}
+	capVer, err := strconv.Atoi(string(sec.Data[kubetypes.KeyCapVer]))
+	if err != nil {
+		log.Infof("[unexpected]: unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer]))
+		return tailcfg.CapabilityVersion(-1)
+	}
+	if !strings.EqualFold(string(pod.ObjectMeta.UID), string(sec.Data[kubetypes.KeyPodUID])) {
+		return tailcfg.CapabilityVersion(-1)
+	}
+	return tailcfg.CapabilityVersion(capVer)
+}

cmd/k8s-operator/sts_test.go

@@ -18,6 +18,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/types/ptr"
@@ -73,6 +74,16 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 					NodeSelector:     map[string]string{"beta.kubernetes.io/os": "linux"},
 					Affinity:         &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}},
 					Tolerations:      []corev1.Toleration{{Key: "", Operator: "Exists"}},
+					TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
+						{
+							WhenUnsatisfiable: "DoNotSchedule",
+							TopologyKey:       "kubernetes.io/hostname",
+							MaxSkew:           3,
+							LabelSelector: &metav1.LabelSelector{
+								MatchLabels: map[string]string{"foo": "bar"},
+							},
+						},
+					},
 					TailscaleContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
 							Privileged: ptr.To(true),
@@ -114,10 +125,26 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 			},
 		},
 	}
-	proxyClassMetrics := &tsapi.ProxyClass{
-		Spec: tsapi.ProxyClassSpec{
-			Metrics: &tsapi.Metrics{Enable: true},
-		},
+
+	proxyClassWithMetricsDebug := func(metrics bool, debug *bool) *tsapi.ProxyClass {
+		return &tsapi.ProxyClass{
+			Spec: tsapi.ProxyClassSpec{
+				Metrics: &tsapi.Metrics{Enable: metrics},
+				StatefulSet: func() *tsapi.StatefulSet {
+					if debug == nil {
+						return nil
+					}
+
+					return &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{Enable: *debug},
+							},
+						},
+					}
+				}(),
+			},
+		}
 	}
 
 	var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
@@ -159,6 +186,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
+	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
@@ -172,7 +200,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 
 	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 2. Test that a ProxyClass with custom labels and annotations for
@@ -185,7 +213,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 3. Test that a ProxyClass with all fields set gets correctly applied
@@ -201,6 +229,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
+	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
@@ -208,7 +237,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
 	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
@@ -220,16 +249,48 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
-	// 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet.
+	// 5. Metrics enabled defaults to enabling both metrics and debug.
 	wantSS = nonUserspaceProxySS.DeepCopy()
-	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
-	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}}
-	gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
+		corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
+		corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
+		corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{
+		{Name: "debug", Protocol: "TCP", ContainerPort: 9001},
+		{Name: "metrics", Protocol: "TCP", ContainerPort: 9002},
+	}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, nil), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
+
+	// 6. Enable _just_ metrics by explicitly disabling debug.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
+		corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9002}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, ptr.To(false)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
+
+	// 7. Enable _just_ debug without metrics.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
+		corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "debug", Protocol: "TCP", ContainerPort: 9001}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(false, ptr.To(true)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
 	}
 }
 

cmd/k8s-operator/svc.go

@@ -121,7 +121,15 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
 	}
 
-	return reconcile.Result{}, a.maybeProvision(ctx, logger, svc)
+	if err := a.maybeProvision(ctx, logger, svc); err != nil {
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			logger.Infof("optimistic lock error, retrying: %s", err)
+		} else {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return reconcile.Result{}, nil
 }
 
 // maybeCleanup removes any existing resources related to serving svc over tailscale.
@@ -131,7 +139,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
 	oldSvcStatus := svc.Status.DeepCopy()
 	defer func() {
-		if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
+		if !apiequality.Semantic.DeepEqual(oldSvcStatus, &svc.Status) {
 			// An error encountered here should get returned by the Reconcile function.
 			err = errors.Join(err, a.Client.Status().Update(ctx, svc))
 		}
@@ -152,7 +160,12 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 		return nil
 	}
 
-	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc")); err != nil {
+	proxyTyp := proxyTypeEgress
+	if a.shouldExpose(svc) {
+		proxyTyp = proxyTypeIngressService
+	}
+
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil {
 		return fmt.Errorf("failed to cleanup: %w", err)
 	} else if !done {
 		logger.Debugf("cleanup not done yet, waiting for next reconcile")
@@ -191,7 +204,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
 	oldSvcStatus := svc.Status.DeepCopy()
 	defer func() {
-		if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
+		if !apiequality.Semantic.DeepEqual(oldSvcStatus, &svc.Status) {
 			// An error encountered here should get returned by the Reconcile function.
 			err = errors.Join(err, a.Client.Status().Update(ctx, svc))
 		}
@@ -256,6 +269,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		ChildResourceLabels: crl,
 		ProxyClassName:      proxyClass,
 	}
+	sts.proxyType = proxyTypeEgress
+	if a.shouldExpose(svc) {
+		sts.proxyType = proxyTypeIngressService
+	}
 
 	a.mu.Lock()
 	if a.shouldExposeClusterIP(svc) {
@@ -311,11 +328,11 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	_, tsHost, tsIPs, err := a.ssr.DeviceInfo(ctx, crl)
+	dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
 	if err != nil {
 		return fmt.Errorf("failed to get device ID: %w", err)
 	}
-	if tsHost == "" {
+	if dev == nil || dev.hostname == "" {
 		msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
 		logger.Debug(msg)
 		// No hostname yet. Wait for the proxy pod to auth.
@@ -324,9 +341,9 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	logger.Debugf("setting Service LoadBalancer status to %q, %s", tsHost, strings.Join(tsIPs, ", "))
+	logger.Debugf("setting Service LoadBalancer status to %q, %s", dev.hostname, strings.Join(dev.ips, ", "))
 	ingress := []corev1.LoadBalancerIngress{
-		{Hostname: tsHost},
+		{Hostname: dev.hostname},
 	}
 	clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
 	if err != nil {
@@ -334,7 +351,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
 		return errors.New(msg)
 	}
-	for _, ip := range tsIPs {
+	for _, ip := range dev.ips {
 		addr, err := netip.ParseAddr(ip)
 		if err != nil {
 			continue
@@ -358,9 +375,14 @@ func validateService(svc *corev1.Service) []string {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
-
-	// TODO(irbekrm): validate that tailscale.com/tailnet-ip annotation is a
-	// valid IP address (tailscale/tailscale#13671).
+	if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" {
+		ip, err := netip.ParseAddr(ipStr)
+		if err != nil {
+			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err))
+		} else if !ip.IsValid() {
+			violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr))
+		}
+	}
 
 	svcName := nameForService(svc)
 	if err := dnsname.ValidLabel(svcName); err != nil {

cmd/k8s-operator/testutils_test.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/netip"
 	"reflect"
 	"strings"
@@ -21,6 +22,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -39,7 +41,10 @@ type configOpts struct {
 	secretName                                     string
 	hostname                                       string
 	namespace                                      string
+	tailscaleNamespace                             string
+	namespaced                                     bool
 	parentType                                     string
+	proxyType                                      string
 	priorityClassName                              string
 	firewallMode                                   string
 	tailnetTargetIP                                string
@@ -48,6 +53,7 @@ type configOpts struct {
 	clusterTargetDNS                               string
 	subnetRoutes                                   string
 	isExitNode                                     bool
+	isAppConnector                                 bool
 	confFileHash                                   string
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
@@ -55,6 +61,7 @@ type configOpts struct {
 	app                                            string
 	shouldRemoveAuthKey                            bool
 	secretExtraData                                map[string][]byte
+	enableMetrics                                  bool
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -69,14 +76,13 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "false"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
-			Capabilities: &corev1.Capabilities{
-				Add: []corev1.Capability{"NET_ADMIN"},
-			},
+			Privileged: ptr.To(true),
 		},
 		ImagePullPolicy: "Always",
 	}
@@ -150,6 +156,29 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		Name:  "TS_INTERNAL_APP",
 		Value: opts.app,
 	})
+	if opts.enableMetrics {
+		tsContainer.Env = append(tsContainer.Env,
+			corev1.EnvVar{
+				Name:  "TS_DEBUG_ADDR_PORT",
+				Value: "$(POD_IP):9001"},
+			corev1.EnvVar{
+				Name:  "TS_TAILSCALED_EXTRA_ARGS",
+				Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
+			},
+			corev1.EnvVar{
+				Name:  "TS_LOCAL_ADDR_PORT",
+				Value: "$(POD_IP):9002",
+			},
+			corev1.EnvVar{
+				Name:  "TS_ENABLE_METRICS",
+				Value: "true",
+			},
+		)
+		tsContainer.Ports = append(tsContainer.Ports,
+			corev1.ContainerPort{Name: "debug", ContainerPort: 9001, Protocol: "TCP"},
+			corev1.ContainerPort{Name: "metrics", ContainerPort: 9002, Protocol: "TCP"},
+		)
+	}
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -228,8 +257,9 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 			{Name: "TS_INTERNAL_APP", Value: opts.app},
@@ -240,6 +270,29 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"},
 		},
 	}
+	if opts.enableMetrics {
+		tsContainer.Env = append(tsContainer.Env,
+			corev1.EnvVar{
+				Name:  "TS_DEBUG_ADDR_PORT",
+				Value: "$(POD_IP):9001"},
+			corev1.EnvVar{
+				Name:  "TS_TAILSCALED_EXTRA_ARGS",
+				Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
+			},
+			corev1.EnvVar{
+				Name:  "TS_LOCAL_ADDR_PORT",
+				Value: "$(POD_IP):9002",
+			},
+			corev1.EnvVar{
+				Name:  "TS_ENABLE_METRICS",
+				Value: "true",
+			},
+		)
+		tsContainer.Ports = append(tsContainer.Ports, corev1.ContainerPort{
+			Name: "debug", ContainerPort: 9001, Protocol: "TCP"},
+			corev1.ContainerPort{Name: "metrics", ContainerPort: 9002, Protocol: "TCP"},
+		)
+	}
 	volumes := []corev1.Volume{
 		{
 			Name: "tailscaledconfig",
@@ -334,6 +387,87 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
 	}
 }
 
+func expectedMetricsService(opts configOpts) *corev1.Service {
+	labels := metricsLabels(opts)
+	selector := map[string]string{
+		"tailscale.com/managed":              "true",
+		"tailscale.com/parent-resource":      "test",
+		"tailscale.com/parent-resource-type": opts.parentType,
+	}
+	if opts.namespaced {
+		selector["tailscale.com/parent-resource-ns"] = opts.namespace
+	}
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      metricsResourceName(opts.stsName),
+			Namespace: opts.tailscaleNamespace,
+			Labels:    labels,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: selector,
+			Type:     corev1.ServiceTypeClusterIP,
+			Ports:    []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}},
+		},
+	}
+}
+
+func metricsLabels(opts configOpts) map[string]string {
+	promJob := fmt.Sprintf("ts_%s_default_test", opts.proxyType)
+	if !opts.namespaced {
+		promJob = fmt.Sprintf("ts_%s_test", opts.proxyType)
+	}
+	labels := map[string]string{
+		"tailscale.com/managed":        "true",
+		"tailscale.com/metrics-target": opts.stsName,
+		"ts_prom_job":                  promJob,
+		"ts_proxy_type":                opts.proxyType,
+		"ts_proxy_parent_name":         "test",
+	}
+	if opts.namespaced {
+		labels["ts_proxy_parent_namespace"] = "default"
+	}
+	return labels
+}
+
+func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstructured {
+	t.Helper()
+	labels := metricsLabels(opts)
+	name := metricsResourceName(opts.stsName)
+	sm := &ServiceMonitor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			Namespace:       opts.tailscaleNamespace,
+			Labels:          labels,
+			ResourceVersion: "1",
+			OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: ptr.To(true), Controller: ptr.To(true)}},
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ServiceMonitor",
+			APIVersion: "monitoring.coreos.com/v1",
+		},
+		Spec: ServiceMonitorSpec{
+			Selector: metav1.LabelSelector{MatchLabels: labels},
+			Endpoints: []ServiceMonitorEndpoint{{
+				Port: "metrics",
+			}},
+			NamespaceSelector: ServiceMonitorNamespaceSelector{
+				MatchNames: []string{opts.tailscaleNamespace},
+			},
+			JobLabel: "ts_prom_job",
+			TargetLabels: []string{
+				"ts_proxy_parent_name",
+				"ts_proxy_parent_namespace",
+				"ts_proxy_type",
+			},
+		},
+	}
+	u, err := serviceMonitorToUnstructured(sm)
+	if err != nil {
+		t.Fatalf("error converting ServiceMonitor to unstructured: %v", err)
+	}
+	return u
+}
+
 func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
 	t.Helper()
 	s := &corev1.Secret{
@@ -350,12 +484,14 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		mak.Set(&s.StringData, "serve-config", string(serveConfigBs))
 	}
 	conf := &ipn.ConfigVAlpha{
-		Version:      "alpha0",
-		AcceptDNS:    "false",
-		Hostname:     &opts.hostname,
-		Locked:       "false",
-		AuthKey:      ptr.To("secret-authkey"),
-		AcceptRoutes: "false",
+		Version:             "alpha0",
+		AcceptDNS:           "false",
+		Hostname:            &opts.hostname,
+		Locked:              "false",
+		AuthKey:             ptr.To("secret-authkey"),
+		AcceptRoutes:        "false",
+		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
+		NoStatefulFiltering: "true",
 	}
 	if opts.proxyClass != "" {
 		t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
@@ -370,6 +506,9 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 	if opts.shouldRemoveAuthKey {
 		conf.AuthKey = nil
 	}
+	if opts.isAppConnector {
+		conf.AppConnector = &ipn.AppConnectorPrefs{Advertise: true}
+	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -385,21 +524,17 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		}
 	}
 	conf.AdvertiseRoutes = routes
-	b, err := json.Marshal(conf)
+	bnn, err := json.Marshal(conf)
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
-	if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
-		conf.NoStatefulFiltering = "true"
-	} else {
-		conf.NoStatefulFiltering = "false"
-	}
+	conf.AppConnector = nil
 	bn, err := json.Marshal(conf)
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
-	mak.Set(&s.StringData, "tailscaled", string(b))
 	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
+	mak.Set(&s.StringData, "cap-107.hujson", string(bnn))
 	labels := map[string]string{
 		"tailscale.com/managed":              "true",
 		"tailscale.com/parent-resource":      "test",
@@ -500,6 +635,21 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 	}
 }
 
+func expectEqualUnstructured(t *testing.T, client client.Client, want *unstructured.Unstructured) {
+	t.Helper()
+	got := &unstructured.Unstructured{}
+	got.SetGroupVersionKind(want.GroupVersionKind())
+	if err := client.Get(context.Background(), types.NamespacedName{
+		Name:      want.GetName(),
+		Namespace: want.GetNamespace(),
+	}, got); err != nil {
+		t.Fatalf("getting %q: %v", want.GetName(), err)
+	}
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Fatalf("unexpected contents of Unstructured (-got +want):\n%s", diff)
+	}
+}
+
 func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string) {
 	t.Helper()
 	obj := O(new(T))
@@ -642,7 +792,7 @@ func removeHashAnnotation(sts *appsv1.StatefulSet) {
 func removeTargetPortsFromSvc(svc *corev1.Service) {
 	newPorts := make([]corev1.ServicePort, 0)
 	for _, p := range svc.Spec.Ports {
-		newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port})
+		newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name})
 	}
 	svc.Spec.Ports = newPorts
 }
@@ -650,18 +800,6 @@ func removeTargetPortsFromSvc(svc *corev1.Service) {
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()
-		if len(secret.StringData["tailscaled"]) != 0 {
-			conf := &ipn.ConfigVAlpha{}
-			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
-				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
-			}
-			conf.AuthKey = nil
-			b, err := json.Marshal(conf)
-			if err != nil {
-				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
-			}
-			mak.Set(&secret.StringData, "tailscaled", string(b))
-		}
 		if len(secret.StringData["cap-95.hujson"]) != 0 {
 			conf := &ipn.ConfigVAlpha{}
 			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
@@ -674,5 +812,17 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 			}
 			mak.Set(&secret.StringData, "cap-95.hujson", string(b))
 		}
+		if len(secret.StringData["cap-107.hujson"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["cap-107.hujson"]), conf); err != nil {
+				t.Fatalf("error umarshalling 'cap-107.hujson' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling 'cap-107.huson' contents: %v", err)
+			}
+			mak.Set(&secret.StringData, "cap-107.hujson", string(b))
+		}
 	}
 }

cmd/k8s-operator/tsrecorder.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/http"
 	"slices"
+	"strings"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -38,6 +39,7 @@ import (
 
 const (
 	reasonRecorderCreationFailed = "RecorderCreationFailed"
+	reasonRecorderCreating       = "RecorderCreating"
 	reasonRecorderCreated        = "RecorderCreated"
 	reasonRecorderInvalid        = "RecorderInvalid"
 
@@ -102,7 +104,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
 	oldTSRStatus := tsr.Status.DeepCopy()
 	setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
-		if !apiequality.Semantic.DeepEqual(oldTSRStatus, tsr.Status) {
+		if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) {
 			// An error encountered here should get returned by the Reconcile function.
 			if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
 				err = errors.Wrap(err, updateErr.Error())
@@ -119,23 +121,28 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
 		logger.Infof("ensuring Recorder is set up")
 		tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
 		if err := r.Update(ctx, tsr); err != nil {
-			logger.Errorf("error adding finalizer: %w", err)
 			return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
 		}
 	}
 
 	if err := r.validate(tsr); err != nil {
-		logger.Errorf("error validating Recorder spec: %w", err)
 		message := fmt.Sprintf("Recorder is invalid: %s", err)
 		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message)
 		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
 	}
 
 	if err = r.maybeProvision(ctx, tsr); err != nil {
-		logger.Errorf("error creating Recorder resources: %w", err)
+		reason := reasonRecorderCreationFailed
 		message := fmt.Sprintf("failed creating Recorder: %s", err)
-		r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
-		return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, message)
+		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+			reason = reasonRecorderCreating
+			message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
+			err = nil
+			logger.Info(message)
+		} else {
+			r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
+		}
+		return setStatusReady(tsr, metav1.ConditionFalse, reason, message)
 	}
 
 	logger.Info("Recorder resources synced")

cmd/k8s-operator/tsrecorder_specs.go

@@ -130,6 +130,15 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
 					fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
 				},
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"get",
+					"create",
+					"patch",
+				},
+			},
 		},
 	}
 }
@@ -203,6 +212,14 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar {
 				},
 			},
 		},
+		{
+			Name: "POD_UID",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.uid",
+				},
+			},
+		},
 		{
 			Name:  "TS_STATE",
 			Value: "kube:$(POD_NAME)",

cmd/stunc/stunc.go

@@ -5,24 +5,40 @@
 package main
 
 import (
+	"flag"
 	"log"
 	"net"
 	"os"
 	"strconv"
+	"time"
 
 	"tailscale.com/net/stun"
 )
 
 func main() {
 	log.SetFlags(0)
-
-	if len(os.Args) < 2 || len(os.Args) > 3 {
-		log.Fatalf("usage: %s <hostname> [port]", os.Args[0])
-	}
-	host := os.Args[1]
+	var host string
 	port := "3478"
-	if len(os.Args) == 3 {
-		port = os.Args[2]
+
+	var readTimeout time.Duration
+	flag.DurationVar(&readTimeout, "timeout", 3*time.Second, "response wait timeout")
+
+	flag.Parse()
+
+	values := flag.Args()
+	if len(values) < 1 || len(values) > 2 {
+		log.Printf("usage: %s <hostname> [port]", os.Args[0])
+		flag.PrintDefaults()
+		os.Exit(1)
+	} else {
+		for i, value := range values {
+			switch i {
+			case 0:
+				host = value
+			case 1:
+				port = value
+			}
+		}
 	}
 	_, err := strconv.ParseUint(port, 10, 16)
 	if err != nil {
@@ -46,6 +62,10 @@ func main() {
 		log.Fatal(err)
 	}
 
+	err = c.SetReadDeadline(time.Now().Add(readTimeout))
+	if err != nil {
+		log.Fatal(err)
+	}
 	var buf [1024]byte
 	n, raddr, err := c.ReadFromUDPAddrPort(buf[:])
 	if err != nil {

cmd/stund/depaware.txt

@@ -8,7 +8,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
@@ -67,15 +66,16 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/types/logger                                   from tailscale.com/tsweb
         tailscale.com/types/opt                                      from tailscale.com/envknob+
         tailscale.com/types/ptr                                      from tailscale.com/tailcfg+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/tailcfg+
         tailscale.com/types/tkatype                                  from tailscale.com/tailcfg+
         tailscale.com/types/views                                    from tailscale.com/net/tsaddr+
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/tailcfg
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
-        tailscale.com/util/lineread                                  from tailscale.com/version/distro
+        tailscale.com/util/lineiter                                  from tailscale.com/version/distro
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/slicesx                                   from tailscale.com/tailcfg
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
         tailscale.com/version                                        from tailscale.com/envknob+
@@ -132,7 +132,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         crypto/tls                                                   from net/http+
         crypto/x509                                                  from crypto/tls
         crypto/x509/pkix                                             from crypto/x509
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -163,7 +162,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from math/big+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/tailscale/cli/advertise.go

@@ -0,0 +1,78 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/envknob"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+)
+
+var advertiseArgs struct {
+	services string // comma-separated list of services to advertise
+}
+
+// TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode
+// envknob is not needed.
+var advertiseCmd = &ffcli.Command{
+	Name:       "advertise",
+	ShortUsage: "tailscale advertise --services=<services>",
+	ShortHelp:  "Advertise this node as a destination for a service",
+	Exec:       runAdvertise,
+	FlagSet: (func() *flag.FlagSet {
+		fs := newFlagSet("advertise")
+		fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")")
+		return fs
+	})(),
+}
+
+func maybeAdvertiseCmd() []*ffcli.Command {
+	if !envknob.UseWIPCode() {
+		return nil
+	}
+	return []*ffcli.Command{advertiseCmd}
+}
+
+func runAdvertise(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return flag.ErrHelp
+	}
+
+	services, err := parseServiceNames(advertiseArgs.services)
+	if err != nil {
+		return err
+	}
+
+	_, err = localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
+		AdvertiseServicesSet: true,
+		Prefs: ipn.Prefs{
+			AdvertiseServices: services,
+		},
+	})
+	return err
+}
+
+// parseServiceNames takes a comma-separated list of service names
+// (eg. "svc:hello,svc:webserver,svc:catphotos"), splits them into
+// a list and validates each service name. If valid, it returns
+// the service names in a slice of strings.
+func parseServiceNames(servicesArg string) ([]string, error) {
+	var services []string
+	if servicesArg != "" {
+		services = strings.Split(servicesArg, ",")
+		for _, svc := range services {
+			err := tailcfg.CheckServiceName(svc)
+			if err != nil {
+				return nil, fmt.Errorf("service %q: %s", svc, err)
+			}
+		}
+	}
+	return services, nil
+}

cmd/tailscale/cli/cli.go

@@ -93,8 +93,13 @@ func Run(args []string) (err error) {
 
 	args = CleanUpArgs(args)
 
-	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
-		args = []string{"version"}
+	if len(args) == 1 {
+		switch args[0] {
+		case "-V", "--version":
+			args = []string{"version"}
+		case "help":
+			args = []string{"--help"}
+		}
 	}
 
 	var warnOnce sync.Once
@@ -177,7 +182,7 @@ For help on subcommands, add --help after: "tailscale status --help".
 This CLI is still under active development. Commands and flags will
 change in the future.
 `),
-		Subcommands: []*ffcli.Command{
+		Subcommands: append([]*ffcli.Command{
 			upCmd,
 			downCmd,
 			setCmd,
@@ -185,10 +190,12 @@ change in the future.
 			logoutCmd,
 			switchCmd,
 			configureCmd,
+			syspolicyCmd,
 			netcheckCmd,
 			ipCmd,
 			dnsCmd,
 			statusCmd,
+			metricsCmd,
 			pingCmd,
 			ncCmd,
 			sshCmd,
@@ -207,7 +214,7 @@ change in the future.
 			debugCmd,
 			driveCmd,
 			idTokenCmd,
-		},
+		}, maybeAdvertiseCmd()...),
 		FlagSet: rootfs,
 		Exec: func(ctx context.Context, args []string) error {
 			if len(args) > 0 {

cmd/tailscale/cli/cli_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"net/netip"
 	"reflect"
 	"strings"
@@ -946,6 +947,10 @@ func TestPrefFlagMapping(t *testing.T) {
 			// Handled by the tailscale share subcommand, we don't want a CLI
 			// flag for this.
 			continue
+		case "AdvertiseServices":
+			// Handled by the tailscale advertise subcommand, we don't want a
+			// CLI flag for this.
+			continue
 		case "InternalExitNodePrior":
 			// Used internally by LocalBackend as part of exit node usage toggling.
 			// No CLI flag for this.
@@ -1476,3 +1481,33 @@ func TestParseNLArgs(t *testing.T) {
 		})
 	}
 }
+
+func TestHelpAlias(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	tstest.Replace[io.Writer](t, &Stdout, &stdout)
+	tstest.Replace[io.Writer](t, &Stderr, &stderr)
+
+	gotExit0 := false
+	defer func() {
+		if !gotExit0 {
+			t.Error("expected os.Exit(0) to be called")
+			return
+		}
+		if !strings.Contains(stderr.String(), "SUBCOMMANDS") {
+			t.Errorf("expected help output to contain SUBCOMMANDS; got stderr=%q; stdout=%q", stderr.String(), stdout.String())
+		}
+	}()
+	defer func() {
+		if e := recover(); e != nil {
+			if strings.Contains(fmt.Sprint(e), "unexpected call to os.Exit(0)") {
+				gotExit0 = true
+			} else {
+				t.Errorf("unexpected panic: %v", e)
+			}
+		}
+	}()
+	err := Run([]string{"help"})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}

cmd/tailscale/cli/debug.go

@@ -36,6 +36,7 @@ import (
 	"tailscale.com/hostinfo"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/ipn"
+	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/paths"
@@ -174,6 +175,12 @@ var debugCmd = &ffcli.Command{
 			Exec:       localAPIAction("pick-new-derp"),
 			ShortHelp:  "Switch to some other random DERP home region for a short time",
 		},
+		{
+			Name:       "force-prefer-derp",
+			ShortUsage: "tailscale debug force-prefer-derp",
+			Exec:       forcePreferDERP,
+			ShortHelp:  "Prefer the given region ID if reachable (until restart, or 0 to clear)",
+		},
 		{
 			Name:       "force-netmap-update",
 			ShortUsage: "tailscale debug force-netmap-update",
@@ -213,6 +220,7 @@ var debugCmd = &ffcli.Command{
 				fs := newFlagSet("watch-ipn")
 				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
 				fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status")
+				fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags")
 				fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap")
 				fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever")
 				return fs
@@ -500,6 +508,7 @@ var watchIPNArgs struct {
 	netmap         bool
 	initial        bool
 	showPrivateKey bool
+	rateLimit      bool
 	count          int
 }
 
@@ -511,6 +520,9 @@ func runWatchIPN(ctx context.Context, args []string) error {
 	if !watchIPNArgs.showPrivateKey {
 		mask |= ipn.NotifyNoPrivateKeys
 	}
+	if watchIPNArgs.rateLimit {
+		mask |= ipn.NotifyRateLimit
+	}
 	watcher, err := localClient.WatchIPNBus(ctx, mask)
 	if err != nil {
 		return err
@@ -571,6 +583,25 @@ func runDERPMap(ctx context.Context, args []string) error {
 	return nil
 }
 
+func forcePreferDERP(ctx context.Context, args []string) error {
+	var n int
+	if len(args) != 1 {
+		return errors.New("expected exactly one integer argument")
+	}
+	n, err := strconv.Atoi(args[0])
+	if err != nil {
+		return fmt.Errorf("expected exactly one integer argument: %w", err)
+	}
+	b, err := json.Marshal(n)
+	if err != nil {
+		return fmt.Errorf("failed to marshal DERP region: %w", err)
+	}
+	if err := localClient.DebugActionBody(ctx, "force-prefer-derp", bytes.NewReader(b)); err != nil {
+		return fmt.Errorf("failed to force preferred DERP: %w", err)
+	}
+	return nil
+}
+
 func localAPIAction(action string) func(context.Context, []string) error {
 	return func(ctx context.Context, args []string) error {
 		if len(args) > 0 {
@@ -845,6 +876,11 @@ func runTS2021(ctx context.Context, args []string) error {
 		logf = log.Printf
 	}
 
+	netMon, err := netmon.New(logger.WithPrefix(logf, "netmon: "))
+	if err != nil {
+		return fmt.Errorf("creating netmon: %w", err)
+	}
+
 	noiseDialer := &controlhttp.Dialer{
 		Hostname:        ts2021Args.host,
 		HTTPPort:        "80",
@@ -854,6 +890,7 @@ func runTS2021(ctx context.Context, args []string) error {
 		ProtocolVersion: uint16(ts2021Args.version),
 		Dialer:          dialFunc,
 		Logf:            logf,
+		NetMon:          netMon,
 	}
 	const tries = 2
 	for i := range tries {

cmd/tailscale/cli/metrics.go

@@ -0,0 +1,88 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/atomicfile"
+)
+
+var metricsCmd = &ffcli.Command{
+	Name:      "metrics",
+	ShortHelp: "Show Tailscale metrics",
+	LongHelp: strings.TrimSpace(`
+
+The 'tailscale metrics' command shows Tailscale user-facing metrics (as opposed
+to internal metrics printed by 'tailscale debug metrics').
+
+For more information about Tailscale metrics, refer to
+https://tailscale.com/s/client-metrics
+
+`),
+	ShortUsage: "tailscale metrics <subcommand> [flags]",
+	UsageFunc:  usageFuncNoDefaultValues,
+	Exec:       runMetricsNoSubcommand,
+	Subcommands: []*ffcli.Command{
+		{
+			Name:       "print",
+			ShortUsage: "tailscale metrics print",
+			Exec:       runMetricsPrint,
+			ShortHelp:  "Prints current metric values in the Prometheus text exposition format",
+		},
+		{
+			Name:       "write",
+			ShortUsage: "tailscale metrics write <path>",
+			Exec:       runMetricsWrite,
+			ShortHelp:  "Writes metric values to a file",
+			LongHelp: strings.TrimSpace(`
+
+The 'tailscale metrics write' command writes metric values to a text file provided as its
+only argument. It's meant to be used alongside Prometheus node exporter, allowing Tailscale
+metrics to be consumed and exported by the textfile collector.
+
+As an example, to export Tailscale metrics on an Ubuntu system running node exporter, you
+can regularly run 'tailscale metrics write /var/lib/prometheus/node-exporter/tailscaled.prom'
+using cron or a systemd timer.
+
+	`),
+		},
+	},
+}
+
+// runMetricsNoSubcommand prints metric values if no subcommand is specified.
+func runMetricsNoSubcommand(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("tailscale metrics: unknown subcommand: %s", args[0])
+	}
+
+	return runMetricsPrint(ctx, args)
+}
+
+// runMetricsPrint prints metric values to stdout.
+func runMetricsPrint(ctx context.Context, args []string) error {
+	out, err := localClient.UserMetrics(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(out)
+	return nil
+}
+
+// runMetricsWrite writes metric values to a file.
+func runMetricsWrite(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: tailscale metrics write <path>")
+	}
+	path := args[0]
+	out, err := localClient.UserMetrics(ctx)
+	if err != nil {
+		return err
+	}
+	return atomicfile.WriteFile(path, out, 0644)
+}

cmd/tailscale/cli/netcheck.go

@@ -136,6 +136,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 
 	printf("\nReport:\n")
+	printf("\t* Time: %v\n", report.Now.Format(time.RFC3339Nano))
 	printf("\t* UDP: %v\n", report.UDP)
 	if report.GlobalV4.IsValid() {
 		printf("\t* IPv4: yes, %s\n", report.GlobalV4)

cmd/tailscale/cli/risks.go

@@ -17,11 +17,18 @@ import (
 )
 
 var (
-	riskTypes   []string
-	riskLoseSSH = registerRiskType("lose-ssh")
-	riskAll     = registerRiskType("all")
+	riskTypes           []string
+	riskLoseSSH         = registerRiskType("lose-ssh")
+	riskMacAppConnector = registerRiskType("mac-app-connector")
+	riskAll             = registerRiskType("all")
 )
 
+const riskMacAppConnectorMessage = `
+You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. 
+
+Do not use a macOS app connector for any mission-critical purposes. For the best experience, Linux is the only recommended platform for app connectors.
+`
+
 func registerRiskType(riskType string) string {
 	riskTypes = append(riskTypes, riskType)
 	return riskType

cmd/tailscale/cli/set.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/netip"
 	"os/exec"
+	"runtime"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
@@ -203,6 +204,12 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 		}
 	}
 
+	if runtime.GOOS == "darwin" && maskedPrefs.AppConnector.Advertise {
+		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, setArgs.acceptedRisks); err != nil {
+			return err
+		}
+	}
+
 	if maskedPrefs.RunSSHSet {
 		wantSSH, haveSSH := maskedPrefs.RunSSH, curPrefs.RunSSH
 		if err := presentSSHToggleRisk(wantSSH, haveSSH, setArgs.acceptedRisks); err != nil {

cmd/tailscale/cli/status.go

@@ -231,6 +231,12 @@ func runStatus(ctx context.Context, args []string) error {
 		outln()
 		printf("# To see the full list of exit nodes, including location-based exit nodes, run `tailscale exit-node list`  \n")
 	}
+
+	outln()
+	printf("# Tailnet:\n")
+	printf("#     - Name: %s\n", st.CurrentTailnet.Name)
+	printf("#     - MagicDNS Suffix: %s\n", st.CurrentTailnet.MagicDNSSuffix)
+
 	if len(st.Health) > 0 {
 		outln()
 		printHealth()

cmd/tailscale/cli/syspolicy.go

@@ -0,0 +1,110 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"slices"
+	"text/tabwriter"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/util/syspolicy/setting"
+)
+
+var syspolicyArgs struct {
+	json bool // JSON output mode
+}
+
+var syspolicyCmd = &ffcli.Command{
+	Name:       "syspolicy",
+	ShortHelp:  "Diagnose the MDM and system policy configuration",
+	LongHelp:   "The 'tailscale syspolicy' command provides tools for diagnosing the MDM and system policy configuration.",
+	ShortUsage: "tailscale syspolicy <subcommand>",
+	UsageFunc:  usageFuncNoDefaultValues,
+	Subcommands: []*ffcli.Command{
+		{
+			Name:       "list",
+			ShortUsage: "tailscale syspolicy list",
+			Exec:       runSysPolicyList,
+			ShortHelp:  "Prints effective policy settings",
+			LongHelp:   "The 'tailscale syspolicy list' subcommand displays the effective policy settings and their sources (e.g., MDM or environment variables).",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("syspolicy list")
+				fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format")
+				return fs
+			})(),
+		},
+		{
+			Name:       "reload",
+			ShortUsage: "tailscale syspolicy reload",
+			Exec:       runSysPolicyReload,
+			ShortHelp:  "Forces a reload of policy settings, even if no changes are detected, and prints the result",
+			LongHelp:   "The 'tailscale syspolicy reload' subcommand forces a reload of policy settings, even if no changes are detected, and prints the result.",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("syspolicy reload")
+				fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format")
+				return fs
+			})(),
+		},
+	},
+}
+
+func runSysPolicyList(ctx context.Context, args []string) error {
+	policy, err := localClient.GetEffectivePolicy(ctx, setting.DefaultScope())
+	if err != nil {
+		return err
+	}
+	printPolicySettings(policy)
+	return nil
+
+}
+
+func runSysPolicyReload(ctx context.Context, args []string) error {
+	policy, err := localClient.ReloadEffectivePolicy(ctx, setting.DefaultScope())
+	if err != nil {
+		return err
+	}
+	printPolicySettings(policy)
+	return nil
+}
+
+func printPolicySettings(policy *setting.Snapshot) {
+	if syspolicyArgs.json {
+		json, err := json.MarshalIndent(policy, "", "\t")
+		if err != nil {
+			errf("syspolicy marshalling error: %v", err)
+		} else {
+			outln(string(json))
+		}
+		return
+	}
+	if policy.Len() == 0 {
+		outln("No policy settings")
+		return
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "Name\tOrigin\tValue\tError")
+	fmt.Fprintln(w, "----\t------\t-----\t-----")
+	for _, k := range slices.Sorted(policy.Keys()) {
+		setting, _ := policy.GetSetting(k)
+		var origin string
+		if o := setting.Origin(); o != nil {
+			origin = o.String()
+		}
+		if err := setting.Error(); err != nil {
+			fmt.Fprintf(w, "%s\t%s\t\t{%v}\n", k, origin, err)
+		} else {
+			fmt.Fprintf(w, "%s\t%s\t%v\t\n", k, origin, setting.Value())
+		}
+	}
+	w.Flush()
+
+	fmt.Println()
+	return
+}

cmd/tailscale/cli/up.go

@@ -164,6 +164,9 @@ func defaultNetfilterMode() string {
 	return "on"
 }
 
+// upArgsT is the type of upArgs, the argument struct for `tailscale up`.
+// As of 2024-10-08, upArgsT is frozen and no new arguments should be
+// added to it. Add new arguments to setArgsT instead.
 type upArgsT struct {
 	qr                     bool
 	reset                  bool
@@ -376,6 +379,12 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, err
 	}
 
+	if runtime.GOOS == "darwin" && env.upArgs.advertiseConnector {
+		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, env.upArgs.acceptedRisks); err != nil {
+			return false, nil, err
+		}
+	}
+
 	if env.upArgs.forceReauth && isSSHOverTailscale() {
 		if err := presentRiskToUser(riskLoseSSH, `You are connected over Tailscale; this action will result in your SSH session disconnecting.`, env.upArgs.acceptedRisks); err != nil {
 			return false, nil, err
@@ -1148,7 +1157,6 @@ func resolveAuthKey(ctx context.Context, v, tags string) (string, error) {
 		ClientID:     "some-client-id", // ignored
 		ClientSecret: clientSecret,
 		TokenURL:     baseURL + "/api/v2/oauth/token",
-		Scopes:       []string{"device"},
 	}
 
 	tsClient := tailscale.NewClient("-", nil)

cmd/tailscale/depaware.txt

@@ -5,10 +5,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
@@ -86,12 +82,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/cmd/tailscale/cli/ffcomplete/internal          from tailscale.com/cmd/tailscale/cli/ffcomplete
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlhttp                            from tailscale.com/cmd/tailscale/cli
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/envknob/featureknob                            from tailscale.com/client/web
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/health/healthmsg                               from tailscale.com/cmd/tailscale/cli
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -120,9 +118,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
         tailscale.com/syncs                                          from tailscale.com/cmd/tailscale/cli+
@@ -146,6 +144,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/types/key+
         tailscale.com/types/views                                    from tailscale.com/tailcfg+
@@ -153,14 +152,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck+
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dnscache+
         tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy+
-        tailscale.com/util/ctxkey                                    from tailscale.com/types/logger
+        tailscale.com/util/ctxkey                                    from tailscale.com/types/logger+
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/groupmember                               from tailscale.com/client/web
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlhttp+
@@ -172,14 +171,18 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
-        tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
@@ -318,7 +321,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
-        runtime/debug                                                from github.com/coder/websocket/internal/xsync+
+        runtime/debug                                                from tailscale.com+
         slices                                                       from tailscale.com/client/web+
         sort                                                         from compress/flate+
         strconv                                                      from archive/tar+

cmd/tailscaled/depaware.txt

@@ -79,10 +79,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -221,7 +217,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
         gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack+
         gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
      💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
@@ -249,6 +245,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/cmd/tailscaled+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/tailscaled+
@@ -263,6 +260,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/drive/driveimpl/dirfs                          from tailscale.com/drive/driveimpl+
         tailscale.com/drive/driveimpl/shared                         from tailscale.com/drive/driveimpl+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
+        tailscale.com/envknob/featureknob                            from tailscale.com/client/web+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
@@ -321,11 +319,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+
    L    tailscale.com/net/tcpinfo                                    from tailscale.com/derp
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
+        tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
         tailscale.com/net/tsdial                                     from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -362,6 +360,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/control/controlclient+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/tka+
         tailscale.com/types/views                                    from tailscale.com/ipn/ipnlocal+
@@ -379,7 +378,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/control/controlclient+
         tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
@@ -399,8 +398,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
@@ -410,7 +412,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+

cmd/tailscaled/deps_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"testing"
+
+	"tailscale.com/tstest/deptest"
+)
+
+func TestOmitSSH(t *testing.T) {
+	const msg = "unexpected with ts_omit_ssh"
+	deptest.DepChecker{
+		GOOS:   "linux",
+		GOARCH: "amd64",
+		Tags:   "ts_omit_ssh",
+		BadDeps: map[string]string{
+			"tailscale.com/ssh/tailssh":            msg,
+			"golang.org/x/crypto/ssh":              msg,
+			"tailscale.com/sessionrecording":       msg,
+			"github.com/anmitsu/go-shlex":          msg,
+			"github.com/creack/pty":                msg,
+			"github.com/kr/fs":                     msg,
+			"github.com/pkg/sftp":                  msg,
+			"github.com/u-root/u-root/pkg/termios": msg,
+			"tempfork/gliderlabs/ssh":              msg,
+		},
+	}.Check(t)
+}

cmd/tailscaled/ssh.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build linux || darwin || freebsd || openbsd
+//go:build (linux || darwin || freebsd || openbsd) && !ts_omit_ssh
 
 package main
 

cmd/tailscaled/tailscaled.go

@@ -788,7 +788,6 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 }
 
 func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
-	tfs, _ := sys.DriveForLocal.GetOK()
 	ret, err := netstack.Create(logf,
 		sys.Tun.Get(),
 		sys.Engine.Get(),
@@ -796,7 +795,6 @@ func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
 		sys.Dialer.Get(),
 		sys.DNSManager.Get(),
 		sys.ProxyMapper(),
-		tfs,
 	)
 	if err != nil {
 		return nil, err

cmd/tailscaled/tailscaled_windows.go

@@ -134,14 +134,13 @@ func runWindowsService(pol *logpolicy.Policy) error {
 		logger.Logf(log.Printf).JSON(1, "SupportInfo", osdiag.SupportInfo(osdiag.LogSupportInfoReasonStartup))
 	}()
 
-	if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions {
-		syslog, err := eventlog.Open(serviceName)
-		if err == nil {
-			syslogf = func(format string, args ...any) {
+	if syslog, err := eventlog.Open(serviceName); err == nil {
+		syslogf = func(format string, args ...any) {
+			if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions {
 				syslog.Info(0, fmt.Sprintf(format, args...))
 			}
-			defer syslog.Close()
 		}
+		defer syslog.Close()
 	}
 
 	syslogf("Service entering svc.Run")
@@ -160,10 +159,7 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	changes <- svc.Status{State: svc.StartPending}
 	syslogf("Service start pending")
 
-	svcAccepts := svc.AcceptStop
-	if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock {
-		svcAccepts |= svc.AcceptSessionChange
-	}
+	svcAccepts := svc.AcceptStop | svc.AcceptSessionChange
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -371,13 +367,15 @@ func handleSessionChange(chgRequest svc.ChangeRequest) {
 		return
 	}
 
-	log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.")
-	go func() {
-		err := dns.Flush()
-		if err != nil {
-			log.Printf("Error flushing DNS on session unlock: %v", err)
-		}
-	}()
+	if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock {
+		log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.")
+		go func() {
+			err := dns.Flush()
+			if err != nil {
+				log.Printf("Error flushing DNS on session unlock: %v", err)
+			}
+		}()
+	}
 }
 
 var (

cmd/testwrapper/testwrapper.go

@@ -42,6 +42,7 @@ type testAttempt struct {
 	testName      string // "TestFoo"
 	outcome       string // "pass", "fail", "skip"
 	logs          bytes.Buffer
+	start, end    time.Time
 	isMarkedFlaky bool   // set if the test is marked as flaky
 	issueURL      string // set if the test is marked as flaky
 
@@ -132,11 +133,17 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 		}
 		pkg := goOutput.Package
 		pkgTests := resultMap[pkg]
+		if pkgTests == nil {
+			pkgTests = make(map[string]*testAttempt)
+			resultMap[pkg] = pkgTests
+		}
 		if goOutput.Test == "" {
 			switch goOutput.Action {
+			case "start":
+				pkgTests[""] = &testAttempt{start: goOutput.Time}
 			case "fail", "pass", "skip":
 				for _, test := range pkgTests {
-					if test.outcome == "" {
+					if test.testName != "" && test.outcome == "" {
 						test.outcome = "fail"
 						ch <- test
 					}
@@ -144,15 +151,13 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 				ch <- &testAttempt{
 					pkg:         goOutput.Package,
 					outcome:     goOutput.Action,
+					start:       pkgTests[""].start,
+					end:         goOutput.Time,
 					pkgFinished: true,
 				}
 			}
 			continue
 		}
-		if pkgTests == nil {
-			pkgTests = make(map[string]*testAttempt)
-			resultMap[pkg] = pkgTests
-		}
 		testName := goOutput.Test
 		if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
 			testName = test
@@ -168,8 +173,10 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 			pkgTests[testName] = &testAttempt{
 				pkg:      pkg,
 				testName: testName,
+				start:    goOutput.Time,
 			}
 		case "skip", "pass", "fail":
+			pkgTests[testName].end = goOutput.Time
 			pkgTests[testName].outcome = goOutput.Action
 			ch <- pkgTests[testName]
 		case "output":
@@ -213,7 +220,7 @@ func main() {
 		firstRun.tests = append(firstRun.tests, &packageTests{Pattern: pkg})
 	}
 	toRun := []*nextRun{firstRun}
-	printPkgOutcome := func(pkg, outcome string, attempt int) {
+	printPkgOutcome := func(pkg, outcome string, attempt int, runtime time.Duration) {
 		if outcome == "skip" {
 			fmt.Printf("?\t%s [skipped/no tests] \n", pkg)
 			return
@@ -225,10 +232,10 @@ func main() {
 			outcome = "FAIL"
 		}
 		if attempt > 1 {
-			fmt.Printf("%s\t%s [attempt=%d]\n", outcome, pkg, attempt)
+			fmt.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt)
 			return
 		}
-		fmt.Printf("%s\t%s\n", outcome, pkg)
+		fmt.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds())
 	}
 
 	// Check for -coverprofile argument and filter it out
@@ -307,7 +314,7 @@ func main() {
 						// when a package times out.
 						failed = true
 					}
-					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt)
+					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt, tr.end.Sub(tr.start))
 					continue
 				}
 				if testingVerbose || tr.outcome == "fail" {

cmd/testwrapper/testwrapper_test.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sync"
 	"testing"
 )
@@ -76,7 +77,10 @@ func TestFlakeRun(t *testing.T) {
 		t.Fatalf("go run . %s: %s with output:\n%s", testfile, err, out)
 	}
 
-	want := []byte("ok\t" + testfile + " [attempt=2]")
+	// Replace the unpredictable timestamp with "0.00s".
+	out = regexp.MustCompile(`\t\d+\.\d\d\ds\t`).ReplaceAll(out, []byte("\t0.00s\t"))
+
+	want := []byte("ok\t" + testfile + "\t0.00s\t[attempt=2]")
 	if !bytes.Contains(out, want) {
 		t.Fatalf("wanted output containing %q but got:\n%s", want, out)
 	}

cmd/tsconnect/common.go

@@ -150,6 +150,7 @@ func runEsbuildServe(buildOptions esbuild.BuildOptions) {
 		log.Fatalf("Cannot start esbuild server: %v", err)
 	}
 	log.Printf("Listening on http://%s:%d\n", result.Host, result.Port)
+	select {}
 }
 
 func runEsbuild(buildOptions esbuild.BuildOptions) esbuild.BuildResult {

cmd/tsconnect/wasm/wasm_js.go

@@ -108,13 +108,14 @@ func newIPN(jsConfig js.Value) map[string]any {
 		SetSubsystem:  sys.Set,
 		ControlKnobs:  sys.ControlKnobs(),
 		HealthTracker: sys.HealthTracker(),
+		Metrics:       sys.UserMetricsRegistry(),
 	})
 	if err != nil {
 		log.Fatal(err)
 	}
 	sys.Set(eng)
 
-	ns, err := netstack.Create(logf, sys.Tun.Get(), eng, sys.MagicSock.Get(), dialer, sys.DNSManager.Get(), sys.ProxyMapper(), nil)
+	ns, err := netstack.Create(logf, sys.Tun.Get(), eng, sys.MagicSock.Get(), dialer, sys.DNSManager.Get(), sys.ProxyMapper())
 	if err != nil {
 		log.Fatalf("netstack.Create: %v", err)
 	}
@@ -128,6 +129,9 @@ func newIPN(jsConfig js.Value) map[string]any {
 	dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
 		return ns.DialContextTCP(ctx, dst)
 	}
+	dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+		return ns.DialContextUDP(ctx, dst)
+	}
 	sys.NetstackRouter.Set(true)
 	sys.Tun.Get().Start()
 
@@ -268,8 +272,8 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 						name = p.Hostinfo().Hostname()
 					}
 					addrs := make([]string, p.Addresses().Len())
-					for i := range p.Addresses().Len() {
-						addrs[i] = p.Addresses().At(i).Addr().String()
+					for i, ap := range p.Addresses().All() {
+						addrs[i] = ap.Addr().String()
 					}
 					return jsNetMapPeerNode{
 						jsNetMapNode: jsNetMapNode{
@@ -585,8 +589,8 @@ func mapSlice[T any, M any](a []T, f func(T) M) []M {
 
 func mapSliceView[T any, M any](a views.Slice[T], f func(T) M) []M {
 	n := make([]M, a.Len())
-	for i := range a.Len() {
-		n[i] = f(a.At(i))
+	for i, v := range a.All() {
+		n[i] = f(v)
 	}
 	return n
 }

cmd/viewer/viewer.go

@@ -258,6 +258,7 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 				writeTemplate("unsupportedField")
 				continue
 			}
+			it.Import("tailscale.com/types/views")
 			args.MapKeyType = it.QualifiedName(key)
 			mElem := m.Elem()
 			var template string

cmd/viewer/viewer_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"testing"
+
+	"tailscale.com/util/codegen"
+)
+
+func TestViewerImports(t *testing.T) {
+	tests := []struct {
+		name        string
+		content     string
+		typeNames   []string
+		wantImports []string
+	}{
+		{
+			name:        "Map",
+			content:     `type Test struct { Map map[string]int }`,
+			typeNames:   []string{"Test"},
+			wantImports: []string{"tailscale.com/types/views"},
+		},
+		{
+			name:        "Slice",
+			content:     `type Test struct { Slice []int }`,
+			typeNames:   []string{"Test"},
+			wantImports: []string{"tailscale.com/types/views"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fset := token.NewFileSet()
+			f, err := parser.ParseFile(fset, "test.go", "package test\n\n"+tt.content, 0)
+			if err != nil {
+				fmt.Println("Error parsing:", err)
+				return
+			}
+
+			info := &types.Info{
+				Types: make(map[ast.Expr]types.TypeAndValue),
+			}
+
+			conf := types.Config{}
+			pkg, err := conf.Check("", fset, []*ast.File{f}, info)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var output bytes.Buffer
+			tracker := codegen.NewImportTracker(pkg)
+			for i := range tt.typeNames {
+				typeName, ok := pkg.Scope().Lookup(tt.typeNames[i]).(*types.TypeName)
+				if !ok {
+					t.Fatalf("type %q does not exist", tt.typeNames[i])
+				}
+				namedType, ok := typeName.Type().(*types.Named)
+				if !ok {
+					t.Fatalf("%q is not a named type", tt.typeNames[i])
+				}
+				genView(&output, tracker, namedType, pkg)
+			}
+
+			for _, pkgName := range tt.wantImports {
+				if !tracker.Has(pkgName) {
+					t.Errorf("missing import %q", pkgName)
+				}
+			}
+		})
+	}
+}

control/controlclient/noise.go

@@ -17,7 +17,6 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlhttp"
-	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/dnscache"
@@ -30,7 +29,6 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/multierr"
 	"tailscale.com/util/singleflight"
-	"tailscale.com/util/testenv"
 )
 
 // NoiseClient provides a http.Client to connect to tailcontrol over
@@ -107,11 +105,6 @@ type NoiseOpts struct {
 	DialPlan func() *tailcfg.ControlDialPlan
 }
 
-// controlIsPlaintext is whether we should assume that the controlplane is only accessible
-// over plaintext HTTP (as the first hop, before the ts2021 encryption begins).
-// This is used by some tests which don't have a real TLS certificate.
-var controlIsPlaintext = envknob.RegisterBool("TS_CONTROL_IS_PLAINTEXT_HTTP")
-
 // NewNoiseClient returns a new noiseClient for the provided server and machine key.
 // serverURL is of the form https://<host>:<port> (no trailing slash).
 //
@@ -129,7 +122,7 @@ func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) {
 		if u.Scheme == "http" {
 			httpPort = port
 			httpsPort = "443"
-			if (testenv.InTest() || controlIsPlaintext()) && (u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost") {
+			if u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost" {
 				httpsPort = ""
 			}
 		} else {

control/controlclient/noise_test.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	"golang.org/x/net/http2"
-	"tailscale.com/control/controlhttp"
+	"tailscale.com/control/controlhttp/controlhttpserver"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
@@ -201,7 +201,7 @@ func (up *Upgrader) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return nil
 	}
 
-	cbConn, err := controlhttp.AcceptHTTP(r.Context(), w, r, up.noiseKeyPriv, earlyWriteFn)
+	cbConn, err := controlhttpserver.AcceptHTTP(r.Context(), w, r, up.noiseKeyPriv, earlyWriteFn)
 	if err != nil {
 		up.logf("controlhttp: Accept: %v", err)
 		return

control/controlclient/sign_supported.go

@@ -13,7 +13,6 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"sync"
 	"time"
 
 	"github.com/tailscale/certstore"
@@ -22,11 +21,6 @@ import (
 	"tailscale.com/util/syspolicy"
 )
 
-var getMachineCertificateSubjectOnce struct {
-	sync.Once
-	v string // Subject of machine certificate to search for
-}
-
 // getMachineCertificateSubject returns the exact name of a Subject that needs
 // to be present in an identity's certificate chain to sign a RegisterRequest,
 // formatted as per pkix.Name.String(). The Subject may be that of the identity
@@ -37,11 +31,8 @@ var getMachineCertificateSubjectOnce struct {
 //
 // Example: "CN=Tailscale Inc Test Root CA,OU=Tailscale Inc Test Certificate Authority,O=Tailscale Inc,ST=ON,C=CA"
 func getMachineCertificateSubject() string {
-	getMachineCertificateSubjectOnce.Do(func() {
-		getMachineCertificateSubjectOnce.v, _ = syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
-	})
-
-	return getMachineCertificateSubjectOnce.v
+	machineCertSubject, _ := syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
+	return machineCertSubject
 }
 
 var (

control/controlhttp/client.go

@@ -38,6 +38,7 @@ import (
 	"time"
 
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
@@ -571,9 +572,9 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad
 		Method: "POST",
 		URL:    u,
 		Header: http.Header{
-			"Upgrade":           []string{upgradeHeaderValue},
-			"Connection":        []string{"upgrade"},
-			handshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
+			"Upgrade":                             []string{controlhttpcommon.UpgradeHeaderValue},
+			"Connection":                          []string{"upgrade"},
+			controlhttpcommon.HandshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
 		},
 	}
 	req = req.WithContext(ctx)
@@ -597,7 +598,7 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad
 		return nil, fmt.Errorf("httptrace didn't provide a connection")
 	}
 
-	if next := resp.Header.Get("Upgrade"); next != upgradeHeaderValue {
+	if next := resp.Header.Get("Upgrade"); next != controlhttpcommon.UpgradeHeaderValue {
 		resp.Body.Close()
 		return nil, fmt.Errorf("server switched to unexpected protocol %q", next)
 	}

control/controlhttp/client_js.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/net/wsconn"
 )
 
@@ -42,11 +43,11 @@ func (d *Dialer) Dial(ctx context.Context) (*ClientConn, error) {
 		// Can't set HTTP headers on the websocket request, so we have to to send
 		// the handshake via an HTTP header.
 		RawQuery: url.Values{
-			handshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
+			controlhttpcommon.HandshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
 		}.Encode(),
 	}
 	wsConn, _, err := websocket.Dial(ctx, wsURL.String(), &websocket.DialOptions{
-		Subprotocols: []string{upgradeHeaderValue},
+		Subprotocols: []string{controlhttpcommon.UpgradeHeaderValue},
 	})
 	if err != nil {
 		return nil, err

control/controlhttp/constants.go

@@ -18,15 +18,6 @@ import (
 )
 
 const (
-	// upgradeHeader is the value of the Upgrade HTTP header used to
-	// indicate the Tailscale control protocol.
-	upgradeHeaderValue = "tailscale-control-protocol"
-
-	// handshakeHeaderName is the HTTP request header that can
-	// optionally contain base64-encoded initial handshake
-	// payload, to save an RTT.
-	handshakeHeaderName = "X-Tailscale-Handshake"
-
 	// serverUpgradePath is where the server-side HTTP handler to
 	// to do the protocol switch is located.
 	serverUpgradePath = "/ts2021"
@@ -85,6 +76,8 @@ type Dialer struct {
 	// dropped.
 	Logf logger.Logf
 
+	// NetMon is the [netmon.Monitor] to use for this Dialer. It must be
+	// non-nil.
 	NetMon *netmon.Monitor
 
 	// HealthTracker, if non-nil, is the health tracker to use.

control/controlhttp/controlhttpcommon/controlhttpcommon.go

@@ -0,0 +1,15 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package controlhttpcommon contains common constants for used
+// by the controlhttp client and controlhttpserver packages.
+package controlhttpcommon
+
+// UpgradeHeader is the value of the Upgrade HTTP header used to
+// indicate the Tailscale control protocol.
+const UpgradeHeaderValue = "tailscale-control-protocol"
+
+// handshakeHeaderName is the HTTP request header that can
+// optionally contain base64-encoded initial handshake
+// payload, to save an RTT.
+const HandshakeHeaderName = "X-Tailscale-Handshake"

control/controlhttp/controlhttpserver/controlhttpserver.go

@@ -3,7 +3,8 @@
 
 //go:build !ios
 
-package controlhttp
+// Package controlhttpserver contains the HTTP server side of the ts2021 control protocol.
+package controlhttpserver
 
 import (
 	"context"
@@ -18,6 +19,7 @@ import (
 
 	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/wsconn"
 	"tailscale.com/types/key"
@@ -45,12 +47,12 @@ func acceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, pri
 	if next == "websocket" {
 		return acceptWebsocket(ctx, w, r, private)
 	}
-	if next != upgradeHeaderValue {
+	if next != controlhttpcommon.UpgradeHeaderValue {
 		http.Error(w, "unknown next protocol", http.StatusBadRequest)
 		return nil, fmt.Errorf("client requested unhandled next protocol %q", next)
 	}
 
-	initB64 := r.Header.Get(handshakeHeaderName)
+	initB64 := r.Header.Get(controlhttpcommon.HandshakeHeaderName)
 	if initB64 == "" {
 		http.Error(w, "missing Tailscale handshake header", http.StatusBadRequest)
 		return nil, errors.New("no tailscale handshake header in HTTP request")
@@ -67,7 +69,7 @@ func acceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, pri
 		return nil, errors.New("can't hijack client connection")
 	}
 
-	w.Header().Set("Upgrade", upgradeHeaderValue)
+	w.Header().Set("Upgrade", controlhttpcommon.UpgradeHeaderValue)
 	w.Header().Set("Connection", "upgrade")
 	w.WriteHeader(http.StatusSwitchingProtocols)
 
@@ -117,7 +119,7 @@ func acceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, pri
 // speak HTTP) to a Tailscale control protocol base transport connection.
 func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request, private key.MachinePrivate) (*controlbase.Conn, error) {
 	c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
-		Subprotocols:   []string{upgradeHeaderValue},
+		Subprotocols:   []string{controlhttpcommon.UpgradeHeaderValue},
 		OriginPatterns: []string{"*"},
 		// Disable compression because we transmit Noise messages that are not
 		// compressible.
@@ -129,7 +131,7 @@ func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request
 	if err != nil {
 		return nil, fmt.Errorf("Could not accept WebSocket connection %v", err)
 	}
-	if c.Subprotocol() != upgradeHeaderValue {
+	if c.Subprotocol() != controlhttpcommon.UpgradeHeaderValue {
 		c.Close(websocket.StatusPolicyViolation, "client must speak the control subprotocol")
 		return nil, fmt.Errorf("Unexpected subprotocol %q", c.Subprotocol())
 	}
@@ -137,7 +139,7 @@ func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request
 		c.Close(websocket.StatusPolicyViolation, "Could not parse parameters")
 		return nil, fmt.Errorf("parse query parameters: %v", err)
 	}
-	initB64 := r.Form.Get(handshakeHeaderName)
+	initB64 := r.Form.Get(controlhttpcommon.HandshakeHeaderName)
 	if initB64 == "" {
 		c.Close(websocket.StatusPolicyViolation, "missing Tailscale handshake parameter")
 		return nil, errors.New("no tailscale handshake parameter in HTTP request")

control/controlhttp/http_test.go

@@ -23,12 +23,16 @@ import (
 	"time"
 
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
+	"tailscale.com/control/controlhttp/controlhttpserver"
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/socks5"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
+	"tailscale.com/tstest/deptest"
 	"tailscale.com/tstime"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -158,7 +162,7 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 				return err
 			}
 		}
-		conn, err := AcceptHTTP(context.Background(), w, r, server, earlyWriteFn)
+		conn, err := controlhttpserver.AcceptHTTP(context.Background(), w, r, server, earlyWriteFn)
 		if err != nil {
 			log.Print(err)
 		}
@@ -225,6 +229,7 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 		omitCertErrorLogging: true,
 		testFallbackDelay:    fallbackDelay,
 		Clock:                clock,
+		HealthTracker:        new(health.Tracker),
 	}
 
 	if param.httpInDial {
@@ -529,7 +534,7 @@ EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA==
 
 func brokenMITMHandler(clock tstime.Clock) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Upgrade", upgradeHeaderValue)
+		w.Header().Set("Upgrade", controlhttpcommon.UpgradeHeaderValue)
 		w.Header().Set("Connection", "upgrade")
 		w.WriteHeader(http.StatusSwitchingProtocols)
 		w.(http.Flusher).Flush()
@@ -574,7 +579,7 @@ func TestDialPlan(t *testing.T) {
 			close(done)
 		})
 		var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			conn, err := AcceptHTTP(context.Background(), w, r, server, nil)
+			conn, err := controlhttpserver.AcceptHTTP(context.Background(), w, r, server, nil)
 			if err != nil {
 				log.Print(err)
 			} else {
@@ -726,6 +731,7 @@ func TestDialPlan(t *testing.T) {
 				omitCertErrorLogging: true,
 				testFallbackDelay:    50 * time.Millisecond,
 				Clock:                clock,
+				HealthTracker:        new(health.Tracker),
 			}
 
 			conn, err := a.dial(ctx)
@@ -816,3 +822,14 @@ func (c *closeTrackConn) Close() error {
 	c.d.noteClose(c)
 	return c.Conn.Close()
 }
+
+func TestDeps(t *testing.T) {
+	deptest.DepChecker{
+		GOOS:   "darwin",
+		GOARCH: "arm64",
+		BadDeps: map[string]string{
+			// Only the controlhttpserver needs WebSockets...
+			"github.com/coder/websocket": "controlhttp client shouldn't need websockets",
+		},
+	}.Check(t)
+}

derp/derp.go

@@ -147,6 +147,7 @@ const (
 	PeerPresentIsRegular  = 1 << 0
 	PeerPresentIsMeshPeer = 1 << 1
 	PeerPresentIsProber   = 1 << 2
+	PeerPresentNotIdeal   = 1 << 3 // client said derp server is not its Region.Nodes[0] ideal node
 )
 
 var bin = binary.BigEndian

derp/derp_server.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/http"
 	"net/netip"
+	"os"
 	"os/exec"
 	"runtime"
 	"strconv"
@@ -46,6 +47,7 @@ import (
 	"tailscale.com/tstime/rate"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/ctxkey"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
 	"tailscale.com/util/slicesx"
@@ -56,6 +58,16 @@ import (
 // verbosely log whenever DERP drops a packet.
 var verboseDropKeys = map[key.NodePublic]bool{}
 
+// IdealNodeHeader is the HTTP request header sent on DERP HTTP client requests
+// to indicate that they're connecting to their ideal (Region.Nodes[0]) node.
+// The HTTP header value is the name of the node they wish they were connected
+// to. This is an optional header.
+const IdealNodeHeader = "Ideal-Node"
+
+// IdealNodeContextKey is the context key used to pass the IdealNodeHeader value
+// from the HTTP handler to the DERP server's Accept method.
+var IdealNodeContextKey = ctxkey.New[string]("ideal-node", "")
+
 func init() {
 	keys := envknob.String("TS_DEBUG_VERBOSE_DROPS")
 	if keys == "" {
@@ -72,10 +84,19 @@ func init() {
 }
 
 const (
-	perClientSendQueueDepth = 32 // packets buffered for sending
-	writeTimeout            = 2 * time.Second
+	defaultPerClientSendQueueDepth = 32 // default packets buffered for sending
+	writeTimeout                   = 2 * time.Second
+	privilegedWriteTimeout         = 30 * time.Second // for clients with the mesh key
 )
 
+func getPerClientSendQueueDepth() int {
+	if v, ok := envknob.LookupInt("TS_DEBUG_DERP_PER_CLIENT_SEND_QUEUE_DEPTH"); ok {
+		return v
+	}
+
+	return defaultPerClientSendQueueDepth
+}
+
 // dupPolicy is a temporary (2021-08-30) mechanism to change the policy
 // of how duplicate connection for the same key are handled.
 type dupPolicy int8
@@ -131,6 +152,7 @@ type Server struct {
 	sentPong                     expvar.Int // number of pong frames enqueued to client
 	accepts                      expvar.Int
 	curClients                   expvar.Int
+	curClientsNotIdeal           expvar.Int
 	curHomeClients               expvar.Int // ones with preferred
 	dupClientKeys                expvar.Int // current number of public keys we have 2+ connections for
 	dupClientConns               expvar.Int // current number of connections sharing a public key
@@ -141,6 +163,7 @@ type Server struct {
 	multiForwarderCreated        expvar.Int
 	multiForwarderDeleted        expvar.Int
 	removePktForwardOther        expvar.Int
+	sclientWriteTimeouts         expvar.Int
 	avgQueueDuration             *uint64          // In milliseconds; accessed atomically
 	tcpRtt                       metrics.LabelMap // histogram
 	meshUpdateBatchSize          *metrics.Histogram
@@ -175,6 +198,9 @@ type Server struct {
 	// maps from netip.AddrPort to a client's public key
 	keyOfAddr map[netip.AddrPort]key.NodePublic
 
+	// Sets the client send queue depth for the server.
+	perClientSendQueueDepth int
+
 	clock tstime.Clock
 }
 
@@ -362,6 +388,8 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 
 	s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
 	s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
+
+	s.perClientSendQueueDepth = getPerClientSendQueueDepth()
 	return s
 }
 
@@ -600,6 +628,9 @@ func (s *Server) registerClient(c *sclient) {
 	}
 	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
+	if c.isNotIdealConn {
+		s.curClientsNotIdeal.Add(1)
+	}
 	s.broadcastPeerStateChangeLocked(c.key, c.remoteIPPort, c.presentFlags(), true)
 }
 
@@ -690,6 +721,9 @@ func (s *Server) unregisterClient(c *sclient) {
 	if c.preferred {
 		s.curHomeClients.Add(-1)
 	}
+	if c.isNotIdealConn {
+		s.curClientsNotIdeal.Add(-1)
+	}
 }
 
 // addPeerGoneFromRegionWatcher adds a function to be called when peer is gone
@@ -806,8 +840,8 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 		return fmt.Errorf("receive client key: %v", err)
 	}
 
-	clientAP, _ := netip.ParseAddrPort(remoteAddr)
-	if err := s.verifyClient(ctx, clientKey, clientInfo, clientAP.Addr()); err != nil {
+	remoteIPPort, _ := netip.ParseAddrPort(remoteAddr)
+	if err := s.verifyClient(ctx, clientKey, clientInfo, remoteIPPort.Addr()); err != nil {
 		return fmt.Errorf("client %v rejected: %v", clientKey, err)
 	}
 
@@ -817,8 +851,6 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	remoteIPPort, _ := netip.ParseAddrPort(remoteAddr)
-
 	c := &sclient{
 		connNum:        connNum,
 		s:              s,
@@ -830,11 +862,12 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 		done:           ctx.Done(),
 		remoteIPPort:   remoteIPPort,
 		connectedAt:    s.clock.Now(),
-		sendQueue:      make(chan pkt, perClientSendQueueDepth),
-		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
+		sendQueue:      make(chan pkt, s.perClientSendQueueDepth),
+		discoSendQueue: make(chan pkt, s.perClientSendQueueDepth),
 		sendPongCh:     make(chan [8]byte, 1),
 		peerGone:       make(chan peerGoneMsg),
 		canMesh:        s.isMeshPeer(clientInfo),
+		isNotIdealConn: IdealNodeContextKey.Value(ctx) != "",
 		peerGoneLim:    rate.NewLimiter(rate.Every(time.Second), 3),
 	}
 
@@ -881,6 +914,9 @@ func (c *sclient) run(ctx context.Context) error {
 			if errors.Is(err, context.Canceled) {
 				c.debugLogf("sender canceled by reader exiting")
 			} else {
+				if errors.Is(err, os.ErrDeadlineExceeded) {
+					c.s.sclientWriteTimeouts.Add(1)
+				}
 				c.logf("sender failed: %v", err)
 			}
 		}
@@ -1505,6 +1541,7 @@ type sclient struct {
 	peerGone       chan peerGoneMsg // write request that a peer is not at this server (not used by mesh peers)
 	meshUpdate     chan struct{}    // write request to write peerStateChange
 	canMesh        bool             // clientInfo had correct mesh token for inter-region routing
+	isNotIdealConn bool             // client indicated it is not its ideal node in the region
 	isDup          atomic.Bool      // whether more than 1 sclient for key is connected
 	isDisabled     atomic.Bool      // whether sends to this peer are disabled due to active/active dups
 	debug          bool             // turn on for verbose logging
@@ -1540,6 +1577,9 @@ func (c *sclient) presentFlags() PeerPresentFlags {
 	if c.canMesh {
 		f |= PeerPresentIsMeshPeer
 	}
+	if c.isNotIdealConn {
+		f |= PeerPresentNotIdeal
+	}
 	if f == 0 {
 		return PeerPresentIsRegular
 	}
@@ -1721,7 +1761,19 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 }
 
 func (c *sclient) setWriteDeadline() {
-	c.nc.SetWriteDeadline(time.Now().Add(writeTimeout))
+	d := writeTimeout
+	if c.canMesh {
+		// Trusted peers get more tolerance.
+		//
+		// The "canMesh" is a bit of a misnomer; mesh peers typically run over a
+		// different interface for a per-region private VPC and are not
+		// throttled. But monitoring software elsewhere over the internet also
+		// use the private mesh key to subscribe to connect/disconnect events
+		// and might hit throttling and need more time to get the initial dump
+		// of connected peers.
+		d = privilegedWriteTimeout
+	}
+	c.nc.SetWriteDeadline(time.Now().Add(d))
 }
 
 // sendKeepAlive sends a keep-alive frame, without flushing.
@@ -2033,6 +2085,7 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("gauge_current_file_descriptors", expvar.Func(func() any { return metrics.CurrentFDs() }))
 	m.Set("gauge_current_connections", &s.curClients)
 	m.Set("gauge_current_home_connections", &s.curHomeClients)
+	m.Set("gauge_current_notideal_connections", &s.curClientsNotIdeal)
 	m.Set("gauge_clients_total", expvar.Func(func() any { return len(s.clientsMesh) }))
 	m.Set("gauge_clients_local", expvar.Func(func() any { return len(s.clients) }))
 	m.Set("gauge_clients_remote", expvar.Func(func() any { return len(s.clientsMesh) - len(s.clients) }))
@@ -2060,6 +2113,7 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("multiforwarder_created", &s.multiForwarderCreated)
 	m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
 	m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
+	m.Set("sclient_write_timeouts", &s.sclientWriteTimeouts)
 	m.Set("average_queue_duration_ms", expvar.Func(func() any {
 		return math.Float64frombits(atomic.LoadUint64(s.avgQueueDuration))
 	}))