control/controlhttp, util/multierr: deduplicate context errors

Add a new function to the util/multierr package that allows
deduplicating the `context.Canceled` and `context.DeadlineExceeded`
errors from a slice, and then use that to reduce log spam in
controlhttp.

Updates #14233

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I5a121124fe27c4449705ba35de6faf83665db5fe

.github/workflows/codeql-analysis.yml

@@ -49,13 +49,13 @@ jobs:
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1

.github/workflows/golangci-lint.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
         with:
           go-version-file: go.mod
           cache: false

.github/workflows/test.yml

@@ -80,7 +80,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -153,13 +153,13 @@ jobs:
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -260,7 +260,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -319,7 +319,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -367,7 +367,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -461,7 +461,7 @@ jobs:
       run: |
         echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts

build_docker.sh

@@ -17,12 +17,20 @@ eval "$(./build_dist.sh shellvars)"
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
 DEFAULT_BASE="tailscale/alpine-base:3.18"
+# Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
+# Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
+# annotations defined here will be overriden by release scripts that call this script.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+DEFAULT_ANNOTATIONS="org.opencontainers.image.source=https://github.com/tailscale/tailscale/blob/main/build_docker.sh,org.opencontainers.image.vendor=Tailscale"
 
 PUSH="${PUSH:-false}"
 TARGET="${TARGET:-${DEFAULT_TARGET}}"
 TAGS="${TAGS:-${DEFAULT_TAGS}}"
 BASE="${BASE:-${DEFAULT_BASE}}"
 PLATFORM="${PLATFORM:-}" # default to all platforms
+# OCI annotations that will be added to the image.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
+ANNOTATIONS="${ANNOTATIONS:-${DEFAULT_ANNOTATIONS}}"
 
 case "$TARGET" in
   client)
@@ -43,6 +51,7 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/containerboot
     ;;
   operator)
@@ -60,6 +69,7 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/operator
     ;;
   k8s-nameserver)
@@ -77,6 +87,7 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/k8s-nameserver
     ;;
   *)

cmd/containerboot/healthz.go

@@ -39,7 +39,7 @@ func runHealthz(addr string, h *healthz) {
 		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
 	}
 	mux := http.NewServeMux()
-	mux.Handle("/healthz", h)
+	mux.Handle("GET /healthz", h)
 	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
 	hs := &http.Server{Handler: mux}
 

cmd/containerboot/kube.go

@@ -61,7 +61,7 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 			Path: "/data/authkey",
 		},
 	}
-	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
+	if err := kc.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
 		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
@@ -81,7 +81,7 @@ func initKubeClient(root string) {
 		kubeclient.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kubeclient.New()
+	kc, err = kubeclient.New("tailscale-container")
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/main.go

@@ -102,7 +102,6 @@ import (
 	"net/netip"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -179,6 +178,14 @@ func main() {
 	}
 	defer killTailscaled()
 
+	if cfg.LocalAddrPort != "" && cfg.MetricsEnabled {
+		m := &metrics{
+			lc:            client,
+			debugEndpoint: cfg.DebugAddrPort,
+		}
+		runMetrics(cfg.LocalAddrPort, m)
+	}
+
 	if cfg.EnableForwardingOptimizations {
 		if err := client.SetUDPGROForwarding(bootCtx); err != nil {
 			log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
@@ -731,7 +738,6 @@ func tailscaledConfigFilePath() string {
 		}
 		cv, err := kubeutils.CapVerFromFileName(e.Name())
 		if err != nil {
-			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
 			continue
 		}
 		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
@@ -739,8 +745,9 @@ func tailscaledConfigFilePath() string {
 		}
 	}
 	if maxCompatVer == -1 {
-		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
+		log.Fatalf("no tailscaled config file found in %q for current capability version %d", dir, tailcfg.CurrentCapabilityVersion)
 	}
-	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	filePath := filepath.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	log.Printf("Using tailscaled config file %q to match current capability version %d", filePath, tailcfg.CurrentCapabilityVersion)
+	return filePath
 }

cmd/containerboot/metrics.go

@@ -0,0 +1,91 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
+)
+
+// metrics is a simple metrics HTTP server, if enabled it forwards requests to
+// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
+type metrics struct {
+	debugEndpoint string
+	lc            *tailscale.LocalClient
+}
+
+func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+
+	resp, err := do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	for key, val := range resp.Header {
+		for _, v := range val {
+			w.Header().Add(key, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
+	localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
+	proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
+}
+
+func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
+	if m.debugEndpoint == "" {
+		http.Error(w, "debug endpoint not configured", http.StatusNotFound)
+		return
+	}
+
+	debugURL := "http://" + m.debugEndpoint + r.URL.Path
+	proxy(w, r, debugURL, http.DefaultClient.Do)
+}
+
+// runMetrics runs a simple HTTP metrics endpoint at <addr>/metrics, forwarding
+// requests to tailscaled's /localapi/v0/usermetrics API.
+//
+// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
+// endpoint if configured to ease migration for a breaking change serving user
+// metrics instead of debug metrics on the "metrics" port.
+func runMetrics(addr string, m *metrics) {
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided metrics endpoint address %q: %v", addr, err)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /metrics", m.handleMetrics)
+	mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
+
+	log.Printf("Running metrics endpoint at %s/metrics", addr)
+	ms := &http.Server{Handler: mux}
+
+	go func() {
+		if err := ms.Serve(ln); err != nil {
+			log.Fatalf("failed running metrics endpoint: %v", err)
+		}
+	}()
+}

cmd/containerboot/services.go

@@ -389,7 +389,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta
 		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
 		Value: bs,
 	}
-	if err := ep.kc.JSONPatchSecret(ctx, ep.stateSecret, []kubeclient.JSONPatch{patch}); err != nil {
+	if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil {
 		return fmt.Errorf("error patching state Secret: %w", err)
 	}
 	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()

cmd/containerboot/settings.go

@@ -67,11 +67,18 @@ type settings struct {
 	PodIP               string
 	PodIPv4             string
 	PodIPv6             string
-	HealthCheckAddrPort string
+	HealthCheckAddrPort string // TODO(tomhjp): use the local addr/port instead.
+	LocalAddrPort       string
+	MetricsEnabled      bool
+	DebugAddrPort       string
 	EgressSvcsCfgPath   string
 }
 
 func configFromEnv() (*settings, error) {
+	defaultLocalAddrPort := ""
+	if v, ok := os.LookupEnv("POD_IP"); ok && v != "" {
+		defaultLocalAddrPort = fmt.Sprintf("%s:9002", v)
+	}
 	cfg := &settings{
 		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
 		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
@@ -98,6 +105,9 @@ func configFromEnv() (*settings, error) {
 		PodIP:                                 defaultEnv("POD_IP", ""),
 		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
 		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
+		LocalAddrPort:                         defaultEnv("TS_LOCAL_ADDR_PORT", defaultLocalAddrPort),
+		MetricsEnabled:                        defaultBool("TS_METRICS_ENABLED", false),
+		DebugAddrPort:                         defaultEnv("TS_DEBUG_ADDR_PORT", ""),
 		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
 	}
 	podIPs, ok := os.LookupEnv("POD_IPS")
@@ -175,6 +185,16 @@ func (s *settings) validate() error {
 			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
 		}
 	}
+	if s.LocalAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
+		}
+	}
+	if s.DebugAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.DebugAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_DEBUG_ADDR_PORT value %q: %w", s.DebugAddrPort, err)
+		}
+	}
 	return nil
 }
 

cmd/containerboot/tailscaled.go

@@ -90,6 +90,12 @@ func tailscaledArgs(cfg *settings) []string {
 	if cfg.TailscaledConfigFilePath != "" {
 		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
 	}
+	// Once enough proxy versions have been released for all the supported
+	// versions to understand this cfg setting, the operator can stop
+	// setting TS_TAILSCALED_EXTRA_ARGS for the debug flag.
+	if cfg.DebugAddrPort != "" && !strings.Contains(cfg.DaemonExtraArgs, cfg.DebugAddrPort) {
+		args = append(args, "--debug="+cfg.DebugAddrPort)
+	}
 	if cfg.DaemonExtraArgs != "" {
 		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
 	}

cmd/derper/depaware.txt

@@ -27,7 +27,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
@@ -152,7 +151,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
@@ -160,6 +158,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/set                                       from tailscale.com/derp+
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
@@ -244,7 +243,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         crypto/tls                                                   from golang.org/x/crypto/acme+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -276,7 +274,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/derper/derper_test.go

@@ -109,6 +109,7 @@ func TestDeps(t *testing.T) {
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
 			"tailscale.com/net/packet":           "not needed in derper",
 			"github.com/gaissmai/bart":           "not needed in derper",
+			"database/sql/driver":                "not needed in derper", // previously came in via github.com/google/uuid
 		},
 	}.Check(t)
 }

cmd/derpprobe/derpprobe.go

@@ -29,6 +29,7 @@ var (
 	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
 	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
 	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	regionCode   = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -47,6 +48,9 @@ func main() {
 	if *bwInterval > 0 {
 		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
 	}
+	if *regionCode != "" {
+		opts = append(opts, prober.WithRegion(*regionCode))
+	}
 	dp, err := prober.DERP(p, *derpMapURL, opts...)
 	if err != nil {
 		log.Fatal(err)

cmd/k8s-operator/connector.go

@@ -13,7 +13,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pkg/errors"
+	"errors"
+
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
@@ -58,6 +59,7 @@ type ConnectorReconciler struct {
 
 	subnetRouters set.Slice[types.UID] // for subnet routers gauge
 	exitNodes     set.Slice[types.UID] // for exit nodes gauge
+	appConnectors set.Slice[types.UID] // for app connectors gauge
 }
 
 var (
@@ -67,6 +69,8 @@ var (
 	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
 	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
+	// gaugeConnectorAppConnectorResources tracks the number of Connectors currently managed by this operator instance that are app connectors.
+	gaugeConnectorAppConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithAppConnectorCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -108,13 +112,12 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	oldCnStatus := cn.Status.DeepCopy()
 	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
+		var updateErr error
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
+			updateErr = a.Client.Status().Update(ctx, cn)
 		}
-		return res, err
+		return res, errors.Join(err, updateErr)
 	}
 
 	if !slices.Contains(cn.Finalizers, FinalizerName) {
@@ -150,6 +153,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 	}
+	if cn.Spec.AppConnector != nil {
+		cn.Status.IsAppConnector = true
+	}
 	cn.Status.SubnetRoutes = ""
 	return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 }
@@ -189,23 +195,37 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 	}
 
+	if cn.Spec.AppConnector != nil {
+		sts.Connector.isAppConnector = true
+		if len(cn.Spec.AppConnector.Routes) != 0 {
+			sts.Connector.routes = cn.Spec.AppConnector.Routes.Stringify()
+		}
+	}
+
 	a.mu.Lock()
-	if sts.Connector.isExitNode {
+	if cn.Spec.ExitNode {
 		a.exitNodes.Add(cn.UID)
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
-	if sts.Connector.routes != "" {
+	if cn.Spec.SubnetRouter != nil {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
+	if cn.Spec.AppConnector != nil {
+		a.appConnectors.Add(cn.GetUID())
+	} else {
+		a.appConnectors.Remove(cn.GetUID())
+	}
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
@@ -248,12 +268,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
 	a.mu.Lock()
 	a.subnetRouters.Remove(cn.UID)
 	a.exitNodes.Remove(cn.UID)
+	a.appConnectors.Remove(cn.UID)
 	a.mu.Unlock()
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 	return true, nil
 }
@@ -262,8 +285,14 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 	// Connector fields are already validated at apply time with CEL validation
 	// on custom resource fields. The checks here are a backup in case the
 	// CEL validation breaks without us noticing.
-	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
-		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
+	if cn.Spec.SubnetRouter == nil && !cn.Spec.ExitNode && cn.Spec.AppConnector == nil {
+		return errors.New("invalid spec: a Connector must be configured as at least one of subnet router, exit node or app connector")
+	}
+	if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
+		return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
+	}
+	if cn.Spec.AppConnector != nil {
+		return validateAppConnector(cn.Spec.AppConnector)
 	}
 	if cn.Spec.SubnetRouter == nil {
 		return nil
@@ -272,19 +301,27 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 }
 
 func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
-	if len(sb.AdvertiseRoutes) < 1 {
+	if len(sb.AdvertiseRoutes) == 0 {
 		return errors.New("invalid subnet router spec: no routes defined")
 	}
-	var err error
-	for _, route := range sb.AdvertiseRoutes {
+	return validateRoutes(sb.AdvertiseRoutes)
+}
+
+func validateAppConnector(ac *tsapi.AppConnector) error {
+	return validateRoutes(ac.Routes)
+}
+
+func validateRoutes(routes tsapi.Routes) error {
+	var errs []error
+	for _, route := range routes {
 		pfx, e := netip.ParsePrefix(string(route))
 		if e != nil {
-			err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err))
+			errs = append(errs, fmt.Errorf("route %v is invalid: %v", route, e))
 			continue
 		}
 		if pfx.Masked() != pfx {
-			err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
+			errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
 		}
 	}
-	return err
+	return errors.Join(errs...)
 }

cmd/k8s-operator/connector_test.go

@@ -8,12 +8,14 @@ package main
 import (
 	"context"
 	"testing"
+	"time"
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -296,3 +298,100 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	expectReconciled(t, cr, "", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 }
+
+func TestConnectorWithAppConnector(t *testing.T) {
+	// Setup
+	cn := &tsapi.Connector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  types.UID("1234-UID"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       tsapi.ConnectorKind,
+			APIVersion: "tailscale.io/v1alpha1",
+		},
+		Spec: tsapi.ConnectorSpec{
+			AppConnector: &tsapi.AppConnector{},
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(cn).
+		WithStatusSubresource(cn).
+		Build()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	fr := record.NewFakeRecorder(1)
+	cr := &ConnectorReconciler{
+		Client: fc,
+		clock:  cl,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		recorder: fr,
+	}
+
+	// 1. Connector with app connnector is created and becomes ready
+	expectReconciled(t, cr, "", "test")
+	fullName, shortName := findGenName(t, fc, "", "test", "connector")
+	opts := configOpts{
+		stsName:        shortName,
+		secretName:     fullName,
+		parentType:     "connector",
+		hostname:       "test-connector",
+		app:            kubetypes.AppConnector,
+		isAppConnector: true,
+	}
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	// Connector's ready condition should be set to true
+
+	cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
+	cn.Status.IsAppConnector = true
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectEqual(t, fc, cn, nil)
+
+	// 2. Connector with invalid app connector routes has status set to invalid
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	expectReconciled(t, cr, "", "test")
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionFalse,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorInvalid,
+		Message:            "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
+	}}
+	expectEqual(t, fc, cn, nil)
+
+	// 3. Connector with valid app connnector routes becomes ready
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectReconciled(t, cr, "", "test")
+}

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -81,6 +81,14 @@ spec:
             - name: PROXY_DEFAULT_CLASS
               value: {{ .Values.proxyConfig.defaultProxyClass }}
             {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.ingressClass.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: IngressClass
 metadata:
@@ -6,3 +7,4 @@ metadata:
 spec:
   controller: tailscale.com/ts-ingress # controller name currently can not be changed
   # parameters: {} # currently no parameters are supported
+{{- end }}

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -16,6 +16,9 @@ rules:
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -54,6 +54,9 @@ operatorConfig:
   # - name: EXTRA_VAR2
   #   value: "value2"
 
+# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
+ingressClass:
+  enabled: true
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -24,6 +24,10 @@ spec:
           jsonPath: .status.isExitNode
           name: IsExitNode
           type: string
+        - description: Whether this Connector instance is an app connector.
+          jsonPath: .status.isAppConnector
+          name: IsAppConnector
+          type: string
         - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
@@ -66,10 +70,40 @@ spec:
                 https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
+                appConnector:
+                  description: |-
+                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                    Connector does not act as an app connector.
+                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                    Admin panel.
+                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                    tested or optimised for.
+                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                    device with a static IP address.
+                    https://tailscale.com/kb/1281/app-connectors
+                  type: object
+                  properties:
+                    routes:
+                      description: |-
+                        Routes are optional preconfigured routes for the domains routed via the app connector.
+                        If not set, routes for the domains will be discovered dynamically.
+                        If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                        also dynamically discover other routes.
+                        https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                      type: array
+                      minItems: 1
+                      items:
+                        type: string
+                        format: cidr
                 exitNode:
                   description: |-
-                    ExitNode defines whether the Connector node should act as a
-                    Tailscale exit node. Defaults to false.
+                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                    This field is mutually exclusive with the appConnector field.
                     https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
@@ -90,9 +124,11 @@ spec:
                   type: string
                 subnetRouter:
                   description: |-
-                    SubnetRouter defines subnet routes that the Connector node should
-                    expose to tailnet. If unset, none are exposed.
+                    SubnetRouter defines subnet routes that the Connector device should
+                    expose to tailnet as a Tailscale subnet router.
                     https://tailscale.com/kb/1019/subnets/
+                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                    This field is mutually exclusive with the appConnector field.
                   type: object
                   required:
                     - advertiseRoutes
@@ -125,8 +161,10 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
               x-kubernetes-validations:
-                - rule: has(self.subnetRouter) || self.exitNode == true
-                  message: A Connector needs to be either an exit node or a subnet router, or both.
+                - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                  message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                - rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
+                  message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
             status:
               description: |-
                 ConnectorStatus describes the status of the Connector. This is set
@@ -200,6 +238,9 @@ spec:
                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                     node.
                   type: string
+                isAppConnector:
+                  description: IsAppConnector is set to true if the Connector acts as an app connector.
+                  type: boolean
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -73,7 +73,12 @@ spec:
                     enable:
                       description: |-
                         Setting enable to true will make the proxy serve Tailscale metrics
-                        at <pod-ip>:9001/debug/metrics.
+                        at <pod-ip>:9002/metrics.
+
+                        In 1.78.x and 1.80.x, this field also serves as the default value for
+                        .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                        fields will independently default to false.
+
                         Defaults to false.
                       type: boolean
                 statefulSet:
@@ -1249,6 +1254,25 @@ spec:
                           description: Configuration for the proxy container running tailscale.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1553,6 +1577,25 @@ spec:
                           description: Configuration for the proxy init container that enables forwarding.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -53,6 +53,10 @@ spec:
               jsonPath: .status.isExitNode
               name: IsExitNode
               type: string
+            - description: Whether this Connector instance is an app connector.
+              jsonPath: .status.isAppConnector
+              name: IsAppConnector
+              type: string
             - description: Status of the deployed Connector resources.
               jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
               name: Status
@@ -91,10 +95,40 @@ spec:
                             More info:
                             https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
+                            appConnector:
+                                description: |-
+                                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                                    Connector does not act as an app connector.
+                                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                                    Admin panel.
+                                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                                    tested or optimised for.
+                                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                                    device with a static IP address.
+                                    https://tailscale.com/kb/1281/app-connectors
+                                properties:
+                                    routes:
+                                        description: |-
+                                            Routes are optional preconfigured routes for the domains routed via the app connector.
+                                            If not set, routes for the domains will be discovered dynamically.
+                                            If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                                            also dynamically discover other routes.
+                                            https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                                        items:
+                                            format: cidr
+                                            type: string
+                                        minItems: 1
+                                        type: array
+                                type: object
                             exitNode:
                                 description: |-
-                                    ExitNode defines whether the Connector node should act as a
-                                    Tailscale exit node. Defaults to false.
+                                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                                    This field is mutually exclusive with the appConnector field.
                                     https://tailscale.com/kb/1103/exit-nodes
                                 type: boolean
                             hostname:
@@ -115,9 +149,11 @@ spec:
                                 type: string
                             subnetRouter:
                                 description: |-
-                                    SubnetRouter defines subnet routes that the Connector node should
-                                    expose to tailnet. If unset, none are exposed.
+                                    SubnetRouter defines subnet routes that the Connector device should
+                                    expose to tailnet as a Tailscale subnet router.
                                     https://tailscale.com/kb/1019/subnets/
+                                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                                    This field is mutually exclusive with the appConnector field.
                                 properties:
                                     advertiseRoutes:
                                         description: |-
@@ -151,8 +187,10 @@ spec:
                                 type: array
                         type: object
                         x-kubernetes-validations:
-                            - message: A Connector needs to be either an exit node or a subnet router, or both.
-                              rule: has(self.subnetRouter) || self.exitNode == true
+                            - message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                              rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                            - message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
+                              rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
                     status:
                         description: |-
                             ConnectorStatus describes the status of the Connector. This is set
@@ -225,6 +263,9 @@ spec:
                                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                                     node.
                                 type: string
+                            isAppConnector:
+                                description: IsAppConnector is set to true if the Connector acts as an app connector.
+                                type: boolean
                             isExitNode:
                                 description: IsExitNode is set to true if the Connector acts as an exit node.
                                 type: boolean
@@ -499,7 +540,12 @@ spec:
                                     enable:
                                         description: |-
                                             Setting enable to true will make the proxy serve Tailscale metrics
-                                            at <pod-ip>:9001/debug/metrics.
+                                            at <pod-ip>:9002/metrics.
+
+                                            In 1.78.x and 1.80.x, this field also serves as the default value for
+                                            .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                                            fields will independently default to false.
+
                                             Defaults to false.
                                         type: boolean
                                 required:
@@ -1675,6 +1721,25 @@ spec:
                                             tailscaleContainer:
                                                 description: Configuration for the proxy container running tailscale.
                                                 properties:
+                                                    debug:
+                                                        description: |-
+                                                            Configuration for enabling extra debug information in the container.
+                                                            Not recommended for production use.
+                                                        properties:
+                                                            enable:
+                                                                description: |-
+                                                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                                                    9001 is a container port named "debug". The endpoints and their responses
+                                                                    may change in backwards incompatible ways in the future, and should not
+                                                                    be considered stable.
+
+                                                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                                                    this setting will default to false, and no requests will be proxied.
+                                                                type: boolean
+                                                        type: object
                                                     env:
                                                         description: |-
                                                             List of environment variables to set in the container.
@@ -1979,6 +2044,25 @@ spec:
                                             tailscaleInitContainer:
                                                 description: Configuration for the proxy init container that enables forwarding.
                                                 properties:
+                                                    debug:
+                                                        description: |-
+                                                            Configuration for enabling extra debug information in the container.
+                                                            Not recommended for production use.
+                                                        properties:
+                                                            enable:
+                                                                description: |-
+                                                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                                                    9001 is a container port named "debug". The endpoints and their responses
+                                                                    may change in backwards incompatible ways in the future, and should not
+                                                                    be considered stable.
+
+                                                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                                                    this setting will default to false, and no requests will be proxied.
+                                                                type: boolean
+                                                        type: object
                                                     env:
                                                         description: |-
                                                             List of environment variables to set in the container.
@@ -4662,6 +4746,14 @@ rules:
         - patch
         - update
         - watch
+    - apiGroups:
+        - ""
+      resources:
+        - events
+      verbs:
+        - create
+        - patch
+        - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -4734,6 +4826,14 @@ spec:
                       value: "false"
                     - name: PROXY_FIREWALL_MODE
                       value: auto
+                    - name: POD_NAME
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.name
+                    - name: POD_UID
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.uid
                   image: tailscale/k8s-operator:unstable
                   imagePullPolicy: Always
                   name: operator

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -30,6 +30,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
           securityContext:
             capabilities:
               add:

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -24,3 +24,11 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid

cmd/k8s-operator/operator_test.go

@@ -1388,7 +1388,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
+		confFileHash:    "a67b5ad3ff605531c822327e8f1a23dd0846e1075b722c13402f7d5d0ba32ba2",
 		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
@@ -1399,7 +1399,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684"
+	o.confFileHash = "888a993ebee20ad6be99623b45015339de117946850cf1252bede0b570e04293"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }

cmd/k8s-operator/proxyclass.go

@@ -160,6 +160,10 @@ func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations fiel
 						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
 					}
 				}
+
+				if tc.Debug != nil {
+					violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "debug"), tc.Debug, "debug settings cannot be configured on the init container"))
+				}
 			}
 		}
 	}

cmd/k8s-operator/proxyclass_test.go

@@ -135,3 +135,56 @@ func TestProxyClass(t *testing.T) {
 	expectReconciled(t, pcr, "", "test")
 	expectEvents(t, fr, expectedEvents)
 }
+
+func TestValidateProxyClass(t *testing.T) {
+	for name, tc := range map[string]struct {
+		pc    *tsapi.ProxyClass
+		valid bool
+	}{
+		"empty": {
+			valid: true,
+			pc:    &tsapi.ProxyClass{},
+		},
+		"debug_enabled_for_main_container": {
+			valid: true,
+			pc: &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{
+									Enable: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"debug_enabled_for_init_container": {
+			valid: false,
+			pc: &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleInitContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{
+									Enable: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			pcr := &ProxyClassReconciler{}
+			err := pcr.validate(tc.pc)
+			valid := err == nil
+			if valid != tc.valid {
+				t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
+			}
+		})
+	}
+}

cmd/k8s-operator/proxygroup.go

@@ -353,7 +353,7 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc
 
 func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) {
 	logger := r.logger(pg.Name)
-	var allConfigs []tailscaledConfigs
+	var configSHA256Sum string
 	for i := range pgReplicas(pg) {
 		cfgSecret := &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
@@ -389,7 +389,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		if err != nil {
 			return "", fmt.Errorf("error creating tailscaled config: %w", err)
 		}
-		allConfigs = append(allConfigs, configs)
 
 		for cap, cfg := range configs {
 			cfgJSON, err := json.Marshal(cfg)
@@ -399,6 +398,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
 		}
 
+		// The config sha256 sum is a value for a hash annotation used to trigger
+		// pod restarts when tailscaled config changes. Any config changes apply
+		// to all replicas, so it is sufficient to only hash the config for the
+		// first replica.
+		//
+		// In future, we're aiming to eliminate restarts altogether and have
+		// pods dynamically reload their config when it changes.
+		if i == 0 {
+			sum := sha256.New()
+			for _, cfg := range configs {
+				// Zero out the auth key so it doesn't affect the sha256 hash when we
+				// remove it from the config after the pods have all authed. Otherwise
+				// all the pods will need to restart immediately after authing.
+				cfg.AuthKey = nil
+				b, err := json.Marshal(cfg)
+				if err != nil {
+					return "", err
+				}
+				if _, err := sum.Write(b); err != nil {
+					return "", err
+				}
+			}
+
+			configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil))
+		}
+
 		if existingCfgSecret != nil {
 			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
 			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
@@ -412,16 +437,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		}
 	}
 
-	sum := sha256.New()
-	b, err := json.Marshal(allConfigs)
-	if err != nil {
-		return "", err
-	}
-	if _, err := sum.Write(b); err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%x", sum.Sum(nil)), nil
+	return configSHA256Sum, nil
 }
 
 func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {

cmd/k8s-operator/proxygroup_specs.go

@@ -93,6 +93,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 	c.Image = image
 	c.VolumeMounts = func() []corev1.VolumeMount {
 		var mounts []corev1.VolumeMount
+
+		// TODO(tomhjp): Read config directly from the secret instead. The
+		// mounts change on scaling up/down which causes unnecessary restarts
+		// for pods that haven't meaningfully changed.
 		for i := range pgReplicas(pg) {
 			mounts = append(mounts, corev1.VolumeMount{
 				Name:      fmt.Sprintf("tailscaledconfig-%d", i),
@@ -122,15 +126,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 					},
 				},
 			},
-			{
-				Name: "POD_NAME",
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						// Secret is named after the pod.
-						FieldPath: "metadata.name",
-					},
-				},
-			},
 			{
 				Name:  "TS_KUBE_SECRET",
 				Value: "$(POD_NAME)",
@@ -143,10 +138,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
 				Value: "/etc/tsconfig/$(POD_NAME)",
 			},
-			{
-				Name:  "TS_USERSPACE",
-				Value: "false",
-			},
 			{
 				Name:  "TS_INTERNAL_APP",
 				Value: kubetypes.AppProxyGroupEgress,
@@ -167,7 +158,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 			})
 		}
 
-		return envs
+		return append(c.Env, envs...)
 	}()
 
 	return ss, nil
@@ -211,6 +202,15 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 					return secrets
 				}(),
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"create",
+					"patch",
+					"get",
+				},
+			},
 		},
 	}
 }

cmd/k8s-operator/proxygroup_test.go

@@ -35,6 +35,8 @@ var defaultProxyClassAnnotations = map[string]string{
 }
 
 func TestProxyGroup(t *testing.T) {
+	const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196"
+
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "default-pc",
@@ -80,6 +82,7 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, false, "")
 	})
 
 	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
@@ -100,10 +103,11 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		if expected := 1; reconciler.proxyGroups.Len() != expected {
 			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
 		}
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		keyReq := tailscale.KeyCapabilities{
 			Devices: tailscale.KeyDeviceCapabilities{
 				Create: tailscale.KeyDeviceCreateCapabilities{
@@ -135,7 +139,7 @@ func TestProxyGroup(t *testing.T) {
 		}
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 	})
 
 	t.Run("scale_up_to_3", func(t *testing.T) {
@@ -146,6 +150,7 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 
 		addNodeIDToStateSecrets(t, fc, pg)
 		expectReconciled(t, reconciler, "", pg.Name)
@@ -155,7 +160,7 @@ func TestProxyGroup(t *testing.T) {
 			TailnetIPs: []string{"1.2.3.4", "::1"},
 		})
 		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 	})
 
 	t.Run("scale_down_to_1", func(t *testing.T) {
@@ -163,11 +168,26 @@ func TestProxyGroup(t *testing.T) {
 		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
 			p.Spec = pg.Spec
 		})
+
 		expectReconciled(t, reconciler, "", pg.Name)
+
 		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+	})
 
-		expectProxyGroupResources(t, fc, pg, true)
+	t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) {
+		pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{
+			AcceptRoutes: true,
+		}
+		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+			p.Spec = pc.Spec
+		})
+
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74")
 	})
 
 	t.Run("delete_and_cleanup", func(t *testing.T) {
@@ -191,13 +211,13 @@ func TestProxyGroup(t *testing.T) {
 	})
 }
 
-func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool) {
+func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
 	t.Helper()
 
 	role := pgRole(pg, tsNamespace)
 	roleBinding := pgRoleBinding(pg, tsNamespace)
 	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", cfgHash)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -207,9 +227,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 		expectEqual(t, fc, role, nil)
 		expectEqual(t, fc, roleBinding, nil)
 		expectEqual(t, fc, serviceAccount, nil)
-		expectEqual(t, fc, statefulSet, func(ss *appsv1.StatefulSet) {
-			ss.Spec.Template.Annotations[podAnnotationLastSetConfigFileHash] = ""
-		})
+		expectEqual(t, fc, statefulSet, nil)
 	} else {
 		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
 		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
@@ -218,11 +236,13 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 	}
 
 	var expectedSecrets []string
-	for i := range pgReplicas(pg) {
-		expectedSecrets = append(expectedSecrets,
-			fmt.Sprintf("%s-%d", pg.Name, i),
-			fmt.Sprintf("%s-%d-config", pg.Name, i),
-		)
+	if shouldExist {
+		for i := range pgReplicas(pg) {
+			expectedSecrets = append(expectedSecrets,
+				fmt.Sprintf("%s-%d", pg.Name, i),
+				fmt.Sprintf("%s-%d-config", pg.Name, i),
+			)
+		}
 	}
 	expectSecrets(t, fc, expectedSecrets)
 }

cmd/k8s-operator/sts.go

@@ -132,10 +132,13 @@ type tailscaleSTSConfig struct {
 }
 
 type connector struct {
-	// routes is a list of subnet routes that this Connector should expose.
+	// routes is a list of routes that this Connector should advertise either as a subnet router or as an app
+	// connector.
 	routes string
 	// isExitNode defines whether this Connector should act as an exit node.
 	isExitNode bool
+	// isAppConnector defines whether this Connector should act as an app connector.
+	isAppConnector bool
 }
 type tsnetServer interface {
 	CertDomains() []string
@@ -473,7 +476,7 @@ var proxyYaml []byte
 //go:embed deploy/manifests/userspace-proxy.yaml
 var userspaceProxyYaml []byte
 
-func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, _ map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -518,11 +521,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_KUBE_SECRET",
 			Value: proxySecret,
 		},
-		corev1.EnvVar{
-			// Old tailscaled config key is still used for backwards compatibility.
-			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
-			Value: "/etc/tsconfig/tailscaled",
-		},
 		corev1.EnvVar{
 			// New style is in the form of cap-<capability-version>.hujson.
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
@@ -668,24 +666,42 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [
 	return custom
 }
 
+func debugSetting(pc *tsapi.ProxyClass) bool {
+	if pc == nil ||
+		pc.Spec.StatefulSet == nil ||
+		pc.Spec.StatefulSet.Pod == nil ||
+		pc.Spec.StatefulSet.Pod.TailscaleContainer == nil ||
+		pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug == nil {
+		// This default will change to false in 1.82.0.
+		return pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
+	}
+
+	return pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug.Enable
+}
+
 func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet {
 	if pc == nil || ss == nil {
 		return ss
 	}
-	if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
-		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
-			enableMetrics(ss, pc)
-		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+
+	metricsEnabled := pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
+	debugEnabled := debugSetting(pc)
+	if metricsEnabled || debugEnabled {
+		isEgress := stsCfg != nil && (stsCfg.TailnetTargetFQDN != "" || stsCfg.TailnetTargetIP != "")
+		isForwardingL7Ingress := stsCfg != nil && stsCfg.ForwardClusterTrafficViaL7IngressProxy
+		if isEgress {
 			// TODO (irbekrm): fix this
 			// For Ingress proxies that have been configured with
 			// tailscale.com/experimental-forward-cluster-traffic-via-ingress
 			// annotation, all cluster traffic is forwarded to the
 			// Ingress backend(s).
-			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
-		} else {
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for egress proxies.")
+		} else if isForwardingL7Ingress {
 			// TODO (irbekrm): fix this
 			// For egress proxies, currently all cluster traffic is forwarded to the tailnet target.
 			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		} else {
+			enableEndpoints(ss, metricsEnabled, debugEnabled)
 		}
 	}
 
@@ -763,16 +779,58 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	return ss
 }
 
-func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) {
+func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
 	for i, c := range ss.Spec.Template.Spec.Containers {
 		if c.Name == "tailscale" {
-			// Serve metrics on on <pod-ip>:9001/debug/metrics. If
-			// we didn't specify Pod IP here, the proxy would, in
-			// some cases, also listen to its Tailscale IP- we don't
-			// want folks to start relying on this side-effect as a
-			// feature.
-			ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
-			ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001})
+			if debug {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
+					// Serve tailscaled's debug metrics on on
+					// <pod-ip>:9001/debug/metrics. If we didn't specify Pod IP
+					// here, the proxy would, in some cases, also listen to its
+					// Tailscale IP- we don't want folks to start relying on this
+					// side-effect as a feature.
+					corev1.EnvVar{
+						Name:  "TS_DEBUG_ADDR_PORT",
+						Value: "$(POD_IP):9001",
+					},
+					// TODO(tomhjp): Can remove this env var once 1.76.x is no
+					// longer supported.
+					corev1.EnvVar{
+						Name:  "TS_TAILSCALED_EXTRA_ARGS",
+						Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
+					},
+				)
+
+				ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
+					corev1.ContainerPort{
+						Name:          "debug",
+						Protocol:      "TCP",
+						ContainerPort: 9001,
+					},
+				)
+			}
+
+			if metrics {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
+					// Serve client metrics on <pod-ip>:9002/metrics.
+					corev1.EnvVar{
+						Name:  "TS_LOCAL_ADDR_PORT",
+						Value: "$(POD_IP):9002",
+					},
+					corev1.EnvVar{
+						Name:  "TS_METRICS_ENABLED",
+						Value: "true",
+					},
+				)
+				ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
+					corev1.ContainerPort{
+						Name:          "metrics",
+						Protocol:      "TCP",
+						ContainerPort: 9002,
+					},
+				)
+			}
+
 			break
 		}
 	}
@@ -786,15 +844,9 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 	return origConf.AuthKey, nil
 }
 
-// tailscaledConfig takes a proxy config, a newly generated auth key if
-// generated and a Secret with the previous proxy state and auth key and
-// returns tailscaled configuration and a hash of that configuration.
-//
-// As of 2024-05-09 it also returns legacy tailscaled config without the
-// later added NoStatefulFilter field to support proxies older than cap95.
-// TODO (irbekrm): remove the legacy config once we no longer need to support
-// versions older than cap94,
-// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
+// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that
+// configuration.
 func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
@@ -803,11 +855,13 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		Locked:              "false",
 		Hostname:            &stsC.Hostname,
 		NoStatefulFiltering: "false",
+		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
 	}
 
 	// For egress proxies only, we need to ensure that stateful filtering is
 	// not in place so that traffic from cluster can be forwarded via
 	// Tailscale IPs.
+	// TODO (irbekrm): set it to true always as this is now the default in core.
 	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	}
@@ -817,6 +871,9 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
 		conf.AdvertiseRoutes = routes
+		if stsC.Connector.isAppConnector {
+			conf.AppConnector.Advertise = true
+		}
 	}
 	if shouldAcceptRoutes(stsC.ProxyClass) {
 		conf.AcceptRoutes = "true"
@@ -831,11 +888,13 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		}
 		conf.AuthKey = key
 	}
+
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[107] = *conf
+
+	// AppConnector config option is only understood by clients of capver 107 and newer.
+	conf.AppConnector = nil
 	capVerConfigs[95] = *conf
-	// legacy config should not contain NoStatefulFiltering field.
-	conf.NoStatefulFiltering.Clear()
-	capVerConfigs[94] = *conf
 	return capVerConfigs, nil
 }
 

cmd/k8s-operator/sts_test.go

@@ -125,10 +125,26 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 			},
 		},
 	}
-	proxyClassMetrics := &tsapi.ProxyClass{
-		Spec: tsapi.ProxyClassSpec{
-			Metrics: &tsapi.Metrics{Enable: true},
-		},
+
+	proxyClassWithMetricsDebug := func(metrics bool, debug *bool) *tsapi.ProxyClass {
+		return &tsapi.ProxyClass{
+			Spec: tsapi.ProxyClassSpec{
+				Metrics: &tsapi.Metrics{Enable: metrics},
+				StatefulSet: func() *tsapi.StatefulSet {
+					if debug == nil {
+						return nil
+					}
+
+					return &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{Enable: *debug},
+							},
+						},
+					}
+				}(),
+			},
+		}
 	}
 
 	var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
@@ -184,7 +200,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 
 	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 2. Test that a ProxyClass with custom labels and annotations for
@@ -197,7 +213,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 3. Test that a ProxyClass with all fields set gets correctly applied
@@ -221,7 +237,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
 	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
@@ -233,16 +249,48 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
-	// 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet.
+	// 5. Metrics enabled defaults to enabling both metrics and debug.
 	wantSS = nonUserspaceProxySS.DeepCopy()
-	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
-	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}}
-	gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
+		corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
+		corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
+		corev1.EnvVar{Name: "TS_METRICS_ENABLED", Value: "true"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{
+		{Name: "debug", Protocol: "TCP", ContainerPort: 9001},
+		{Name: "metrics", Protocol: "TCP", ContainerPort: 9002},
+	}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, nil), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
+
+	// 6. Enable _just_ metrics by explicitly disabling debug.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
+		corev1.EnvVar{Name: "TS_METRICS_ENABLED", Value: "true"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9002}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, ptr.To(false)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
+
+	// 7. Enable _just_ debug without metrics.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
+		corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "debug", Protocol: "TCP", ContainerPort: 9001}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(false, ptr.To(true)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
 	}
 }
 

cmd/k8s-operator/testutils_test.go

@@ -48,6 +48,7 @@ type configOpts struct {
 	clusterTargetDNS                               string
 	subnetRoutes                                   string
 	isExitNode                                     bool
+	isAppConnector                                 bool
 	confFileHash                                   string
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
@@ -69,8 +70,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "false"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
@@ -228,8 +230,9 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 			{Name: "TS_INTERNAL_APP", Value: opts.app},
@@ -356,6 +359,7 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		Locked:       "false",
 		AuthKey:      ptr.To("secret-authkey"),
 		AcceptRoutes: "false",
+		AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
 	}
 	if opts.proxyClass != "" {
 		t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
@@ -370,6 +374,9 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 	if opts.shouldRemoveAuthKey {
 		conf.AuthKey = nil
 	}
+	if opts.isAppConnector {
+		conf.AppConnector = &ipn.AppConnectorPrefs{Advertise: true}
+	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -384,22 +391,23 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 			routes = append(routes, prefix)
 		}
 	}
-	conf.AdvertiseRoutes = routes
-	b, err := json.Marshal(conf)
-	if err != nil {
-		t.Fatalf("error marshalling tailscaled config")
-	}
 	if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	} else {
 		conf.NoStatefulFiltering = "false"
 	}
+	conf.AdvertiseRoutes = routes
+	bnn, err := json.Marshal(conf)
+	if err != nil {
+		t.Fatalf("error marshalling tailscaled config")
+	}
+	conf.AppConnector = nil
 	bn, err := json.Marshal(conf)
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
-	mak.Set(&s.StringData, "tailscaled", string(b))
 	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
+	mak.Set(&s.StringData, "cap-107.hujson", string(bnn))
 	labels := map[string]string{
 		"tailscale.com/managed":              "true",
 		"tailscale.com/parent-resource":      "test",
@@ -650,18 +658,6 @@ func removeTargetPortsFromSvc(svc *corev1.Service) {
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()
-		if len(secret.StringData["tailscaled"]) != 0 {
-			conf := &ipn.ConfigVAlpha{}
-			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
-				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
-			}
-			conf.AuthKey = nil
-			b, err := json.Marshal(conf)
-			if err != nil {
-				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
-			}
-			mak.Set(&secret.StringData, "tailscaled", string(b))
-		}
 		if len(secret.StringData["cap-95.hujson"]) != 0 {
 			conf := &ipn.ConfigVAlpha{}
 			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
@@ -674,5 +670,17 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 			}
 			mak.Set(&secret.StringData, "cap-95.hujson", string(b))
 		}
+		if len(secret.StringData["cap-107.hujson"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["cap-107.hujson"]), conf); err != nil {
+				t.Fatalf("error umarshalling 'cap-107.hujson' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling 'cap-107.huson' contents: %v", err)
+			}
+			mak.Set(&secret.StringData, "cap-107.hujson", string(b))
+		}
 	}
 }

cmd/k8s-operator/tsrecorder_specs.go

@@ -130,6 +130,15 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
 					fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
 				},
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"get",
+					"create",
+					"patch",
+				},
+			},
 		},
 	}
 }
@@ -203,6 +212,14 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar {
 				},
 			},
 		},
+		{
+			Name: "POD_UID",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.uid",
+				},
+			},
+		},
 		{
 			Name:  "TS_STATE",
 			Value: "kube:$(POD_NAME)",

cmd/stund/depaware.txt

@@ -8,7 +8,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
@@ -74,9 +73,9 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/tailcfg
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
         tailscale.com/util/lineiter                                  from tailscale.com/version/distro
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/slicesx                                   from tailscale.com/tailcfg
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
         tailscale.com/version                                        from tailscale.com/envknob+
@@ -133,7 +132,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         crypto/tls                                                   from net/http+
         crypto/x509                                                  from crypto/tls
         crypto/x509/pkix                                             from crypto/x509
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -164,7 +162,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from math/big+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/tailscale/cli/cli.go

@@ -93,8 +93,13 @@ func Run(args []string) (err error) {
 
 	args = CleanUpArgs(args)
 
-	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
-		args = []string{"version"}
+	if len(args) == 1 {
+		switch args[0] {
+		case "-V", "--version":
+			args = []string{"version"}
+		case "help":
+			args = []string{"--help"}
+		}
 	}
 
 	var warnOnce sync.Once

cmd/tailscale/cli/cli_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"net/netip"
 	"reflect"
 	"strings"
@@ -1480,3 +1481,33 @@ func TestParseNLArgs(t *testing.T) {
 		})
 	}
 }
+
+func TestHelpAlias(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	tstest.Replace[io.Writer](t, &Stdout, &stdout)
+	tstest.Replace[io.Writer](t, &Stderr, &stderr)
+
+	gotExit0 := false
+	defer func() {
+		if !gotExit0 {
+			t.Error("expected os.Exit(0) to be called")
+			return
+		}
+		if !strings.Contains(stderr.String(), "SUBCOMMANDS") {
+			t.Errorf("expected help output to contain SUBCOMMANDS; got stderr=%q; stdout=%q", stderr.String(), stdout.String())
+		}
+	}()
+	defer func() {
+		if e := recover(); e != nil {
+			if strings.Contains(fmt.Sprint(e), "unexpected call to os.Exit(0)") {
+				gotExit0 = true
+			} else {
+				t.Errorf("unexpected panic: %v", e)
+			}
+		}
+	}()
+	err := Run([]string{"help"})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}

cmd/tailscale/cli/debug.go

@@ -36,6 +36,7 @@ import (
 	"tailscale.com/hostinfo"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/ipn"
+	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/paths"
@@ -213,6 +214,7 @@ var debugCmd = &ffcli.Command{
 				fs := newFlagSet("watch-ipn")
 				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
 				fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status")
+				fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags")
 				fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap")
 				fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever")
 				return fs
@@ -500,6 +502,7 @@ var watchIPNArgs struct {
 	netmap         bool
 	initial        bool
 	showPrivateKey bool
+	rateLimit      bool
 	count          int
 }
 
@@ -511,6 +514,9 @@ func runWatchIPN(ctx context.Context, args []string) error {
 	if !watchIPNArgs.showPrivateKey {
 		mask |= ipn.NotifyNoPrivateKeys
 	}
+	if watchIPNArgs.rateLimit {
+		mask |= ipn.NotifyRateLimit
+	}
 	watcher, err := localClient.WatchIPNBus(ctx, mask)
 	if err != nil {
 		return err
@@ -845,6 +851,11 @@ func runTS2021(ctx context.Context, args []string) error {
 		logf = log.Printf
 	}
 
+	netMon, err := netmon.New(logger.WithPrefix(logf, "netmon: "))
+	if err != nil {
+		return fmt.Errorf("creating netmon: %w", err)
+	}
+
 	noiseDialer := &controlhttp.Dialer{
 		Hostname:        ts2021Args.host,
 		HTTPPort:        "80",
@@ -854,6 +865,7 @@ func runTS2021(ctx context.Context, args []string) error {
 		ProtocolVersion: uint16(ts2021Args.version),
 		Dialer:          dialFunc,
 		Logf:            logf,
+		NetMon:          netMon,
 	}
 	const tries = 2
 	for i := range tries {

cmd/tailscale/cli/risks.go

@@ -17,11 +17,18 @@ import (
 )
 
 var (
-	riskTypes   []string
-	riskLoseSSH = registerRiskType("lose-ssh")
-	riskAll     = registerRiskType("all")
+	riskTypes           []string
+	riskLoseSSH         = registerRiskType("lose-ssh")
+	riskMacAppConnector = registerRiskType("mac-app-connector")
+	riskAll             = registerRiskType("all")
 )
 
+const riskMacAppConnectorMessage = `
+You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. 
+
+Do not use a macOS app connector for any mission-critical purposes. For the best experience, Linux is the only recommended platform for app connectors.
+`
+
 func registerRiskType(riskType string) string {
 	riskTypes = append(riskTypes, riskType)
 	return riskType

cmd/tailscale/cli/set.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/netip"
 	"os/exec"
+	"runtime"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
@@ -203,6 +204,12 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 		}
 	}
 
+	if runtime.GOOS == "darwin" && maskedPrefs.AppConnector.Advertise {
+		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, setArgs.acceptedRisks); err != nil {
+			return err
+		}
+	}
+
 	if maskedPrefs.RunSSHSet {
 		wantSSH, haveSSH := maskedPrefs.RunSSH, curPrefs.RunSSH
 		if err := presentSSHToggleRisk(wantSSH, haveSSH, setArgs.acceptedRisks); err != nil {

cmd/tailscale/cli/syspolicy.go

@@ -98,9 +98,9 @@ func printPolicySettings(policy *setting.Snapshot) {
 			origin = o.String()
 		}
 		if err := setting.Error(); err != nil {
-			fmt.Fprintf(w, "%s\t%s\t\t{%s}\n", k, origin, err)
+			fmt.Fprintf(w, "%s\t%s\t\t{%v}\n", k, origin, err)
 		} else {
-			fmt.Fprintf(w, "%s\t%s\t%s\t\n", k, origin, setting.Value())
+			fmt.Fprintf(w, "%s\t%s\t%v\t\n", k, origin, setting.Value())
 		}
 	}
 	w.Flush()

cmd/tailscale/cli/up.go

@@ -379,6 +379,12 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, err
 	}
 
+	if runtime.GOOS == "darwin" && env.upArgs.advertiseConnector {
+		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, env.upArgs.acceptedRisks); err != nil {
+			return false, nil, err
+		}
+	}
+
 	if env.upArgs.forceReauth && isSSHOverTailscale() {
 		if err := presentRiskToUser(riskLoseSSH, `You are connected over Tailscale; this action will result in your SSH session disconnecting.`, env.upArgs.acceptedRisks); err != nil {
 			return false, nil, err

cmd/tailscaled/tailscaled_windows.go

@@ -134,14 +134,13 @@ func runWindowsService(pol *logpolicy.Policy) error {
 		logger.Logf(log.Printf).JSON(1, "SupportInfo", osdiag.SupportInfo(osdiag.LogSupportInfoReasonStartup))
 	}()
 
-	if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions {
-		syslog, err := eventlog.Open(serviceName)
-		if err == nil {
-			syslogf = func(format string, args ...any) {
+	if syslog, err := eventlog.Open(serviceName); err == nil {
+		syslogf = func(format string, args ...any) {
+			if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions {
 				syslog.Info(0, fmt.Sprintf(format, args...))
 			}
-			defer syslog.Close()
 		}
+		defer syslog.Close()
 	}
 
 	syslogf("Service entering svc.Run")
@@ -160,10 +159,7 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	changes <- svc.Status{State: svc.StartPending}
 	syslogf("Service start pending")
 
-	svcAccepts := svc.AcceptStop
-	if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock {
-		svcAccepts |= svc.AcceptSessionChange
-	}
+	svcAccepts := svc.AcceptStop | svc.AcceptSessionChange
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -371,13 +367,15 @@ func handleSessionChange(chgRequest svc.ChangeRequest) {
 		return
 	}
 
-	log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.")
-	go func() {
-		err := dns.Flush()
-		if err != nil {
-			log.Printf("Error flushing DNS on session unlock: %v", err)
-		}
-	}()
+	if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock {
+		log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.")
+		go func() {
+			err := dns.Flush()
+			if err != nil {
+				log.Printf("Error flushing DNS on session unlock: %v", err)
+			}
+		}()
+	}
 }
 
 var (

cmd/tsconnect/wasm/wasm_js.go

@@ -272,8 +272,8 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 						name = p.Hostinfo().Hostname()
 					}
 					addrs := make([]string, p.Addresses().Len())
-					for i := range p.Addresses().Len() {
-						addrs[i] = p.Addresses().At(i).Addr().String()
+					for i, ap := range p.Addresses().All() {
+						addrs[i] = ap.Addr().String()
 					}
 					return jsNetMapPeerNode{
 						jsNetMapNode: jsNetMapNode{
@@ -589,8 +589,8 @@ func mapSlice[T any, M any](a []T, f func(T) M) []M {
 
 func mapSliceView[T any, M any](a views.Slice[T], f func(T) M) []M {
 	n := make([]M, a.Len())
-	for i := range a.Len() {
-		n[i] = f(a.At(i))
+	for i, v := range a.All() {
+		n[i] = f(v)
 	}
 	return n
 }

control/controlclient/noise.go

@@ -17,7 +17,6 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlhttp"
-	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/dnscache"
@@ -30,7 +29,6 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/multierr"
 	"tailscale.com/util/singleflight"
-	"tailscale.com/util/testenv"
 )
 
 // NoiseClient provides a http.Client to connect to tailcontrol over
@@ -107,11 +105,6 @@ type NoiseOpts struct {
 	DialPlan func() *tailcfg.ControlDialPlan
 }
 
-// controlIsPlaintext is whether we should assume that the controlplane is only accessible
-// over plaintext HTTP (as the first hop, before the ts2021 encryption begins).
-// This is used by some tests which don't have a real TLS certificate.
-var controlIsPlaintext = envknob.RegisterBool("TS_CONTROL_IS_PLAINTEXT_HTTP")
-
 // NewNoiseClient returns a new noiseClient for the provided server and machine key.
 // serverURL is of the form https://<host>:<port> (no trailing slash).
 //
@@ -129,7 +122,7 @@ func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) {
 		if u.Scheme == "http" {
 			httpPort = port
 			httpsPort = "443"
-			if (testenv.InTest() || controlIsPlaintext()) && (u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost") {
+			if u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost" {
 				httpsPort = ""
 			}
 		} else {

control/controlclient/sign_supported.go

@@ -13,7 +13,6 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"sync"
 	"time"
 
 	"github.com/tailscale/certstore"
@@ -22,11 +21,6 @@ import (
 	"tailscale.com/util/syspolicy"
 )
 
-var getMachineCertificateSubjectOnce struct {
-	sync.Once
-	v string // Subject of machine certificate to search for
-}
-
 // getMachineCertificateSubject returns the exact name of a Subject that needs
 // to be present in an identity's certificate chain to sign a RegisterRequest,
 // formatted as per pkix.Name.String(). The Subject may be that of the identity
@@ -37,11 +31,8 @@ var getMachineCertificateSubjectOnce struct {
 //
 // Example: "CN=Tailscale Inc Test Root CA,OU=Tailscale Inc Test Certificate Authority,O=Tailscale Inc,ST=ON,C=CA"
 func getMachineCertificateSubject() string {
-	getMachineCertificateSubjectOnce.Do(func() {
-		getMachineCertificateSubjectOnce.v, _ = syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
-	})
-
-	return getMachineCertificateSubjectOnce.v
+	machineCertSubject, _ := syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
+	return machineCertSubject
 }
 
 var (

control/controlhttp/client.go

@@ -246,7 +246,7 @@ func (a *Dialer) dial(ctx context.Context) (*ClientConn, error) {
 		results[i].conn = nil // so we don't close it in the defer
 		return conn, nil
 	}
-	merr := multierr.New(errs...)
+	merr := multierr.New(multierr.DeduplicateContextErrors(errs)...)
 
 	// If we get here, then we didn't get anywhere with our dial plan; fall back to just using DNS.
 	a.logf("controlhttp: failed dialing using DialPlan, falling back to DNS; errs=%s", merr.Error())

control/controlhttp/constants.go

@@ -76,6 +76,8 @@ type Dialer struct {
 	// dropped.
 	Logf logger.Logf
 
+	// NetMon is the [netmon.Monitor] to use for this Dialer. It must be
+	// non-nil.
 	NetMon *netmon.Monitor
 
 	// HealthTracker, if non-nil, is the health tracker to use.

control/controlhttp/http_test.go

@@ -25,6 +25,7 @@ import (
 	"tailscale.com/control/controlbase"
 	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/control/controlhttp/controlhttpserver"
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/socks5"
@@ -228,6 +229,7 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 		omitCertErrorLogging: true,
 		testFallbackDelay:    fallbackDelay,
 		Clock:                clock,
+		HealthTracker:        new(health.Tracker),
 	}
 
 	if param.httpInDial {
@@ -729,6 +731,7 @@ func TestDialPlan(t *testing.T) {
 				omitCertErrorLogging: true,
 				testFallbackDelay:    50 * time.Millisecond,
 				Clock:                clock,
+				HealthTracker:        new(health.Tracker),
 			}
 
 			conn, err := a.dial(ctx)

docs/k8s/proxy.yaml

@@ -44,6 +44,14 @@ spec:
       value: "{{TS_DEST_IP}}"
     - name: TS_AUTH_ONCE
       value: "true"
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
     securityContext:
       capabilities:
         add:

docs/k8s/role.yaml

@@ -13,3 +13,6 @@ rules:
   resourceNames: ["{{TS_KUBE_SECRET}}"]
   resources: ["secrets"]
   verbs: ["get", "update", "patch"]
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["events"]
+  verbs: ["get", "create", "patch"]

docs/k8s/sidecar.yaml

@@ -26,6 +26,14 @@ spec:
           name: tailscale-auth
           key: TS_AUTHKEY
           optional: true
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
     securityContext:
       capabilities:
         add:

docs/k8s/subnet.yaml

@@ -28,6 +28,14 @@ spec:
           optional: true
     - name: TS_ROUTES
       value: "{{TS_ROUTES}}"
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
     securityContext:
       capabilities:
         add:

docs/k8s/userspace-sidecar.yaml

@@ -27,3 +27,11 @@ spec:
           name: tailscale-auth
           key: TS_AUTHKEY
           optional: true
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid

docs/windows/policy/en-US/tailscale.adml

@@ -15,16 +15,18 @@
             <string id="SINCE_V1_58">Tailscale version 1.58.0 and later</string>
             <string id="SINCE_V1_62">Tailscale version 1.62.0 and later</string>
             <string id="SINCE_V1_74">Tailscale version 1.74.0 and later</string>
+            <string id="SINCE_V1_78">Tailscale version 1.78.0 and later</string>
             <string id="Tailscale_Category">Tailscale</string>
             <string id="UI_Category">UI customization</string>
             <string id="Settings_Category">Settings</string>
             <string id="LoginURL">Require using a specific Tailscale coordination server</string>
             <string id="LoginURL_Help"><![CDATA[This policy can be used to require the use of a particular Tailscale coordination server.
-See https://tailscale.com/kb/1315/mdm-keys#set-a-custom-control-server-url for more details.
 
-If you configure this policy, set it to the URL of your coordination server, beginning with https:// and ending with no trailing slash. If blank or "https://controlplane.tailscale.com", the default coordination server will be required.
+If you enable this policy, set it to the URL of your coordination server, beginning with https:// and ending with no trailing slash. If blank or "https://controlplane.tailscale.com", the default coordination server will be required.
 
-If you disable this policy, the Tailscale SaaS coordination server will be used by default, but a non-standard Tailscale coordination server can be configured using the CLI.]]></string>
+If you disable or do not configure this policy, the Tailscale SaaS coordination server will be used by default, but a non-standard Tailscale coordination server can be configured using the CLI.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-a-custom-control-server-url for more details.]]></string>
             <string id="LogTarget">Require using a specific Tailscale log server</string>
             <string id="LogTarget_Help"><![CDATA[This policy can be used to require the use of a non-standard log server.
 Please note that using a non-standard log server will limit Tailscale Support's ability to diagnose problems.
@@ -34,15 +36,16 @@ If you configure this policy, set it to the URL of your log server, beginning wi
 If you disable this policy, the Tailscale standard log server will be used by default, but a non-standard Tailscale log server can be configured using the TS_LOG_TARGET environment variable.]]></string>
             <string id="Tailnet">Specify which Tailnet should be used for Login</string>
             <string id="Tailnet_Help"><![CDATA[This policy can be used to suggest or require a specific tailnet when opening the login page.
-See https://tailscale.com/kb/1315/mdm-keys#set-a-suggested-or-required-tailnet for more details.
 
 To suggest a tailnet at login time, set this to the name of the tailnet, as shown in the top-left of the admin panel, such as "example.com". That tailnet's SSO button will be shown prominently, along with the option to select a different tailnet.
 
 To require logging in to a particular tailnet, add the "required:" prefix, such as "required:example.com". The result is similar to the suggested tailnet but there will be no option to choose a different tailnet.
 
-If you configure this policy, set it to the name of the tailnet, possibly with the "required:" prefix, as described above.
+If you enable this policy, set it to the name of the tailnet, possibly with the "required:" prefix, as described above.
 
-If you disable this policy, the standard login page will be used.]]></string>
+If you disable or do not configure this policy, the standard login page will be used.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-a-suggested-or-required-tailnet for more details.]]></string>
             <string id="AuthKey">Specify the auth key to authenticate devices without user interaction</string>
             <string id="AuthKey_Help"><![CDATA[This policy allows specifying the default auth key to be used when registering new devices without requiring sign-in via a web browser, unless the user specifies a different auth key via the CLI.
 
@@ -52,85 +55,101 @@ While MDM solutions tend to offer better control over who can access the policy
 
 Only consider this option after carefully reviewing the organization's security posture. For example, ensure you configure the auth keys specifically for the tag of the device and that access control policies only grant necessary access between the tailnet and the tagged device. Additionally, consider using short-lived auth keys, one-time auth keys (with one GPO/MDM configuration per device), Device Approval, and/or Tailnet lock to minimize risk. If you suspect an auth key has been compromised, revoke the auth key immediately.
 
-If you configure this policy setting and specify an auth key, it will be used to authenticate the device unless the device is already logged in or an auth key is explicitly specified via the CLI.
+If you enable this policy setting and specify an auth key, it will be used to authenticate the device unless the device is already logged in or an auth key is explicitly specified via the CLI.
 
 If you disable or do not configure this policy setting, an interactive user login will be required..
 
 See https://tailscale.com/kb/1315/mdm-keys#set-an-auth-key for more details.]]></string>
             <string id="ExitNodeID">Require using a specific Exit Node</string>
             <string id="ExitNodeID_Help"><![CDATA[This policy can be used to require always using the specified Exit Node whenever the Tailscale client is connected.
-See https://tailscale.com/kb/1315/mdm-keys#force-an-exit-node-to-always-be-used and https://tailscale.com/kb/1103/exit-nodes for more details.
-
-If you enable this policy, set it to the ID of an exit node. The ID is visible on the Machines page of the admin console, or can be queried using the Tailscale API. If the specified exit node is unavailable, this device will have no Internet access unless Tailscale is disconnected.
+    
+If you enable this policy, set it to the ID of an exit node. The ID is visible on the Machines page of the admin console, or can be queried using the Tailscale API. If the specified exit node is unavailable, this device will have no Internet access unless Tailscale is disconnected. Alternatively, you can set it to "auto:any" (without quotes), which allows the Tailscale client to automatically select the most suitable exit node.
 
 If you disable this policy or supply an empty exit node ID, then usage of exit nodes will be disallowed.
 
-If you do not configure this policy, no exit node will be used by default but an exit node (if one is available and permitted by ACLs) can be chosen by the user if desired.]]></string>
+If you do not configure this policy, no exit node will be used by default but an exit node (if one is available and permitted by ACLs) can be chosen by the user if desired.
+
+See https://tailscale.com/kb/1315/mdm-keys#force-an-exit-node-to-always-be-used and https://tailscale.com/kb/1103/exit-nodes for more details.]]></string>
+            <string id="AllowedSuggestedExitNodes">Limit automated Exit Node suggestions to specific nodes</string>
+            <string id="AllowedSuggestedExitNodes_Help"><![CDATA[This policy setting allows configuring a pool of exit nodes from which the Tailscale client will automatically select the most suitable suggested exit node when required. The suggested exit node is displayed in the GUI and CLI and is automatically selected and enforced when the "Require using a specific Exit Node" policy setting is enabled and set to "auto:any".
+
+If you enable this policy setting, suggestions will be limited to exit nodes with the specified IDs. The IDs are visible on the Machines page of the admin console, or can be queried using the Tailscale API.
+
+If you disable or do not configure this policy setting, no limitations will apply, and all available exit nodes will be considered when selecting the most suitable suggested node.
+
+See https://tailscale.com/kb/1315/mdm-keys#suggest-allowed-forced-exit-nodes and https://tailscale.com/kb/1103/exit-nodes for more details.]]></string>
             <string id="AllowIncomingConnections">Allow incoming connections</string>
             <string id="AllowIncomingConnections_Help"><![CDATA[This policy can be used to require that the Allow Incoming Connections setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-whether-to-allow-incoming-connections and https://tailscale.com/kb/1072/client-preferences#allow-incoming-connections for more details.
 
 If you enable this policy, then Allow Incoming Connections is always enabled and the menu option is hidden.
 
 If you disable this policy, then Allow Incoming Connections is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Allow Incoming Connections depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Allow Incoming Connections depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-whether-to-allow-incoming-connections and https://tailscale.com/kb/1072/client-preferences#allow-incoming-connections for more details.]]></string>
             <string id="UnattendedMode">Run Tailscale in Unattended Mode</string>
             <string id="UnattendedMode_Help"><![CDATA[This policy can be used to require that the Run Unattended setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-unattended-mode and https://tailscale.com/kb/1088/run-unattended for more details.
 
 If you enable this policy, then Run Unattended is always enabled and the menu option is hidden.
 
 If you disable this policy, then Run Unattended is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Run Unattended depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Run Unattended depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-unattended-mode and https://tailscale.com/kb/1088/run-unattended for more details.]]></string>
             <string id="ExitNodeAllowLANAccess">Allow Local Network Access when an Exit Node is in use</string>
             <string id="ExitNodeAllowLANAccess_Help"><![CDATA[This policy can be used to require that the Allow Local Network Access setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#toggle-local-network-access-when-an-exit-node-is-in-use and https://tailscale.com/kb/1103/exit-nodes#step-4-use-the-exit-node for more details.
 
 If you enable this policy, then Allow Local Network Access is always enabled and the menu option is hidden.
 
 If you disable this policy, then Allow Local Network Access is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Allow Local Network Access depends on what is selected in the Exit Node submenu.]]></string>
+If you do not configure this policy, then Allow Local Network Access depends on what is selected in the Exit Node submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#toggle-local-network-access-when-an-exit-node-is-in-use and https://tailscale.com/kb/1103/exit-nodes#step-4-use-the-exit-node for more details.]]></string>
             <string id="UseTailscaleDNSSettings">Use Tailscale DNS Settings</string>
             <string id="UseTailscaleDNSSettings_Help"><![CDATA[This policy can be used to require that Use Tailscale DNS is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-uses-tailscale-dns-settings for more details.
 
 If you enable this policy, then Use Tailscale DNS is always enabled and the menu option is hidden.
 
 If you disable this policy, then Use Tailscale DNS is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Use Tailscale DNS depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Use Tailscale DNS depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-uses-tailscale-dns-settings for more details.]]></string>
             <string id="UseTailscaleSubnets">Use Tailscale Subnets</string>
             <string id="UseTailscaleSubnets_Help"><![CDATA[This policy can be used to require that Use Tailscale Subnets is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-accepts-tailscale-subnets or https://tailscale.com/kb/1019/subnets for more details.
 
 If you enable this policy, then Use Tailscale Subnets is always enabled and the menu option is hidden.
 
 If you disable this policy, then Use Tailscale Subnets is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Use Tailscale Subnets depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Use Tailscale Subnets depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-accepts-tailscale-subnets or https://tailscale.com/kb/1019/subnets for more details.]]></string>
             <string id="InstallUpdates">Automatically install updates</string>
             <string id="InstallUpdates_Help"><![CDATA[This policy can be used to require that Automatically Install Updates is configured a certain way.
-See https://tailscale.com/kb/1067/update#auto-updates for more details.
 
 If you enable this policy, then Automatically Install Updates is always enabled and the menu option is hidden.
 
 If you disable this policy, then Automatically Install Updates is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Automatically Install Updates depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Automatically Install Updates depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1067/update#auto-updates for more details.]]></string>
             <string id="AdvertiseExitNode">Run Tailscale as an Exit Node</string>
             <string id="AdvertiseExitNode_Help"><![CDATA[This policy can be used to require that Run Exit Node is configured a certain way.
-See https://tailscale.com/kb/1103/exit-nodes for more details.
 
 If you enable this policy, then Run Exit Node is always enabled and the menu option is hidden.
 
 If you disable this policy, then Run Exit Node is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Run Exit Node depends on what is selected in the Exit Node submenu.]]></string>
-            <string id="AdminPanel">Show the "Admin Panel" menu item</string>
-            <string id="AdminPanel_Help"><![CDATA[This policy can be used to show or hide the Admin Console item in the Tailscale Menu.
+If you do not configure this policy, then Run Exit Node depends on what is selected in the Exit Node submenu.
+
+See https://tailscale.com/kb/1103/exit-nodes for more details.]]></string>
+            <string id="AdminConsole">Show the "Admin Console" menu item</string>
+            <string id="AdminConsole_Help"><![CDATA[This policy can be used to show or hide the Admin Console item in the Tailscale Menu.
 
 If you enable or don't configure this policy, the Admin Console item will be shown in the Tailscale menu when available.
 
@@ -143,49 +162,55 @@ If you enable or don't configure this policy, the Network Devices submenu will b
 If you disable this policy, the Network Devices submenu will be hidden from the Tailscale menu. This does not affect other devices' visibility in the CLI.]]></string>
             <string id="TestMenu">Show the "Debug" submenu</string>
             <string id="TestMenu_Help"><![CDATA[This policy can be used to show or hide the Debug submenu of the Tailscale menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-debug-menu for more details.
 
 If you enable or don't configure this policy, the Debug submenu will be shown in the Tailscale menu when opened while holding Ctrl.
 
-If you disable this policy, the Debug submenu will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Debug submenu will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-debug-menu for more details.]]></string>
             <string id="UpdateMenu">Show the "Update Available" menu item</string>
             <string id="UpdateMenu_Help"><![CDATA[This policy can be used to show or hide the Update Available item in the Tailscale Menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-update-menu for more details.
 
 If you enable or don't configure this policy, the Update Available item will be shown in the Tailscale menu when there is an update.
 
-If you disable this policy, the Update Available item will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Update Available item will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-update-menu for more details.]]></string>
             <string id="RunExitNode">Show the "Run Exit Node" menu item</string>
             <string id="RunExitNode_Help"><![CDATA[This policy can be used to show or hide the Run Exit Node item in the Exit Node submenu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-run-as-exit-node-menu-item for more details.
 This does not affect using the CLI to enable or disable advertising an exit node. If you wish to enable or disable this feature, see the Run Exit Node policy in the Settings category.
 
 If you enable or don't configure this policy, the Run Exit Node item will be shown in the Exit Node submenu.
 
-If you disable this policy, the Run Exit Node item will be hidden from the Exit Node submenu.]]></string>
+If you disable this policy, the Run Exit Node item will be hidden from the Exit Node submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-run-as-exit-node-menu-item for more details.]]></string>
             <string id="PreferencesMenu">Show the "Preferences" submenu</string>
             <string id="PreferencesMenu_Help"><![CDATA[This policy can be used to show or hide the Preferences submenu of the Tailscale menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-preferences-menu for more details.
 This does not affect using the CLI to modify that menu's preferences. If you wish to control those, look at the policies in the Settings category.
 
 If you enable or don't configure this policy, the Preferences submenu will be shown in the Tailscale menu.
 
-If you disable this policy, the Preferences submenu will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Preferences submenu will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-preferences-menu for more details.]]></string>
             <string id="ExitNodesPicker">Show the "Exit Node" submenu</string>
             <string id="ExitNodesPicker_Help"><![CDATA[This policy can be used to show or hide the Exit Node submenu of the Tailscale menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-exit-node-picker for more details.
 This does not affect using the CLI to select or stop using an exit node. If you wish to control exit node usage, look at the "Require using a specific Exit Node" policy in the Settings category.
 
 If you enable or don't configure this policy, the Exit Node submenu will be shown in the Tailscale menu.
 
-If you disable this policy, the Exit Node submenu will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Exit Node submenu will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-exit-node-picker for more details.]]></string>
             <string id="KeyExpirationNotice">Specify a custom key expiration notification time</string>
             <string id="KeyExpirationNotice_Help"><![CDATA[This policy can be used to configure how soon the notification appears before key expiry.
-See https://tailscale.com/kb/1315/mdm-keys#set-the-key-expiration-notice-period for more details.
 
 If you enable this policy and supply a valid time interval, the key expiry notification will begin to display when the current key has less than that amount of time remaining.
 
-If you disable or don't configure this policy, the default time period will be used (as of Tailscale 1.56, this is 24 hours).]]></string>
+If you disable or don't configure this policy, the default time period will be used (as of Tailscale 1.56, this is 24 hours).
+
+See https://tailscale.com/kb/1315/mdm-keys#set-the-key-expiration-notice-period for more details.]]></string>
             <string id="LogSCMInteractions">Log extra details about service events</string>
             <string id="LogSCMInteractions_Help"><![CDATA[This policy can be used to enable additional logging related to Service Control Manager for debugging purposes.
 This should only be enabled if recommended by Tailscale Support.
@@ -202,13 +227,14 @@ If you enable this policy, the DNS cache will be flushed on session unlock in ad
 If you disable or don't configure this policy, the DNS cache is managed normally.]]></string>
             <string id="PostureChecking">Collect data for posture checking</string>
             <string id="PostureChecking_Help"><![CDATA[This policy can be used to require that the Posture Checking setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#enable-gathering-device-posture-data and https://tailscale.com/kb/1326/device-identity for more details.
 
 If you enable this policy, then data collection is always enabled.
 
 If you disable this policy, then data collection is always disabled.
 
-If you do not configure this policy, then data collection depends on if it has been enabled from the CLI (as of Tailscale 1.56), it may be present in the GUI in later versions.]]></string>
+If you do not configure this policy, then data collection depends on if it has been enabled from the CLI (as of Tailscale 1.56), it may be present in the GUI in later versions.
+
+See https://tailscale.com/kb/1315/mdm-keys#enable-gathering-device-posture-data and https://tailscale.com/kb/1326/device-identity for more details.]]></string>
             <string id="ManagedBy">Show the "Managed By {Organization}" menu item</string>
             <string id="ManagedBy_Help"><![CDATA[Use this policy to configure the “Managed By {Organization}” item in the Tailscale Menu.
 
@@ -244,6 +270,9 @@ See https://tailscale.com/kb/1315/mdm-keys#set-your-organization-name for more d
                     <label>Exit Node:</label>
                 </textBox>
             </presentation>
+            <presentation id="AllowedSuggestedExitNodes">
+                <listBox refId="AllowedSuggestedExitNodesList">Target IDs:</listBox>
+            </presentation>
             <presentation id="ManagedBy">
                 <textBox refId="ManagedByOrganization">
                     <label>Organization Name:</label>

docs/windows/policy/tailscale.admx

@@ -50,6 +50,10 @@
                   displayName="$(string.SINCE_V1_74)">
         <and><reference ref="TAILSCALE_PRODUCT"/></and>
       </definition>
+      <definition name="SINCE_V1_78"
+                  displayName="$(string.SINCE_V1_78)">
+        <and><reference ref="TAILSCALE_PRODUCT"/></and>
+      </definition>
     </definitions>
   </supportedOn>
   <categories>
@@ -94,7 +98,14 @@
       <parentCategory ref="Settings_Category" />
       <supportedOn ref="SINCE_V1_56" />
       <elements>
-        <text id="ExitNodeIDPrompt" valueName="ExitNodeID" required="true" />
+        <text id="ExitNodeIDPrompt" valueName="ExitNodeID" required="true" />>
+      </elements>
+    </policy>
+    <policy name="AllowedSuggestedExitNodes" class="Machine" displayName="$(string.AllowedSuggestedExitNodes)" explainText="$(string.AllowedSuggestedExitNodes_Help)" presentation="$(presentation.AllowedSuggestedExitNodes)" key="Software\Policies\Tailscale\AllowedSuggestedExitNodes">
+      <parentCategory ref="Settings_Category" />
+      <supportedOn ref="SINCE_V1_78" />
+      <elements>
+        <list id="AllowedSuggestedExitNodesList" />
       </elements>
     </policy>
     <policy name="AllowIncomingConnections" class="Machine" displayName="$(string.AllowIncomingConnections)" explainText="$(string.AllowIncomingConnections_Help)" key="Software\Policies\Tailscale" valueName="AllowIncomingConnections">
@@ -197,7 +208,7 @@
         <decimal value="0" />
       </disabledValue>
     </policy>
-    <policy name="AdminPanel" class="Machine" displayName="$(string.AdminPanel)" explainText="$(string.AdminPanel_Help)" key="Software\Policies\Tailscale" valueName="AdminPanel">
+    <policy name="AdminConsole" class="Both" displayName="$(string.AdminConsole)" explainText="$(string.AdminConsole_Help)" key="Software\Policies\Tailscale" valueName="AdminConsole">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -207,7 +218,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="NetworkDevices" class="Machine" displayName="$(string.NetworkDevices)" explainText="$(string.NetworkDevices_Help)" key="Software\Policies\Tailscale" valueName="NetworkDevices">
+    <policy name="NetworkDevices" class="Both" displayName="$(string.NetworkDevices)" explainText="$(string.NetworkDevices_Help)" key="Software\Policies\Tailscale" valueName="NetworkDevices">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -217,7 +228,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="TestMenu" class="Machine" displayName="$(string.TestMenu)" explainText="$(string.TestMenu_Help)" key="Software\Policies\Tailscale" valueName="TestMenu">
+    <policy name="TestMenu" class="Both" displayName="$(string.TestMenu)" explainText="$(string.TestMenu_Help)" key="Software\Policies\Tailscale" valueName="TestMenu">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -227,7 +238,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="UpdateMenu" class="Machine" displayName="$(string.UpdateMenu)" explainText="$(string.UpdateMenu_Help)" key="Software\Policies\Tailscale" valueName="UpdateMenu">
+    <policy name="UpdateMenu" class="Both" displayName="$(string.UpdateMenu)" explainText="$(string.UpdateMenu_Help)" key="Software\Policies\Tailscale" valueName="UpdateMenu">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -237,7 +248,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="RunExitNode" class="Machine" displayName="$(string.RunExitNode)" explainText="$(string.RunExitNode_Help)" key="Software\Policies\Tailscale" valueName="RunExitNode">
+    <policy name="RunExitNode" class="Both" displayName="$(string.RunExitNode)" explainText="$(string.RunExitNode_Help)" key="Software\Policies\Tailscale" valueName="RunExitNode">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -247,7 +258,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="PreferencesMenu" class="Machine" displayName="$(string.PreferencesMenu)" explainText="$(string.PreferencesMenu_Help)" key="Software\Policies\Tailscale" valueName="PreferencesMenu">
+    <policy name="PreferencesMenu" class="Both" displayName="$(string.PreferencesMenu)" explainText="$(string.PreferencesMenu_Help)" key="Software\Policies\Tailscale" valueName="PreferencesMenu">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -257,7 +268,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="ExitNodesPicker" class="Machine" displayName="$(string.ExitNodesPicker)" explainText="$(string.ExitNodesPicker_Help)" key="Software\Policies\Tailscale" valueName="ExitNodesPicker">
+    <policy name="ExitNodesPicker" class="Both" displayName="$(string.ExitNodesPicker)" explainText="$(string.ExitNodesPicker_Help)" key="Software\Policies\Tailscale" valueName="ExitNodesPicker">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -267,7 +278,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="ManagedBy" class="Machine" displayName="$(string.ManagedBy)" explainText="$(string.ManagedBy_Help)" presentation="$(presentation.ManagedBy)" key="Software\Policies\Tailscale">
+    <policy name="ManagedBy" class="Both" displayName="$(string.ManagedBy)" explainText="$(string.ManagedBy_Help)" presentation="$(presentation.ManagedBy)" key="Software\Policies\Tailscale">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_62" />
       <elements>
@@ -276,7 +287,7 @@
         <text id="ManagedBySupportURL" valueName="ManagedByURL" />
       </elements>
     </policy>
-    <policy name="KeyExpirationNotice" class="Machine" displayName="$(string.KeyExpirationNotice)" explainText="$(string.KeyExpirationNotice_Help)" presentation="$(presentation.KeyExpirationNotice)" key="Software\Policies\Tailscale">
+    <policy name="KeyExpirationNotice" class="Both" displayName="$(string.KeyExpirationNotice)" explainText="$(string.KeyExpirationNotice_Help)" presentation="$(presentation.KeyExpirationNotice)" key="Software\Policies\Tailscale">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_50" />
       <elements>

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/golangci/golangci-lint v1.57.1
 	github.com/google/go-cmp v0.6.0
-	github.com/google/go-containerregistry v0.18.0
+	github.com/google/go-containerregistry v0.20.2
 	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
@@ -55,7 +55,7 @@ require (
 	github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86
 	github.com/jsimonetti/rtnetlink v1.4.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
-	github.com/klauspost/compress v1.17.4
+	github.com/klauspost/compress v1.17.11
 	github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-isatty v0.0.20
@@ -80,12 +80,12 @@ require (
 	github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
-	github.com/tailscale/mkctr v0.0.0-20240628074852-17ca944da6ba
+	github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10
 	github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7
 	github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4
 	github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1
 	github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6
-	github.com/tailscale/wireguard-go v0.0.0-20240905161824-799c1978fafc
+	github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3
 	github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e
 	github.com/tc-hib/winres v0.2.1
 	github.com/tcnksm/go-httpstat v0.2.0
@@ -100,8 +100,8 @@ require (
 	golang.org/x/mod v0.19.0
 	golang.org/x/net v0.27.0
 	golang.org/x/oauth2 v0.16.0
-	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.22.0
+	golang.org/x/sync v0.9.0
+	golang.org/x/sys v0.27.0
 	golang.org/x/term v0.22.0
 	golang.org/x/time v0.5.0
 	golang.org/x/tools v0.23.0
@@ -125,7 +125,7 @@ require (
 	github.com/Antonboom/testifylint v1.2.0 // indirect
 	github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect
 	github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
@@ -138,7 +138,7 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect
 	github.com/dave/brenda v1.1.0 // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/ghostiam/protogetter v0.3.5 // indirect
@@ -160,10 +160,10 @@ require (
 	github.com/ykadowak/zerologlint v0.1.5 // indirect
 	go-simpler.org/musttag v0.9.0 // indirect
 	go-simpler.org/sloglint v0.5.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel v1.22.0 // indirect
-	go.opentelemetry.io/otel/metric v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
 	go.uber.org/automaxprocs v1.5.3 // indirect
 	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
 )
@@ -220,10 +220,10 @@ require (
 	github.com/daixiang0/gci v0.12.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/denis-tingaikin/go-header v0.5.0 // indirect
-	github.com/docker/cli v25.0.0+incompatible // indirect
+	github.com/docker/cli v27.3.1+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/docker v26.1.4+incompatible // indirect
-	github.com/docker/docker-credential-helpers v0.8.1 // indirect
+	github.com/docker/docker v27.3.1+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.8.2 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/ettle/strcase v0.2.0 // indirect
@@ -322,7 +322,7 @@ require (
 	github.com/nunnatsa/ginkgolinter v0.16.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc6 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -376,7 +376,7 @@ require (
 	github.com/ultraware/funlen v0.1.0 // indirect
 	github.com/ultraware/whitespace v0.1.0 // indirect
 	github.com/uudashr/gocognit v1.1.2 // indirect
-	github.com/vbatts/tar-split v0.11.5 // indirect
+	github.com/vbatts/tar-split v0.11.6 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect

go.sum

@@ -79,8 +79,8 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
-github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
-github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
@@ -277,16 +277,16 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
-github.com/docker/cli v25.0.0+incompatible h1:zaimaQdnX7fYWFqzN88exE9LDEvRslexpFowZBX6GoQ=
-github.com/docker/cli v25.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v27.3.1+incompatible h1:qEGdFBF3Xu6SCvCYhc7CzaQTlBmqDuzxPDpigSyeKQQ=
+github.com/docker/cli v27.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU=
-github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo=
-github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
-github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
-github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI=
+github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
+github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
@@ -490,8 +490,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.18.0 h1:ShE7erKNPqRh5ue6Z9DUOlk04WsnFWPO6YGr3OxnfoQ=
-github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
+github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo=
+github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -627,8 +627,8 @@ github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkHAIKE/contextcheck v1.1.4 h1:B6zAaLhOEEcjvUgIYEqystmnFk1Oemn8bvJhbt0GMb8=
 github.com/kkHAIKE/contextcheck v1.1.4/go.mod h1:1+i/gWqokIa+dm31mqGLZhZJ7Uh44DJGZVmr6QRBNJg=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -749,8 +749,8 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
 github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU=
-github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
@@ -931,8 +931,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw=
 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8=
-github.com/tailscale/mkctr v0.0.0-20240628074852-17ca944da6ba h1:uNo1VCm/xg4alMkIKo8RWTKNx5y1otfVOcKbp+irkL4=
-github.com/tailscale/mkctr v0.0.0-20240628074852-17ca944da6ba/go.mod h1:DxnqIXBplij66U2ZkL688xy07q97qQ83P+TVueLiHq4=
+github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 h1:ZB47BgnHcEHQJODkDubs5ZiNeJxMhcgzefV3lykRwVQ=
+github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10/go.mod h1:iDx/0Rr9VV/KanSUDpJ6I/ROf0sQ7OqljXc/esl0UIA=
 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4ZoF094vE6iYTLDl0qCiKzYXlL6UeWObU=
 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
 github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 h1:Gz0rz40FvFVLTBk/K8UNAenb36EbDSnh+q7Z9ldcC8w=
@@ -941,8 +941,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M=
 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y=
-github.com/tailscale/wireguard-go v0.0.0-20240905161824-799c1978fafc h1:cezaQN9pvKVaw56Ma5qr/G646uKIYP0yQf+OyWN/okc=
-github.com/tailscale/wireguard-go v0.0.0-20240905161824-799c1978fafc/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3 h1:dmoPb3dG27tZgMtrvqfD/LW4w7gA6BSWl8prCPNmkCQ=
+github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA=
@@ -981,8 +981,8 @@ github.com/ultraware/whitespace v0.1.0 h1:O1HKYoh0kIeqE8sFqZf1o0qbORXUCOQFrlaQyZ
 github.com/ultraware/whitespace v0.1.0/go.mod h1:/se4r3beMFNmewJ4Xmz0nMQ941GJt+qmSHGP9emHYe0=
 github.com/uudashr/gocognit v1.1.2 h1:l6BAEKJqQH2UpKAPKdMfZf5kE4W/2xk8pfU1OVLvniI=
 github.com/uudashr/gocognit v1.1.2/go.mod h1:aAVdLURqcanke8h3vg35BC++eseDm66Z7KmchI5et4k=
-github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
-github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
+github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs=
+github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
@@ -1022,20 +1022,20 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
-go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
-go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
-go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
@@ -1176,8 +1176,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1239,8 +1239,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

go.toolchain.rev

@@ -1 +1 @@
-bf15628b759344c6fc7763795a405ba65b8be5d7
+96578f73d04e1a231fa2a495ad3fa97747785bc6

ipn/backend.go

@@ -73,6 +73,8 @@ const (
 	NotifyInitialOutgoingFiles // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles
 
 	NotifyInitialHealthState // if set, the first Notify message (sent immediately) will contain the current health.State of the client
+
+	NotifyRateLimit // if set, rate limit spammy netmap updates to every few seconds
 )
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend
@@ -100,7 +102,6 @@ type Notify struct {
 	NetMap        *netmap.NetworkMap // if non-nil, the new or current netmap
 	Engine        *EngineStatus      // if non-nil, the new or current wireguard stats
 	BrowseToURL   *string            // if non-nil, UI should open a browser right now
-	BackendLogID  *string            // if non-nil, the public logtail ID used by backend
 
 	// FilesWaiting if non-nil means that files are buffered in
 	// the Tailscale daemon and ready for local transfer to the
@@ -173,9 +174,6 @@ func (n Notify) String() string {
 	if n.BrowseToURL != nil {
 		sb.WriteString("URL=<...> ")
 	}
-	if n.BackendLogID != nil {
-		sb.WriteString("BackendLogID ")
-	}
 	if n.FilesWaiting != nil {
 		sb.WriteString("FilesWaiting ")
 	}

ipn/ipnlocal/bus.go

@@ -0,0 +1,160 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package ipnlocal
+
+import (
+	"context"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/tstime"
+)
+
+type rateLimitingBusSender struct {
+	fn              func(*ipn.Notify) (keepGoing bool)
+	lastFlush       time.Time           // last call to fn, or zero value if none
+	interval        time.Duration       // 0 to flush immediately; non-zero to rate limit sends
+	clock           tstime.DefaultClock // non-nil for testing
+	didSendTestHook func()              // non-nil for testing
+
+	// pending, if non-nil, is the pending notification that we
+	// haven't sent yet. We own this memory to mutate.
+	pending *ipn.Notify
+
+	// flushTimer is non-nil if the timer is armed.
+	flushTimer  tstime.TimerController // effectively a *time.Timer
+	flushTimerC <-chan time.Time       // ... said ~Timer's C chan
+}
+
+func (s *rateLimitingBusSender) close() {
+	if s.flushTimer != nil {
+		s.flushTimer.Stop()
+	}
+}
+
+func (s *rateLimitingBusSender) flushChan() <-chan time.Time {
+	return s.flushTimerC
+}
+
+func (s *rateLimitingBusSender) flush() (keepGoing bool) {
+	if n := s.pending; n != nil {
+		s.pending = nil
+		return s.flushNotify(n)
+	}
+	return true
+}
+
+func (s *rateLimitingBusSender) flushNotify(n *ipn.Notify) (keepGoing bool) {
+	s.lastFlush = s.clock.Now()
+	return s.fn(n)
+}
+
+// send conditionally sends n to the underlying fn, possibly rate
+// limiting it, depending on whether s.interval is set, and whether
+// n is a notable notification that the client (typically a GUI) would
+// want to act on (render) immediately.
+//
+// It returns whether the caller should keep looping.
+//
+// The passed-in memory 'n' is owned by the caller and should
+// not be mutated.
+func (s *rateLimitingBusSender) send(n *ipn.Notify) (keepGoing bool) {
+	if s.interval <= 0 {
+		// No rate limiting case.
+		return s.fn(n)
+	}
+	if isNotableNotify(n) {
+		// Notable notifications are always sent immediately.
+		// But first send any boring one that was pending.
+		// TODO(bradfitz): there might be a boring one pending
+		// with a NetMap or Engine field that is redundant
+		// with the new one (n) with NetMap or Engine populated.
+		// We should clear the pending one's NetMap/Engine in
+		// that case. Or really, merge the two, but mergeBoringNotifies
+		// only handles the case of both sides being boring.
+		// So for now, flush both.
+		if !s.flush() {
+			return false
+		}
+		return s.flushNotify(n)
+	}
+	s.pending = mergeBoringNotifies(s.pending, n)
+	d := s.clock.Now().Sub(s.lastFlush)
+	if d > s.interval {
+		return s.flush()
+	}
+	nextFlushIn := s.interval - d
+	if s.flushTimer == nil {
+		s.flushTimer, s.flushTimerC = s.clock.NewTimer(nextFlushIn)
+	} else {
+		s.flushTimer.Reset(nextFlushIn)
+	}
+	return true
+}
+
+func (s *rateLimitingBusSender) Run(ctx context.Context, ch <-chan *ipn.Notify) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case n, ok := <-ch:
+			if !ok {
+				return
+			}
+			if !s.send(n) {
+				return
+			}
+			if f := s.didSendTestHook; f != nil {
+				f()
+			}
+		case <-s.flushChan():
+			if !s.flush() {
+				return
+			}
+		}
+	}
+}
+
+// mergeBoringNotify merges new notify 'src' into possibly-nil 'dst',
+// either mutating 'dst' or allocating a new one if 'dst' is nil,
+// returning the merged result.
+//
+// dst and src must both be "boring" (i.e. not notable per isNotifiableNotify).
+func mergeBoringNotifies(dst, src *ipn.Notify) *ipn.Notify {
+	if dst == nil {
+		dst = &ipn.Notify{Version: src.Version}
+	}
+	if src.NetMap != nil {
+		dst.NetMap = src.NetMap
+	}
+	if src.Engine != nil {
+		dst.Engine = src.Engine
+	}
+	return dst
+}
+
+// isNotableNotify reports whether n is a "notable" notification that
+// should be sent on the IPN bus immediately (e.g. to GUIs) without
+// rate limiting it for a few seconds.
+//
+// It effectively reports whether n contains any field set that's
+// not NetMap or Engine.
+func isNotableNotify(n *ipn.Notify) bool {
+	if n == nil {
+		return false
+	}
+	return n.State != nil ||
+		n.SessionID != "" ||
+		n.BrowseToURL != nil ||
+		n.LocalTCPPort != nil ||
+		n.ClientVersion != nil ||
+		n.Prefs != nil ||
+		n.ErrMessage != nil ||
+		n.LoginFinished != nil ||
+		!n.DriveShares.IsNil() ||
+		n.Health != nil ||
+		len(n.IncomingFiles) > 0 ||
+		len(n.OutgoingFiles) > 0 ||
+		n.FilesWaiting != nil
+}

ipn/ipnlocal/bus_test.go

@@ -0,0 +1,220 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package ipnlocal
+
+import (
+	"context"
+	"reflect"
+	"slices"
+	"testing"
+	"time"
+
+	"tailscale.com/drive"
+	"tailscale.com/ipn"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/views"
+)
+
+func TestIsNotableNotify(t *testing.T) {
+	tests := []struct {
+		name   string
+		notify *ipn.Notify
+		want   bool
+	}{
+		{"nil", nil, false},
+		{"empty", &ipn.Notify{}, false},
+		{"version", &ipn.Notify{Version: "foo"}, false},
+		{"netmap", &ipn.Notify{NetMap: new(netmap.NetworkMap)}, false},
+		{"engine", &ipn.Notify{Engine: new(ipn.EngineStatus)}, false},
+	}
+
+	// Then for all other fields, assume they're notable.
+	// We use reflect to catch fields that might be added in the future without
+	// remembering to update the [isNotableNotify] function.
+	rt := reflect.TypeFor[ipn.Notify]()
+	for i := range rt.NumField() {
+		n := &ipn.Notify{}
+		sf := rt.Field(i)
+		switch sf.Name {
+		case "_", "NetMap", "Engine", "Version":
+			// Already covered above or not applicable.
+			continue
+		case "DriveShares":
+			n.DriveShares = views.SliceOfViews[*drive.Share, drive.ShareView](make([]*drive.Share, 1))
+		default:
+			rf := reflect.ValueOf(n).Elem().Field(i)
+			switch rf.Kind() {
+			case reflect.Pointer:
+				rf.Set(reflect.New(rf.Type().Elem()))
+			case reflect.String:
+				rf.SetString("foo")
+			case reflect.Slice:
+				rf.Set(reflect.MakeSlice(rf.Type(), 1, 1))
+			default:
+				t.Errorf("unhandled field kind %v for %q", rf.Kind(), sf.Name)
+			}
+		}
+
+		tests = append(tests, struct {
+			name   string
+			notify *ipn.Notify
+			want   bool
+		}{
+			name:   "field-" + rt.Field(i).Name,
+			notify: n,
+			want:   true,
+		})
+	}
+
+	for _, tt := range tests {
+		if got := isNotableNotify(tt.notify); got != tt.want {
+			t.Errorf("%v: got %v; want %v", tt.name, got, tt.want)
+		}
+	}
+}
+
+type rateLimitingBusSenderTester struct {
+	tb    testing.TB
+	got   []*ipn.Notify
+	clock *tstest.Clock
+	s     *rateLimitingBusSender
+}
+
+func (st *rateLimitingBusSenderTester) init() {
+	if st.s != nil {
+		return
+	}
+	st.clock = tstest.NewClock(tstest.ClockOpts{
+		Start: time.Unix(1731777537, 0), // time I wrote this test :)
+	})
+	st.s = &rateLimitingBusSender{
+		clock: tstime.DefaultClock{Clock: st.clock},
+		fn: func(n *ipn.Notify) bool {
+			st.got = append(st.got, n)
+			return true
+		},
+	}
+}
+
+func (st *rateLimitingBusSenderTester) send(n *ipn.Notify) {
+	st.tb.Helper()
+	st.init()
+	if !st.s.send(n) {
+		st.tb.Fatal("unexpected send failed")
+	}
+}
+
+func (st *rateLimitingBusSenderTester) advance(d time.Duration) {
+	st.tb.Helper()
+	st.clock.Advance(d)
+	select {
+	case <-st.s.flushChan():
+		if !st.s.flush() {
+			st.tb.Fatal("unexpected flush failed")
+		}
+	default:
+	}
+}
+
+func TestRateLimitingBusSender(t *testing.T) {
+	nm1 := &ipn.Notify{NetMap: new(netmap.NetworkMap)}
+	nm2 := &ipn.Notify{NetMap: new(netmap.NetworkMap)}
+	eng1 := &ipn.Notify{Engine: new(ipn.EngineStatus)}
+	eng2 := &ipn.Notify{Engine: new(ipn.EngineStatus)}
+
+	t.Run("unbuffered", func(t *testing.T) {
+		st := &rateLimitingBusSenderTester{tb: t}
+		st.send(nm1)
+		st.send(nm2)
+		st.send(eng1)
+		st.send(eng2)
+		if !slices.Equal(st.got, []*ipn.Notify{nm1, nm2, eng1, eng2}) {
+			t.Errorf("got %d items; want 4 specific ones, unmodified", len(st.got))
+		}
+	})
+
+	t.Run("buffered", func(t *testing.T) {
+		st := &rateLimitingBusSenderTester{tb: t}
+		st.init()
+		st.s.interval = 1 * time.Second
+		st.send(&ipn.Notify{Version: "initial"})
+		if len(st.got) != 1 {
+			t.Fatalf("got %d items; expected 1 (first to flush immediately)", len(st.got))
+		}
+		st.send(nm1)
+		st.send(nm2)
+		st.send(eng1)
+		st.send(eng2)
+		if len(st.got) != 1 {
+			if len(st.got) != 1 {
+				t.Fatalf("got %d items; expected still just that first 1", len(st.got))
+			}
+		}
+
+		// But moving the clock should flush the rest, collasced into one new one.
+		st.advance(5 * time.Second)
+		if len(st.got) != 2 {
+			t.Fatalf("got %d items; want 2", len(st.got))
+		}
+		gotn := st.got[1]
+		if gotn.NetMap != nm2.NetMap {
+			t.Errorf("got wrong NetMap; got %p", gotn.NetMap)
+		}
+		if gotn.Engine != eng2.Engine {
+			t.Errorf("got wrong Engine; got %p", gotn.Engine)
+		}
+		if t.Failed() {
+			t.Logf("failed Notify was: %v", logger.AsJSON(gotn))
+		}
+	})
+
+	// Test the Run method
+	t.Run("run", func(t *testing.T) {
+		st := &rateLimitingBusSenderTester{tb: t}
+		st.init()
+		st.s.interval = 1 * time.Second
+		st.s.lastFlush = st.clock.Now() // pretend we just flushed
+
+		flushc := make(chan *ipn.Notify, 1)
+		st.s.fn = func(n *ipn.Notify) bool {
+			flushc <- n
+			return true
+		}
+		didSend := make(chan bool, 2)
+		st.s.didSendTestHook = func() { didSend <- true }
+		waitSend := func() {
+			select {
+			case <-didSend:
+			case <-time.After(5 * time.Second):
+				t.Error("timeout waiting for call to send")
+			}
+		}
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		incoming := make(chan *ipn.Notify, 2)
+		go func() {
+			incoming <- nm1
+			waitSend()
+			incoming <- nm2
+			waitSend()
+			st.advance(5 * time.Second)
+			select {
+			case n := <-flushc:
+				if n.NetMap != nm2.NetMap {
+					t.Errorf("got wrong NetMap; got %p", n.NetMap)
+				}
+			case <-time.After(10 * time.Second):
+				t.Error("timeout")
+			}
+			cancel()
+		}()
+
+		st.s.Run(ctx, incoming)
+	})
+}

ipn/ipnlocal/c2n.go

@@ -77,6 +77,9 @@ var c2nHandlers = map[methodAndPath]c2nHandler{
 
 	// Linux netfilter.
 	req("POST /netfilter-kind"): handleC2NSetNetfilterKind,
+
+	// VIP services.
+	req("GET /vip-services"): handleC2NVIPServicesGet,
 }
 
 type c2nHandler func(*LocalBackend, http.ResponseWriter, *http.Request)
@@ -269,6 +272,12 @@ func handleC2NSetNetfilterKind(b *LocalBackend, w http.ResponseWriter, r *http.R
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func handleC2NVIPServicesGet(b *LocalBackend, w http.ResponseWriter, r *http.Request) {
+	b.logf("c2n: GET /vip-services received")
+
+	json.NewEncoder(w).Encode(b.VIPServices())
+}
+
 func handleC2NUpdateGet(b *LocalBackend, w http.ResponseWriter, r *http.Request) {
 	b.logf("c2n: GET /update received")
 

ipn/ipnlocal/drive.go

@@ -354,9 +354,8 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem
 
 				// Check that the peer is allowed to share with us.
 				addresses := peer.Addresses()
-				for i := range addresses.Len() {
-					addr := addresses.At(i)
-					capsMap := b.PeerCaps(addr.Addr())
+				for _, p := range addresses.All() {
+					capsMap := b.PeerCaps(p.Addr())
 					if capsMap.HasCapability(tailcfg.PeerCapabilityTaildriveSharer) {
 						return true
 					}

ipn/ipnlocal/local.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -86,7 +87,6 @@ import (
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/empty"
 	"tailscale.com/types/key"
-	"tailscale.com/types/lazy"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
@@ -105,6 +105,7 @@ import (
 	"tailscale.com/util/rands"
 	"tailscale.com/util/set"
 	"tailscale.com/util/syspolicy"
+	"tailscale.com/util/syspolicy/rsop"
 	"tailscale.com/util/systemd"
 	"tailscale.com/util/testenv"
 	"tailscale.com/util/uniq"
@@ -177,27 +178,28 @@ type watchSession struct {
 // state machine generates events back out to zero or more components.
 type LocalBackend struct {
 	// Elements that are thread-safe or constant after construction.
-	ctx                   context.Context    // canceled by Close
-	ctxCancel             context.CancelFunc // cancels ctx
-	logf                  logger.Logf        // general logging
-	keyLogf               logger.Logf        // for printing list of peers on change
-	statsLogf             logger.Logf        // for printing peers stats on change
-	sys                   *tsd.System
-	health                *health.Tracker // always non-nil
-	metrics               metrics
-	e                     wgengine.Engine // non-nil; TODO(bradfitz): remove; use sys
-	store                 ipn.StateStore  // non-nil; TODO(bradfitz): remove; use sys
-	dialer                *tsdial.Dialer  // non-nil; TODO(bradfitz): remove; use sys
-	pushDeviceToken       syncs.AtomicValue[string]
-	backendLogID          logid.PublicID
-	unregisterNetMon      func()
-	unregisterHealthWatch func()
-	portpoll              *portlist.Poller // may be nil
-	portpollOnce          sync.Once        // guards starting readPoller
-	varRoot               string           // or empty if SetVarRoot never called
-	logFlushFunc          func()           // or nil if SetLogFlusher wasn't called
-	em                    *expiryManager   // non-nil
-	sshAtomicBool         atomic.Bool
+	ctx                      context.Context    // canceled by Close
+	ctxCancel                context.CancelFunc // cancels ctx
+	logf                     logger.Logf        // general logging
+	keyLogf                  logger.Logf        // for printing list of peers on change
+	statsLogf                logger.Logf        // for printing peers stats on change
+	sys                      *tsd.System
+	health                   *health.Tracker // always non-nil
+	metrics                  metrics
+	e                        wgengine.Engine // non-nil; TODO(bradfitz): remove; use sys
+	store                    ipn.StateStore  // non-nil; TODO(bradfitz): remove; use sys
+	dialer                   *tsdial.Dialer  // non-nil; TODO(bradfitz): remove; use sys
+	pushDeviceToken          syncs.AtomicValue[string]
+	backendLogID             logid.PublicID
+	unregisterNetMon         func()
+	unregisterHealthWatch    func()
+	unregisterSysPolicyWatch func()
+	portpoll                 *portlist.Poller // may be nil
+	portpollOnce             sync.Once        // guards starting readPoller
+	varRoot                  string           // or empty if SetVarRoot never called
+	logFlushFunc             func()           // or nil if SetLogFlusher wasn't called
+	em                       *expiryManager   // non-nil
+	sshAtomicBool            atomic.Bool
 	// webClientAtomicBool controls whether the web client is running. This should
 	// be true unless the disable-web-client node attribute has been set.
 	webClientAtomicBool atomic.Bool
@@ -353,6 +355,12 @@ type LocalBackend struct {
 	// avoid unnecessary churn between multiple equally-good options.
 	lastSuggestedExitNode tailcfg.StableNodeID
 
+	// allowedSuggestedExitNodes is a set of exit nodes permitted by the most recent
+	// [syspolicy.AllowedSuggestedExitNodes] value. The allowedSuggestedExitNodesMu
+	// mutex guards access to this set.
+	allowedSuggestedExitNodesMu sync.Mutex
+	allowedSuggestedExitNodes   set.Set[tailcfg.StableNodeID]
+
 	// refreshAutoExitNode indicates if the exit node should be recomputed when the next netcheck report is available.
 	refreshAutoExitNode bool
 
@@ -409,7 +417,7 @@ type clientGen func(controlclient.Options) (controlclient.Client, error)
 // but is not actually running.
 //
 // If dialer is nil, a new one is made.
-func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, loginFlags controlclient.LoginFlags) (*LocalBackend, error) {
+func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, loginFlags controlclient.LoginFlags) (_ *LocalBackend, err error) {
 	e := sys.Engine.Get()
 	store := sys.StateStore.Get()
 	dialer := sys.Dialer.Get()
@@ -484,6 +492,15 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
 		}
 	}
 
+	if b.unregisterSysPolicyWatch, err = b.registerSysPolicyWatch(); err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			b.unregisterSysPolicyWatch()
+		}
+	}()
+
 	netMon := sys.NetMon.Get()
 	b.sockstatLogger, err = sockstatlog.NewLogger(logpolicy.LogsDir(logf), logf, logID, netMon, sys.HealthTracker())
 	if err != nil {
@@ -980,6 +997,7 @@ func (b *LocalBackend) Shutdown() {
 
 	b.unregisterNetMon()
 	b.unregisterHealthWatch()
+	b.unregisterSysPolicyWatch()
 	if cc != nil {
 		cc.Shutdown()
 	}
@@ -1488,10 +1506,10 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control
 			b.logf("SetControlClientStatus failed to select auto exit node: %v", err)
 		}
 	}
-	if setExitNodeID(prefs, curNetMap, b.lastSuggestedExitNode) {
+	if applySysPolicy(prefs, b.lastSuggestedExitNode) {
 		prefsChanged = true
 	}
-	if applySysPolicy(prefs) {
+	if setExitNodeID(prefs, curNetMap) {
 		prefsChanged = true
 	}
 
@@ -1657,12 +1675,37 @@ var preferencePolicies = []preferencePolicyInfo{
 
 // applySysPolicy overwrites configured preferences with policies that may be
 // configured by the system administrator in an OS-specific way.
-func applySysPolicy(prefs *ipn.Prefs) (anyChange bool) {
+func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID) (anyChange bool) {
 	if controlURL, err := syspolicy.GetString(syspolicy.ControlURL, prefs.ControlURL); err == nil && prefs.ControlURL != controlURL {
 		prefs.ControlURL = controlURL
 		anyChange = true
 	}
 
+	if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" {
+		exitNodeID := tailcfg.StableNodeID(exitNodeIDStr)
+		if shouldAutoExitNode() && lastSuggestedExitNode != "" {
+			exitNodeID = lastSuggestedExitNode
+		}
+		// Note: when exitNodeIDStr == "auto" && lastSuggestedExitNode == "",
+		// then exitNodeID is now "auto" which will never match a peer's node ID.
+		// When there is no a peer matching the node ID, traffic will blackhole,
+		// preventing accidental non-exit-node usage when a policy is in effect that requires an exit node.
+		if prefs.ExitNodeID != exitNodeID || prefs.ExitNodeIP.IsValid() {
+			anyChange = true
+		}
+		prefs.ExitNodeID = exitNodeID
+		prefs.ExitNodeIP = netip.Addr{}
+	} else if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" {
+		exitNodeIP, err := netip.ParseAddr(exitNodeIPStr)
+		if exitNodeIP.IsValid() && err == nil {
+			if prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP {
+				anyChange = true
+			}
+			prefs.ExitNodeID = ""
+			prefs.ExitNodeIP = exitNodeIP
+		}
+	}
+
 	for _, opt := range preferencePolicies {
 		if po, err := syspolicy.GetPreferenceOption(opt.key); err == nil {
 			curVal := opt.get(prefs.View())
@@ -1677,6 +1720,54 @@ func applySysPolicy(prefs *ipn.Prefs) (anyChange bool) {
 	return anyChange
 }
 
+// registerSysPolicyWatch subscribes to syspolicy change notifications
+// and immediately applies the effective syspolicy settings to the current profile.
+func (b *LocalBackend) registerSysPolicyWatch() (unregister func(), err error) {
+	if unregister, err = syspolicy.RegisterChangeCallback(b.sysPolicyChanged); err != nil {
+		return nil, fmt.Errorf("syspolicy: LocalBacked failed to register policy change callback: %v", err)
+	}
+	if prefs, anyChange := b.applySysPolicy(); anyChange {
+		b.logf("syspolicy: changed initial profile prefs: %v", prefs.Pretty())
+	}
+	b.refreshAllowedSuggestions()
+	return unregister, nil
+}
+
+// applySysPolicy overwrites the current profile's preferences with policies
+// that may be configured by the system administrator in an OS-specific way.
+//
+// b.mu must not be held.
+func (b *LocalBackend) applySysPolicy() (_ ipn.PrefsView, anyChange bool) {
+	unlock := b.lockAndGetUnlock()
+	prefs := b.pm.CurrentPrefs().AsStruct()
+	if !applySysPolicy(prefs, b.lastSuggestedExitNode) {
+		unlock.UnlockEarly()
+		return prefs.View(), false
+	}
+	return b.setPrefsLockedOnEntry(prefs, unlock), true
+}
+
+// sysPolicyChanged is a callback triggered by syspolicy when it detects
+// a change in one or more syspolicy settings.
+func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) {
+	if policy.HasChanged(syspolicy.AllowedSuggestedExitNodes) {
+		b.refreshAllowedSuggestions()
+		// Re-evaluate exit node suggestion now that the policy setting has changed.
+		b.mu.Lock()
+		_, err := b.suggestExitNodeLocked(nil)
+		b.mu.Unlock()
+		if err != nil && !errors.Is(err, ErrNoPreferredDERP) {
+			b.logf("failed to select auto exit node: %v", err)
+		}
+		// If [syspolicy.ExitNodeID] is set to `auto:any`, the suggested exit node ID
+		// will be used when [applySysPolicy] updates the current profile's prefs.
+	}
+
+	if prefs, anyChange := b.applySysPolicy(); anyChange {
+		b.logf("syspolicy: changed profile prefs: %v", prefs.Pretty())
+	}
+}
+
 var _ controlclient.NetmapDeltaUpdater = (*LocalBackend)(nil)
 
 // UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater.
@@ -1769,30 +1860,7 @@ func (b *LocalBackend) updateNetmapDeltaLocked(muts []netmap.NodeMutation) (hand
 
 // setExitNodeID updates prefs to reference an exit node by ID, rather
 // than by IP. It returns whether prefs was mutated.
-func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap, lastSuggestedExitNode tailcfg.StableNodeID) (prefsChanged bool) {
-	if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" {
-		exitNodeID := tailcfg.StableNodeID(exitNodeIDStr)
-		if shouldAutoExitNode() && lastSuggestedExitNode != "" {
-			exitNodeID = lastSuggestedExitNode
-		}
-		// Note: when exitNodeIDStr == "auto" && lastSuggestedExitNode == "", then exitNodeID is now "auto" which will never match a peer's node ID.
-		// When there is no a peer matching the node ID, traffic will blackhole, preventing accidental non-exit-node usage when a policy is in effect that requires an exit node.
-		changed := prefs.ExitNodeID != exitNodeID || prefs.ExitNodeIP.IsValid()
-		prefs.ExitNodeID = exitNodeID
-		prefs.ExitNodeIP = netip.Addr{}
-		return changed
-	}
-
-	oldExitNodeID := prefs.ExitNodeID
-	if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" {
-		exitNodeIP, err := netip.ParseAddr(exitNodeIPStr)
-		if exitNodeIP.IsValid() && err == nil {
-			prefsChanged = prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP
-			prefs.ExitNodeID = ""
-			prefs.ExitNodeIP = exitNodeIP
-		}
-	}
-
+func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) {
 	if nm == nil {
 		// No netmap, can't resolve anything.
 		return false
@@ -1810,9 +1878,9 @@ func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap, lastSuggestedExitNod
 		prefsChanged = true
 	}
 
+	oldExitNodeID := prefs.ExitNodeID
 	for _, peer := range nm.Peers {
-		for i := range peer.Addresses().Len() {
-			addr := peer.Addresses().At(i)
+		for _, addr := range peer.Addresses().All() {
 			if !addr.IsSingleIP() || addr.Addr() != prefs.ExitNodeIP {
 				continue
 			}
@@ -1820,7 +1888,7 @@ func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap, lastSuggestedExitNod
 			// reference it directly for next time.
 			prefs.ExitNodeID = peer.StableID()
 			prefs.ExitNodeIP = netip.Addr{}
-			return oldExitNodeID != prefs.ExitNodeID
+			return prefsChanged || oldExitNodeID != prefs.ExitNodeID
 		}
 	}
 
@@ -2157,10 +2225,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 
 	blid := b.backendLogID.String()
 	b.logf("Backend: logs: be:%v fe:%v", blid, opts.FrontendLogID)
-	b.sendToLocked(ipn.Notify{
-		BackendLogID: &blid,
-		Prefs:        &prefs,
-	}, allClients)
+	b.sendToLocked(ipn.Notify{Prefs: &prefs}, allClients)
 
 	if !loggedOut && (b.hasNodeKeyLocked() || confWantRunning) {
 		// If we know that we're either logged in or meant to be
@@ -2783,20 +2848,17 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 		go b.pollRequestEngineStatus(ctx)
 	}
 
-	// TODO(marwan-at-work): check err
 	// TODO(marwan-at-work): streaming background logs?
 	defer b.DeleteForegroundSession(sessionID)
 
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case n := <-ch:
-			if !fn(n) {
-				return
-			}
-		}
+	sender := &rateLimitingBusSender{fn: fn}
+	defer sender.close()
+
+	if mask&ipn.NotifyRateLimit != 0 {
+		sender.interval = 3 * time.Second
 	}
+
+	sender.Run(ctx, ch)
 }
 
 // pollRequestEngineStatus calls b.e.RequestStatus every 2 seconds until ctx
@@ -3678,11 +3740,16 @@ func updateExitNodeUsageWarning(p ipn.PrefsView, state *netmon.State, healthTrac
 }
 
 func (b *LocalBackend) checkExitNodePrefsLocked(p *ipn.Prefs) error {
+	tryingToUseExitNode := p.ExitNodeIP.IsValid() || p.ExitNodeID != ""
+	if !tryingToUseExitNode {
+		return nil
+	}
+
 	if err := featureknob.CanUseExitNode(); err != nil {
 		return err
 	}
 
-	if (p.ExitNodeIP.IsValid() || p.ExitNodeID != "") && p.AdvertisesExitNode() {
+	if p.AdvertisesExitNode() {
 		return errors.New("Cannot advertise an exit node and use an exit node at the same time.")
 	}
 	return nil
@@ -3850,12 +3917,12 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce)
 	if oldp.Valid() {
 		newp.Persist = oldp.Persist().AsStruct() // caller isn't allowed to override this
 	}
-	// setExitNodeID returns whether it updated b.prefs, but
-	// everything in this function treats b.prefs as completely new
-	// anyway. No-op if no exit node resolution is needed.
-	setExitNodeID(newp, netMap, b.lastSuggestedExitNode)
-	// applySysPolicy does likewise so we can also ignore its return value.
-	applySysPolicy(newp)
+	// applySysPolicyToPrefsLocked returns whether it updated newp,
+	// but everything in this function treats b.prefs as completely new
+	// anyway, so its return value can be ignored here.
+	applySysPolicy(newp, b.lastSuggestedExitNode)
+	// setExitNodeID does likewise. No-op if no exit node resolution is needed.
+	setExitNodeID(newp, netMap)
 	// We do this to avoid holding the lock while doing everything else.
 
 	oldHi := b.hostinfo
@@ -3892,10 +3959,14 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce)
 	}
 
 	prefs := newp.View()
-	if err := b.pm.SetPrefs(prefs, ipn.NetworkProfile{
-		MagicDNSName: b.netMap.MagicDNSSuffix(),
-		DomainName:   b.netMap.DomainName(),
-	}); err != nil {
+	np := b.pm.CurrentProfile().NetworkProfile
+	if netMap != nil {
+		np = ipn.NetworkProfile{
+			MagicDNSName: b.netMap.MagicDNSSuffix(),
+			DomainName:   b.netMap.DomainName(),
+		}
+	}
+	if err := b.pm.SetPrefs(prefs, np); err != nil {
 		b.logf("failed to save new controlclient state: %v", err)
 	}
 
@@ -4889,6 +4960,14 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip
 	}
 	hi.SSH_HostKeys = sshHostKeys
 
+	services := vipServicesFromPrefs(prefs)
+	if len(services) > 0 {
+		buf, _ := json.Marshal(services)
+		hi.ServicesHash = fmt.Sprintf("%02x", sha256.Sum256(buf))
+	} else {
+		hi.ServicesHash = ""
+	}
+
 	// The Hostinfo.WantIngress field tells control whether this node wants to
 	// be wired up for ingress connections. If harmless if it's accidentally
 	// true; the actual policy is controlled in tailscaled by ServeConfig. But
@@ -4997,8 +5076,8 @@ func (b *LocalBackend) enterStateLockedOnEntry(newState ipn.State, unlock unlock
 	case ipn.Running:
 		var addrStrs []string
 		addrs := netMap.GetAddresses()
-		for i := range addrs.Len() {
-			addrStrs = append(addrStrs, addrs.At(i).Addr().String())
+		for _, p := range addrs.All() {
+			addrStrs = append(addrStrs, p.Addr().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrStrs, " "))
 	case ipn.NoState:
@@ -6089,8 +6168,7 @@ func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
 
 func peerAPIPorts(peer tailcfg.NodeView) (p4, p6 uint16) {
 	svcs := peer.Hostinfo().Services()
-	for i := range svcs.Len() {
-		s := svcs.At(i)
+	for _, s := range svcs.All() {
 		switch s.Proto {
 		case tailcfg.PeerAPI4:
 			p4 = s.Port
@@ -6122,8 +6200,7 @@ func peerAPIBase(nm *netmap.NetworkMap, peer tailcfg.NodeView) string {
 
 	var have4, have6 bool
 	addrs := nm.GetAddresses()
-	for i := range addrs.Len() {
-		a := addrs.At(i)
+	for _, a := range addrs.All() {
 		if !a.IsSingleIP() {
 			continue
 		}
@@ -6145,10 +6222,9 @@ func peerAPIBase(nm *netmap.NetworkMap, peer tailcfg.NodeView) string {
 }
 
 func nodeIP(n tailcfg.NodeView, pred func(netip.Addr) bool) netip.Addr {
-	for i := range n.Addresses().Len() {
-		a := n.Addresses().At(i)
-		if a.IsSingleIP() && pred(a.Addr()) {
-			return a.Addr()
+	for _, pfx := range n.Addresses().All() {
+		if pfx.IsSingleIP() && pred(pfx.Addr()) {
+			return pfx.Addr()
 		}
 	}
 	return netip.Addr{}
@@ -6378,8 +6454,8 @@ func peerCanProxyDNS(p tailcfg.NodeView) bool {
 	// If p.Cap is not populated (e.g. older control server), then do the old
 	// thing of searching through services.
 	services := p.Hostinfo().Services()
-	for i := range services.Len() {
-		if s := services.At(i); s.Proto == tailcfg.PeerAPIDNS && s.Port >= 1 {
+	for _, s := range services.All() {
+		if s.Proto == tailcfg.PeerAPIDNS && s.Port >= 1 {
 			return true
 		}
 	}
@@ -7145,7 +7221,7 @@ func (b *LocalBackend) suggestExitNodeLocked(netMap *netmap.NetworkMap) (respons
 	lastReport := b.MagicConn().GetLastNetcheckReport(b.ctx)
 	prevSuggestion := b.lastSuggestedExitNode
 
-	res, err := suggestExitNode(lastReport, netMap, prevSuggestion, randomRegion, randomNode, getAllowedSuggestions())
+	res, err := suggestExitNode(lastReport, netMap, prevSuggestion, randomRegion, randomNode, b.getAllowedSuggestions())
 	if err != nil {
 		return res, err
 	}
@@ -7159,6 +7235,22 @@ func (b *LocalBackend) SuggestExitNode() (response apitype.ExitNodeSuggestionRes
 	return b.suggestExitNodeLocked(nil)
 }
 
+// getAllowedSuggestions returns a set of exit nodes permitted by the most recent
+// [syspolicy.AllowedSuggestedExitNodes] value. Callers must not mutate the returned set.
+func (b *LocalBackend) getAllowedSuggestions() set.Set[tailcfg.StableNodeID] {
+	b.allowedSuggestedExitNodesMu.Lock()
+	defer b.allowedSuggestedExitNodesMu.Unlock()
+	return b.allowedSuggestedExitNodes
+}
+
+// refreshAllowedSuggestions rebuilds the set of permitted exit nodes
+// from the current [syspolicy.AllowedSuggestedExitNodes] value.
+func (b *LocalBackend) refreshAllowedSuggestions() {
+	b.allowedSuggestedExitNodesMu.Lock()
+	defer b.allowedSuggestedExitNodesMu.Unlock()
+	b.allowedSuggestedExitNodes = fillAllowedSuggestions()
+}
+
 // selectRegionFunc returns a DERP region from the slice of candidate regions.
 // The value is returned, not the slice index.
 type selectRegionFunc func(views.Slice[int]) int
@@ -7168,8 +7260,6 @@ type selectRegionFunc func(views.Slice[int]) int
 // choice.
 type selectNodeFunc func(nodes views.Slice[tailcfg.NodeView], last tailcfg.StableNodeID) tailcfg.NodeView
 
-var getAllowedSuggestions = lazy.SyncFunc(fillAllowedSuggestions)
-
 func fillAllowedSuggestions() set.Set[tailcfg.StableNodeID] {
 	nodes, err := syspolicy.GetStringArray(syspolicy.AllowedSuggestedExitNodes, nil)
 	if err != nil {
@@ -7489,3 +7579,42 @@ func maybeUsernameOf(actor ipnauth.Actor) string {
 	}
 	return username
 }
+
+// VIPServices returns the list of tailnet services that this node
+// is serving as a destination for.
+// The returned memory is owned by the caller.
+func (b *LocalBackend) VIPServices() []*tailcfg.VIPService {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return vipServicesFromPrefs(b.pm.CurrentPrefs())
+}
+
+func vipServicesFromPrefs(prefs ipn.PrefsView) []*tailcfg.VIPService {
+	// keyed by service name
+	var services map[string]*tailcfg.VIPService
+
+	// TODO(naman): this envknob will be replaced with service-specific port
+	// information once we start storing that.
+	var allPortsServices []string
+	if env := envknob.String("TS_DEBUG_ALLPORTS_SERVICES"); env != "" {
+		allPortsServices = strings.Split(env, ",")
+	}
+
+	for _, s := range allPortsServices {
+		mak.Set(&services, s, &tailcfg.VIPService{
+			Name:  s,
+			Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+		})
+	}
+
+	for _, s := range prefs.AdvertiseServices().AsSlice() {
+		if services == nil || services[s] == nil {
+			mak.Set(&services, s, &tailcfg.VIPService{
+				Name: s,
+			})
+		}
+		services[s].Active = true
+	}
+
+	return slices.Collect(maps.Values(services))
+}

ipn/ipnlocal/local_test.go

@@ -30,6 +30,7 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive"
 	"tailscale.com/drive/driveimpl"
+	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -457,6 +458,7 @@ func newTestLocalBackendWithSys(t testing.TB, sys *tsd.System) *LocalBackend {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(lb.Shutdown)
 	return lb
 }
 
@@ -1787,10 +1789,13 @@ func TestSetExitNodeIDPolicy(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			b := newTestBackend(t)
 
-			policyStore := source.NewTestStoreOf(t,
-				source.TestSettingOf(syspolicy.ExitNodeID, test.exitNodeID),
-				source.TestSettingOf(syspolicy.ExitNodeIP, test.exitNodeIP),
-			)
+			policyStore := source.NewTestStore(t)
+			if test.exitNodeIDKey {
+				policyStore.SetStrings(source.TestSettingOf(syspolicy.ExitNodeID, test.exitNodeID))
+			}
+			if test.exitNodeIPKey {
+				policyStore.SetStrings(source.TestSettingOf(syspolicy.ExitNodeIP, test.exitNodeIP))
+			}
 			syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
 
 			if test.nm == nil {
@@ -1804,7 +1809,16 @@ func TestSetExitNodeIDPolicy(t *testing.T) {
 			b.netMap = test.nm
 			b.pm = pm
 			b.lastSuggestedExitNode = test.lastSuggestedExitNode
-			changed := setExitNodeID(b.pm.prefs.AsStruct(), test.nm, tailcfg.StableNodeID(test.lastSuggestedExitNode))
+
+			prefs := b.pm.prefs.AsStruct()
+			if changed := applySysPolicy(prefs, test.lastSuggestedExitNode) || setExitNodeID(prefs, test.nm); changed != test.prefsChanged {
+				t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed)
+			}
+
+			// Both [LocalBackend.SetPrefsForTest] and [LocalBackend.EditPrefs]
+			// apply syspolicy settings to the current profile's preferences. Therefore,
+			// we pass the current, unmodified preferences and expect the effective
+			// preferences to change.
 			b.SetPrefsForTest(pm.CurrentPrefs().AsStruct())
 
 			if got := b.pm.prefs.ExitNodeID(); got != tailcfg.StableNodeID(test.exitNodeIDWant) {
@@ -1817,10 +1831,6 @@ func TestSetExitNodeIDPolicy(t *testing.T) {
 			} else if got.String() != test.exitNodeIPWant {
 				t.Errorf("got %v want %v", got, test.exitNodeIPWant)
 			}
-
-			if changed != test.prefsChanged {
-				t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed)
-			}
 		})
 	}
 }
@@ -2330,7 +2340,7 @@ func TestApplySysPolicy(t *testing.T) {
 			t.Run("unit", func(t *testing.T) {
 				prefs := tt.prefs.Clone()
 
-				gotAnyChange := applySysPolicy(prefs)
+				gotAnyChange := applySysPolicy(prefs, "")
 
 				if gotAnyChange && prefs.Equals(&tt.prefs) {
 					t.Errorf("anyChange but prefs is unchanged: %v", prefs.Pretty())
@@ -2478,7 +2488,7 @@ func TestPreferencePolicyInfo(t *testing.T) {
 					prefs := defaultPrefs.AsStruct()
 					pp.set(prefs, tt.initialValue)
 
-					gotAnyChange := applySysPolicy(prefs)
+					gotAnyChange := applySysPolicy(prefs, "")
 
 					if gotAnyChange != tt.wantChange {
 						t.Errorf("anyChange=%v, want %v", gotAnyChange, tt.wantChange)
@@ -3041,12 +3051,10 @@ func deterministicNodeForTest(t testing.TB, want views.Slice[tailcfg.StableNodeI
 		var ret tailcfg.NodeView
 
 		gotIDs := make([]tailcfg.StableNodeID, got.Len())
-		for i := range got.Len() {
-			nv := got.At(i)
+		for i, nv := range got.All() {
 			if !nv.Valid() {
 				t.Fatalf("invalid node at index %v", i)
 			}
-
 			gotIDs[i] = nv.StableID()
 			if nv.StableID() == use {
 				ret = nv
@@ -4110,6 +4118,7 @@ func newLocalBackendWithTestControl(t *testing.T, enableLogging bool, newControl
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(b.Shutdown)
 	b.DisablePortMapperForTest()
 
 	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {
@@ -4466,3 +4475,213 @@ func TestConfigFileReload(t *testing.T) {
 		t.Fatalf("got %q; want %q", hn, "bar")
 	}
 }
+
+func TestGetVIPServices(t *testing.T) {
+	tests := []struct {
+		name       string
+		advertised []string
+		mapped     []string
+		want       []*tailcfg.VIPService
+	}{
+		{
+			"advertised-only",
+			[]string{"svc:abc", "svc:def"},
+			[]string{},
+			[]*tailcfg.VIPService{
+				{
+					Name:   "svc:abc",
+					Active: true,
+				},
+				{
+					Name:   "svc:def",
+					Active: true,
+				},
+			},
+		},
+		{
+			"mapped-only",
+			[]string{},
+			[]string{"svc:abc"},
+			[]*tailcfg.VIPService{
+				{
+					Name:  "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+				},
+			},
+		},
+		{
+			"mapped-and-advertised",
+			[]string{"svc:abc"},
+			[]string{"svc:abc"},
+			[]*tailcfg.VIPService{
+				{
+					Name:   "svc:abc",
+					Active: true,
+					Ports:  []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+				},
+			},
+		},
+		{
+			"mapped-and-advertised-separately",
+			[]string{"svc:def"},
+			[]string{"svc:abc"},
+			[]*tailcfg.VIPService{
+				{
+					Name:  "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+				},
+				{
+					Name:   "svc:def",
+					Active: true,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			envknob.Setenv("TS_DEBUG_ALLPORTS_SERVICES", strings.Join(tt.mapped, ","))
+			prefs := &ipn.Prefs{
+				AdvertiseServices: tt.advertised,
+			}
+			got := vipServicesFromPrefs(prefs.View())
+			slices.SortFunc(got, func(a, b *tailcfg.VIPService) int {
+				return strings.Compare(a.Name, b.Name)
+			})
+			if !reflect.DeepEqual(tt.want, got) {
+				t.Logf("want:")
+				for _, s := range tt.want {
+					t.Logf("%+v", s)
+				}
+				t.Logf("got:")
+				for _, s := range got {
+					t.Logf("%+v", s)
+				}
+				t.Fail()
+				return
+			}
+		})
+	}
+}
+
+func TestUpdatePrefsOnSysPolicyChange(t *testing.T) {
+	const enableLogging = false
+
+	type fieldChange struct {
+		name string
+		want any
+	}
+
+	wantPrefsChanges := func(want ...fieldChange) *wantedNotification {
+		return &wantedNotification{
+			name: "Prefs",
+			cond: func(t testing.TB, actor ipnauth.Actor, n *ipn.Notify) bool {
+				if n.Prefs != nil {
+					prefs := reflect.Indirect(reflect.ValueOf(n.Prefs.AsStruct()))
+					for _, f := range want {
+						got := prefs.FieldByName(f.name).Interface()
+						if !reflect.DeepEqual(got, f.want) {
+							t.Errorf("%v: got %v; want %v", f.name, got, f.want)
+						}
+					}
+				}
+				return n.Prefs != nil
+			},
+		}
+	}
+
+	unexpectedPrefsChange := func(t testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool {
+		if n.Prefs != nil {
+			t.Errorf("Unexpected Prefs: %v", n.Prefs.Pretty())
+			return true
+		}
+		return false
+	}
+
+	tests := []struct {
+		name           string
+		initialPrefs   *ipn.Prefs
+		stringSettings []source.TestSetting[string]
+		want           *wantedNotification
+	}{
+		{
+			name:           "ShieldsUp/True",
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.EnableIncomingConnections, "never")},
+			want:           wantPrefsChanges(fieldChange{"ShieldsUp", true}),
+		},
+		{
+			name:           "ShieldsUp/False",
+			initialPrefs:   &ipn.Prefs{ShieldsUp: true},
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.EnableIncomingConnections, "always")},
+			want:           wantPrefsChanges(fieldChange{"ShieldsUp", false}),
+		},
+		{
+			name:           "ExitNodeID",
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.ExitNodeID, "foo")},
+			want:           wantPrefsChanges(fieldChange{"ExitNodeID", tailcfg.StableNodeID("foo")}),
+		},
+		{
+			name:           "EnableRunExitNode",
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.EnableRunExitNode, "always")},
+			want:           wantPrefsChanges(fieldChange{"AdvertiseRoutes", []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()}}),
+		},
+		{
+			name: "Multiple",
+			initialPrefs: &ipn.Prefs{
+				ExitNodeAllowLANAccess: true,
+			},
+			stringSettings: []source.TestSetting[string]{
+				source.TestSettingOf(syspolicy.EnableServerMode, "always"),
+				source.TestSettingOf(syspolicy.ExitNodeAllowLANAccess, "never"),
+				source.TestSettingOf(syspolicy.ExitNodeIP, "127.0.0.1"),
+			},
+			want: wantPrefsChanges(
+				fieldChange{"ForceDaemon", true},
+				fieldChange{"ExitNodeAllowLANAccess", false},
+				fieldChange{"ExitNodeIP", netip.MustParseAddr("127.0.0.1")},
+			),
+		},
+		{
+			name: "NoChange",
+			initialPrefs: &ipn.Prefs{
+				CorpDNS:         true,
+				ExitNodeID:      "foo",
+				AdvertiseRoutes: []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()},
+			},
+			stringSettings: []source.TestSetting[string]{
+				source.TestSettingOf(syspolicy.EnableTailscaleDNS, "always"),
+				source.TestSettingOf(syspolicy.ExitNodeID, "foo"),
+				source.TestSettingOf(syspolicy.EnableRunExitNode, "always"),
+			},
+			want: nil, // syspolicy settings match the preferences; no change notification is expected.
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			syspolicy.RegisterWellKnownSettingsForTest(t)
+			store := source.NewTestStoreOf[string](t)
+			syspolicy.MustRegisterStoreForTest(t, "TestSource", setting.DeviceScope, store)
+
+			lb := newLocalBackendWithTestControl(t, enableLogging, func(tb testing.TB, opts controlclient.Options) controlclient.Client {
+				return newClient(tb, opts)
+			})
+			if tt.initialPrefs != nil {
+				lb.SetPrefsForTest(tt.initialPrefs)
+			}
+			if err := lb.Start(ipn.Options{}); err != nil {
+				t.Fatalf("(*LocalBackend).Start(): %v", err)
+			}
+
+			nw := newNotificationWatcher(t, lb, &ipnauth.TestActor{})
+			if tt.want != nil {
+				nw.watch(0, []wantedNotification{*tt.want})
+			} else {
+				nw.watch(0, nil, unexpectedPrefsChange)
+			}
+
+			store.SetStrings(tt.stringSettings...)
+
+			nw.check()
+		})
+	}
+}

ipn/ipnlocal/network-lock.go

@@ -430,8 +430,7 @@ func (b *LocalBackend) tkaBootstrapFromGenesisLocked(g tkatype.MarshaledAUM, per
 		}
 		bootstrapStateID := fmt.Sprintf("%d:%d", genesis.State.StateID1, genesis.State.StateID2)
 
-		for i := range persist.DisallowedTKAStateIDs().Len() {
-			stateID := persist.DisallowedTKAStateIDs().At(i)
+		for _, stateID := range persist.DisallowedTKAStateIDs().All() {
 			if stateID == bootstrapStateID {
 				return fmt.Errorf("TKA with stateID of %q is disallowed on this node", stateID)
 			}
@@ -572,8 +571,7 @@ func tkaStateFromPeer(p tailcfg.NodeView) ipnstate.TKAPeer {
 		TailscaleIPs: make([]netip.Addr, 0, p.Addresses().Len()),
 		NodeKey:      p.Key(),
 	}
-	for i := range p.Addresses().Len() {
-		addr := p.Addresses().At(i)
+	for _, addr := range p.Addresses().All() {
 		if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.Addr()) {
 			fp.TailscaleIPs = append(fp.TailscaleIPs, addr.Addr())
 		}

ipn/ipnlocal/serve.go

@@ -242,8 +242,7 @@ func (b *LocalBackend) updateServeTCPPortNetMapAddrListenersLocked(ports []uint1
 	}
 
 	addrs := nm.GetAddresses()
-	for i := range addrs.Len() {
-		a := addrs.At(i)
+	for _, a := range addrs.All() {
 		for _, p := range ports {
 			addrPort := netip.AddrPortFrom(a.Addr(), p)
 			if _, ok := b.serveListeners[addrPort]; ok {

ipn/ipnlocal/state_test.go

@@ -309,6 +309,7 @@ func TestStateMachine(t *testing.T) {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(b.Shutdown)
 	b.DisablePortMapperForTest()
 
 	var cc, previousCC *mockControl
@@ -942,6 +943,7 @@ func TestEditPrefsHasNoKeys(t *testing.T) {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(b.Shutdown)
 	b.hostinfo = &tailcfg.Hostinfo{OS: "testos"}
 	b.pm.SetPrefs((&ipn.Prefs{
 		Persist: &persist.Persist{
@@ -1023,6 +1025,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	sys.Set(eng)
 	b, err := NewLocalBackend(logf, logid.PublicID{}, sys, 0)
 	c.Assert(err, qt.IsNil)
+	t.Cleanup(b.Shutdown)
 
 	var cc *mockControl
 	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {

ipn/ipnlocal/web_client.go

@@ -121,8 +121,8 @@ func (b *LocalBackend) updateWebClientListenersLocked() {
 	}
 
 	addrs := b.netMap.GetAddresses()
-	for i := range addrs.Len() {
-		addrPort := netip.AddrPortFrom(addrs.At(i).Addr(), webClientPort)
+	for _, pfx := range addrs.All() {
+		addrPort := netip.AddrPortFrom(pfx.Addr(), webClientPort)
 		if _, ok := b.webClientListeners[addrPort]; ok {
 			continue // already listening
 		}

ipn/localapi/localapi_test.go

@@ -349,6 +349,7 @@ func newTestLocalBackend(t testing.TB) *ipnlocal.LocalBackend {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(lb.Shutdown)
 	return lb
 }
 

ipn/store/kubestore/store_kube.go

@@ -7,6 +7,7 @@ package kubestore
 import (
 	"context"
 	"fmt"
+	"log"
 	"net"
 	"os"
 	"strings"
@@ -19,8 +20,18 @@ import (
 	"tailscale.com/types/logger"
 )
 
-// TODO(irbekrm): should we bump this? should we have retries? See tailscale/tailscale#13024
-const timeout = 5 * time.Second
+const (
+	// timeout is the timeout for a single state update that includes calls to the API server to write or read a
+	// state Secret and emit an Event.
+	timeout = 30 * time.Second
+
+	reasonTailscaleStateUpdated      = "TailscaledStateUpdated"
+	reasonTailscaleStateLoaded       = "TailscaleStateLoaded"
+	reasonTailscaleStateUpdateFailed = "TailscaleStateUpdateFailed"
+	reasonTailscaleStateLoadFailed   = "TailscaleStateLoadFailed"
+	eventTypeWarning                 = "Warning"
+	eventTypeNormal                  = "Normal"
+)
 
 // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
 type Store struct {
@@ -35,7 +46,7 @@ type Store struct {
 
 // New returns a new Store that persists to the named Secret.
 func New(_ logger.Logf, secretName string) (*Store, error) {
-	c, err := kubeclient.New()
+	c, err := kubeclient.New("tailscale-state-store")
 	if err != nil {
 		return nil, err
 	}
@@ -72,13 +83,22 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
 
 // WriteState implements the StateStore interface.
 func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer func() {
 		if err == nil {
 			s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
 		}
+		if err != nil {
+			if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil {
+				log.Printf("kubestore: error creating tailscaled state update Event: %v", err)
+			}
+		} else {
+			if err := s.client.Event(ctx, eventTypeNormal, reasonTailscaleStateUpdated, "Successfully updated tailscaled state Secret"); err != nil {
+				log.Printf("kubestore: error creating tailscaled state Event: %v", err)
+			}
+		}
+		cancel()
 	}()
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
 
 	secret, err := s.client.GetSecret(ctx, s.secretName)
 	if err != nil {
@@ -107,7 +127,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
 					Value: map[string][]byte{sanitizeKey(id): bs},
 				},
 			}
-			if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
+			if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
 				return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err)
 			}
 			return nil
@@ -119,8 +139,8 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
 				Value: bs,
 			},
 		}
-		if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
-			return fmt.Errorf("error patching Secret %s with /data/%s field", s.secretName, sanitizeKey(id))
+		if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
+			return fmt.Errorf("error patching Secret %s with /data/%s field: %v", s.secretName, sanitizeKey(id), err)
 		}
 		return nil
 	}
@@ -131,7 +151,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
 	return err
 }
 
-func (s *Store) loadState() error {
+func (s *Store) loadState() (err error) {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
@@ -140,8 +160,14 @@ func (s *Store) loadState() error {
 		if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 {
 			return ipn.ErrStateNotExist
 		}
+		if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateLoadFailed, err.Error()); err != nil {
+			log.Printf("kubestore: error creating Event: %v", err)
+		}
 		return err
 	}
+	if err := s.client.Event(ctx, eventTypeNormal, reasonTailscaleStateLoaded, "Successfully loaded tailscaled state from Secret"); err != nil {
+		log.Printf("kubestore: error creating Event: %v", err)
+	}
 	s.memory.LoadFromMap(secret.Data)
 	return nil
 }

k8s-operator/api.md

@@ -21,6 +21,22 @@
 
 
 
+#### AppConnector
+
+
+
+AppConnector defines a Tailscale app connector node configured via Connector.
+
+
+
+_Appears in:_
+- [ConnectorSpec](#connectorspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `routes` _[Routes](#routes)_ | Routes are optional preconfigured routes for the domains routed via the app connector.<br />If not set, routes for the domains will be discovered dynamically.<br />If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may<br />also dynamically discover other routes.<br />https://tailscale.com/kb/1332/apps-best-practices#preconfiguration |  | Format: cidr <br />MinItems: 1 <br />Type: string <br /> |
+
+
 
 
 #### Connector
@@ -86,8 +102,9 @@ _Appears in:_
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale node will be tagged with.<br />Defaults to [tag:k8s].<br />To autoapprove the subnet routes or exit node defined by a Connector,<br />you can configure Tailscale ACLs to give these tags the necessary<br />permissions.<br />See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.<br />If you specify custom tags here, you must also make the operator an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a Connector node has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
 | `hostname` _[Hostname](#hostname)_ | Hostname is the tailnet hostname that should be assigned to the<br />Connector node. If unset, hostname defaults to <connector<br />name>-connector. Hostname can contain lower case letters, numbers and<br />dashes, it must not start or end with a dash and must be between 2<br />and 63 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$` <br />Type: string <br /> |
 | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that<br />contains configuration options that should be applied to the<br />resources created for this Connector. If unset, the operator will<br />create resources with the default configuration. |  |  |
-| `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector node should<br />expose to tailnet. If unset, none are exposed.<br />https://tailscale.com/kb/1019/subnets/ |  |  |
-| `exitNode` _boolean_ | ExitNode defines whether the Connector node should act as a<br />Tailscale exit node. Defaults to false.<br />https://tailscale.com/kb/1103/exit-nodes |  |  |
+| `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector device should<br />expose to tailnet as a Tailscale subnet router.<br />https://tailscale.com/kb/1019/subnets/<br />If this field is unset, the device does not get configured as a Tailscale subnet router.<br />This field is mutually exclusive with the appConnector field. |  |  |
+| `appConnector` _[AppConnector](#appconnector)_ | AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is<br />configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the<br />Connector does not act as an app connector.<br />Note that you will need to manually configure the permissions and the domains for the app connector via the<br />Admin panel.<br />Note also that the main tested and supported use case of this config option is to deploy an app connector on<br />Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose<br />cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have<br />tested or optimised for.<br />If you are using the app connector to access SaaS applications because you need a predictable egress IP that<br />can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows<br />via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT<br />device with a static IP address.<br />https://tailscale.com/kb/1281/app-connectors |  |  |
+| `exitNode` _boolean_ | ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.<br />This field is mutually exclusive with the appConnector field.<br />https://tailscale.com/kb/1103/exit-nodes |  |  |
 
 
 #### ConnectorStatus
@@ -106,6 +123,7 @@ _Appears in:_
 | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the Connector.<br />Known condition types are `ConnectorReady`. |  |  |
 | `subnetRoutes` _string_ | SubnetRoutes are the routes currently exposed to tailnet via this<br />Connector instance. |  |  |
 | `isExitNode` _boolean_ | IsExitNode is set to true if the Connector acts as an exit node. |  |  |
+| `isAppConnector` _boolean_ | IsAppConnector is set to true if the Connector acts as an app connector. |  |  |
 | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the Connector node. |  |  |
 | `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. |  |  |
 
@@ -128,6 +146,7 @@ _Appears in:_
 | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#pullpolicy-v1-core)_ | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image |  | Enum: [Always Never IfNotPresent] <br /> |
 | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#resourcerequirements-v1-core)_ | Container resource requirements.<br />By default Tailscale Kubernetes operator does not apply any resource<br />requirements. The amount of resources required wil depend on the<br />amount of resources the operator needs to parse, usage patterns and<br />cluster size.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources |  |  |
 | `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#securitycontext-v1-core)_ | Container security context.<br />Security context specified here will override the security context by the operator.<br />By default the operator:<br />- sets 'privileged: true' for the init container<br />- set NET_ADMIN capability for tailscale container for proxies that<br />are created for Services or Connector.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context |  |  |
+| `debug` _[Debug](#debug)_ | Configuration for enabling extra debug information in the container.<br />Not recommended for production use. |  |  |
 
 
 #### DNSConfig
@@ -230,6 +249,22 @@ _Appears in:_
 | `nameserver` _[NameserverStatus](#nameserverstatus)_ | Nameserver describes the status of nameserver cluster resources. |  |  |
 
 
+#### Debug
+
+
+
+
+
+
+
+_Appears in:_
+- [Container](#container)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `enable` _boolean_ | Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/<br />and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where<br />9001 is a container port named "debug". The endpoints and their responses<br />may change in backwards incompatible ways in the future, and should not<br />be considered stable.<br />In 1.78.x and 1.80.x, this setting will default to the value of<br />.spec.metrics.enable, and requests to the "metrics" port matching the<br />mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,<br />this setting will default to false, and no requests will be proxied. |  |  |
+
+
 #### Env
 
 
@@ -291,7 +326,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enable` _boolean_ | Setting enable to true will make the proxy serve Tailscale metrics<br />at <pod-ip>:9001/debug/metrics.<br />Defaults to false. |  |  |
+| `enable` _boolean_ | Setting enable to true will make the proxy serve Tailscale metrics<br />at <pod-ip>:9002/metrics.<br />In 1.78.x and 1.80.x, this field also serves as the default value for<br />.spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both<br />fields will independently default to false.<br />Defaults to false. |  |  |
 
 
 #### Name
@@ -746,6 +781,7 @@ _Validation:_
 - Type: string
 
 _Appears in:_
+- [AppConnector](#appconnector)
 - [SubnetRouter](#subnetrouter)
 
 

k8s-operator/apis/v1alpha1/types_connector.go

@@ -22,6 +22,7 @@ var ConnectorKind = "Connector"
 // +kubebuilder:resource:scope=Cluster,shortName=cn
 // +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRoutes`,description="CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance."
 // +kubebuilder:printcolumn:name="IsExitNode",type="string",JSONPath=`.status.isExitNode`,description="Whether this Connector instance defines an exit node."
+// +kubebuilder:printcolumn:name="IsAppConnector",type="string",JSONPath=`.status.isAppConnector`,description="Whether this Connector instance is an app connector."
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the deployed Connector resources."
 
 // Connector defines a Tailscale node that will be deployed in the cluster. The
@@ -55,7 +56,8 @@ type ConnectorList struct {
 }
 
 // ConnectorSpec describes a Tailscale node to be deployed in the cluster.
-// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || self.exitNode == true",message="A Connector needs to be either an exit node or a subnet router, or both."
+// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)",message="A Connector needs to have at least one of exit node, subnet router or app connector configured."
+// +kubebuilder:validation:XValidation:rule="!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))",message="The appConnector field is mutually exclusive with exitNode and subnetRouter fields."
 type ConnectorSpec struct {
 	// Tags that the Tailscale node will be tagged with.
 	// Defaults to [tag:k8s].
@@ -82,13 +84,31 @@ type ConnectorSpec struct {
 	// create resources with the default configuration.
 	// +optional
 	ProxyClass string `json:"proxyClass,omitempty"`
-	// SubnetRouter defines subnet routes that the Connector node should
-	// expose to tailnet. If unset, none are exposed.
+	// SubnetRouter defines subnet routes that the Connector device should
+	// expose to tailnet as a Tailscale subnet router.
 	// https://tailscale.com/kb/1019/subnets/
+	// If this field is unset, the device does not get configured as a Tailscale subnet router.
+	// This field is mutually exclusive with the appConnector field.
 	// +optional
-	SubnetRouter *SubnetRouter `json:"subnetRouter"`
-	// ExitNode defines whether the Connector node should act as a
-	// Tailscale exit node. Defaults to false.
+	SubnetRouter *SubnetRouter `json:"subnetRouter,omitempty"`
+	// AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+	// configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+	// Connector does not act as an app connector.
+	// Note that you will need to manually configure the permissions and the domains for the app connector via the
+	// Admin panel.
+	// Note also that the main tested and supported use case of this config option is to deploy an app connector on
+	// Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+	// cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+	// tested or optimised for.
+	// If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+	// can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+	// via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+	// device with a static IP address.
+	// https://tailscale.com/kb/1281/app-connectors
+	// +optional
+	AppConnector *AppConnector `json:"appConnector,omitempty"`
+	// ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+	// This field is mutually exclusive with the appConnector field.
 	// https://tailscale.com/kb/1103/exit-nodes
 	// +optional
 	ExitNode bool `json:"exitNode"`
@@ -104,6 +124,17 @@ type SubnetRouter struct {
 	AdvertiseRoutes Routes `json:"advertiseRoutes"`
 }
 
+// AppConnector defines a Tailscale app connector node configured via Connector.
+type AppConnector struct {
+	// Routes are optional preconfigured routes for the domains routed via the app connector.
+	// If not set, routes for the domains will be discovered dynamically.
+	// If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+	// also dynamically discover other routes.
+	// https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+	// +optional
+	Routes Routes `json:"routes"`
+}
+
 type Tags []Tag
 
 func (tags Tags) Stringify() []string {
@@ -156,6 +187,9 @@ type ConnectorStatus struct {
 	// IsExitNode is set to true if the Connector acts as an exit node.
 	// +optional
 	IsExitNode bool `json:"isExitNode"`
+	// IsAppConnector is set to true if the Connector acts as an app connector.
+	// +optional
+	IsAppConnector bool `json:"isAppConnector"`
 	// TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
 	// assigned to the Connector node.
 	// +optional

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -163,7 +163,12 @@ type Pod struct {
 
 type Metrics struct {
 	// Setting enable to true will make the proxy serve Tailscale metrics
-	// at <pod-ip>:9001/debug/metrics.
+	// at <pod-ip>:9002/metrics.
+	//
+	// In 1.78.x and 1.80.x, this field also serves as the default value for
+	// .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+	// fields will independently default to false.
+	//
 	// Defaults to false.
 	Enable bool `json:"enable"`
 }
@@ -209,6 +214,26 @@ type Container struct {
 	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
 	// +optional
 	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
+	// Configuration for enabling extra debug information in the container.
+	// Not recommended for production use.
+	// +optional
+	Debug *Debug `json:"debug,omitempty"`
+}
+
+type Debug struct {
+	// Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+	// and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+	// 9001 is a container port named "debug". The endpoints and their responses
+	// may change in backwards incompatible ways in the future, and should not
+	// be considered stable.
+	//
+	// In 1.78.x and 1.80.x, this setting will default to the value of
+	// .spec.metrics.enable, and requests to the "metrics" port matching the
+	// mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+	// this setting will default to false, and no requests will be proxied.
+	//
+	// +optional
+	Enable bool `json:"enable"`
 }
 
 type Env struct {

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -13,6 +13,26 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AppConnector) DeepCopyInto(out *AppConnector) {
+	*out = *in
+	if in.Routes != nil {
+		in, out := &in.Routes, &out.Routes
+		*out = make(Routes, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppConnector.
+func (in *AppConnector) DeepCopy() *AppConnector {
+	if in == nil {
+		return nil
+	}
+	out := new(AppConnector)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Connector) DeepCopyInto(out *Connector) {
 	*out = *in
@@ -85,6 +105,11 @@ func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
 		*out = new(SubnetRouter)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.AppConnector != nil {
+		in, out := &in.AppConnector, &out.AppConnector
+		*out = new(AppConnector)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.
@@ -138,6 +163,11 @@ func (in *Container) DeepCopyInto(out *Container) {
 		*out = new(corev1.SecurityContext)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Debug != nil {
+		in, out := &in.Debug, &out.Debug
+		*out = new(Debug)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container.
@@ -256,6 +286,21 @@ func (in *DNSConfigStatus) DeepCopy() *DNSConfigStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Debug) DeepCopyInto(out *Debug) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Debug.
+func (in *Debug) DeepCopy() *Debug {
+	if in == nil {
+		return nil
+	}
+	out := new(Debug)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Env) DeepCopyInto(out *Env) {
 	*out = *in

k8s-operator/sessionrecording/hijacker.go

@@ -102,7 +102,7 @@ type Hijacker struct {
 // connection succeeds. In case of success, returns a list with a single
 // successful recording attempt and an error channel. If the connection errors
 // after having been established, an error is sent down the channel.
-type RecorderDialFn func(context.Context, []netip.AddrPort, func(context.Context, string, string) (net.Conn, error)) (io.WriteCloser, []*tailcfg.SSHRecordingAttempt, <-chan error, error)
+type RecorderDialFn func(context.Context, []netip.AddrPort, sessionrecording.DialFunc) (io.WriteCloser, []*tailcfg.SSHRecordingAttempt, <-chan error, error)
 
 // Hijack hijacks a 'kubectl exec' session and configures for the session
 // contents to be sent to a recorder.

k8s-operator/sessionrecording/hijacker_test.go

@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/netip"
 	"net/url"
@@ -20,6 +19,7 @@ import (
 	"go.uber.org/zap"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/k8s-operator/sessionrecording/fakes"
+	"tailscale.com/sessionrecording"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/tstest"
@@ -80,7 +80,7 @@ func Test_Hijacker(t *testing.T) {
 			h := &Hijacker{
 				connectToRecorder: func(context.Context,
 					[]netip.AddrPort,
-					func(context.Context, string, string) (net.Conn, error),
+					sessionrecording.DialFunc,
 				) (wc io.WriteCloser, rec []*tailcfg.SSHRecordingAttempt, _ <-chan error, err error) {
 					if tt.failRecorderConnect {
 						err = errors.New("test")

k8s-operator/utils.go

@@ -32,9 +32,6 @@ type Records struct {
 // TailscaledConfigFileName returns a tailscaled config file name in
 // format expected by containerboot for the given CapVer.
 func TailscaledConfigFileName(cap tailcfg.CapabilityVersion) string {
-	if cap < 95 {
-		return "tailscaled"
-	}
 	return fmt.Sprintf("cap-%v.hujson", cap)
 }
 

kube/kubeapi/api.go

@@ -7,7 +7,9 @@
 // dependency size for those consumers when adding anything new here.
 package kubeapi
 
-import "time"
+import (
+	"time"
+)
 
 // Note: The API types are copied from k8s.io/api{,machinery} to not introduce a
 // module dependency on the Kubernetes API as it pulls in many more dependencies.
@@ -151,6 +153,57 @@ type Secret struct {
 	Data map[string][]byte `json:"data,omitempty"`
 }
 
+// Event contains a subset of fields from corev1.Event.
+// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7034
+// It is copied here to avoid having to import kube libraries.
+type Event struct {
+	TypeMeta   `json:",inline"`
+	ObjectMeta `json:"metadata"`
+	Message    string      `json:"message,omitempty"`
+	Reason     string      `json:"reason,omitempty"`
+	Source     EventSource `json:"source,omitempty"` // who is emitting this Event
+	Type       string      `json:"type,omitempty"`   // Normal or Warning
+	// InvolvedObject is the subject of the Event. `kubectl describe` will, for most object types, display any
+	// currently present cluster Events matching the object (but you probably want to set UID for this to work).
+	InvolvedObject ObjectReference `json:"involvedObject"`
+	Count          int32           `json:"count,omitempty"` // how many times Event was observed
+	FirstTimestamp time.Time       `json:"firstTimestamp,omitempty"`
+	LastTimestamp  time.Time       `json:"lastTimestamp,omitempty"`
+}
+
+// EventSource includes a subset of fields from corev1.EventSource.
+// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7007
+// It is copied here to avoid having to import kube libraries.
+type EventSource struct {
+	// Component is the name of the component that is emitting the Event.
+	Component string `json:"component,omitempty"`
+}
+
+// ObjectReference contains a subset of fields from corev1.ObjectReference.
+// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L6902
+// It is copied here to avoid having to import kube libraries.
+type ObjectReference struct {
+	// Kind of the referent.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	// +optional
+	Kind string `json:"kind,omitempty"`
+	// Namespace of the referent.
+	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+	// +optional
+	Namespace string `json:"namespace,omitempty"`
+	// Name of the referent.
+	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+	// +optional
+	Name string `json:"name,omitempty"`
+	// UID of the referent.
+	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+	// +optional
+	UID string `json:"uid,omitempty"`
+	// API version of the referent.
+	// +optional
+	APIVersion string `json:"apiVersion,omitempty"`
+}
+
 // Status is a return value for calls that don't return other objects.
 type Status struct {
 	TypeMeta `json:",inline"`
@@ -186,6 +239,6 @@ type Status struct {
 	Code int `json:"code,omitempty"`
 }
 
-func (s *Status) Error() string {
+func (s Status) Error() string {
 	return s.Message
 }

kube/kubeclient/client.go

@@ -23,16 +23,21 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
 	"tailscale.com/kube/kubeapi"
+	"tailscale.com/tstime"
 	"tailscale.com/util/multierr"
 )
 
 const (
 	saPath     = "/var/run/secrets/kubernetes.io/serviceaccount"
 	defaultURL = "https://kubernetes.default.svc"
+
+	TypeSecrets = "secrets"
+	typeEvents  = "events"
 )
 
 // rootPathForTests is set by tests to override the root path to the
@@ -57,8 +62,13 @@ type Client interface {
 	GetSecret(context.Context, string) (*kubeapi.Secret, error)
 	UpdateSecret(context.Context, *kubeapi.Secret) error
 	CreateSecret(context.Context, *kubeapi.Secret) error
+	// Event attempts to ensure an event with the specified options associated with the Pod in which we are
+	// currently running. This is best effort - if the client is not able to create events, this operation will be a
+	// no-op. If there is already an Event with the given reason for the current Pod, it will get updated (only
+	// count and timestamp are expected to change), else a new event will be created.
+	Event(_ context.Context, typ, reason, msg string) error
 	StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error
-	JSONPatchSecret(context.Context, string, []JSONPatch) error
+	JSONPatchResource(_ context.Context, resourceName string, resourceType string, patches []JSONPatch) error
 	CheckSecretPermissions(context.Context, string) (bool, bool, error)
 	SetDialer(dialer func(context.Context, string, string) (net.Conn, error))
 	SetURL(string)
@@ -66,15 +76,24 @@ type Client interface {
 
 type client struct {
 	mu          sync.Mutex
+	name        string
 	url         string
-	ns          string
+	podName     string
+	podUID      string
+	ns          string // Pod namespace
 	client      *http.Client
 	token       string
 	tokenExpiry time.Time
+	cl          tstime.Clock
+	// hasEventsPerms is true if client can emit Events for the Pod in which it runs. If it is set to false any
+	// calls to Events() will be a no-op.
+	hasEventsPerms bool
+	// kubeAPIRequest sends a request to the kube API server. It can set to a fake in tests.
+	kubeAPIRequest kubeAPIRequestFunc
 }
 
 // New returns a new client
-func New() (Client, error) {
+func New(name string) (Client, error) {
 	ns, err := readFile("namespace")
 	if err != nil {
 		return nil, err
@@ -87,9 +106,11 @@ func New() (Client, error) {
 	if ok := cp.AppendCertsFromPEM(caCert); !ok {
 		return nil, fmt.Errorf("kube: error in creating root cert pool")
 	}
-	return &client{
-		url: defaultURL,
-		ns:  string(ns),
+	c := &client{
+		url:  defaultURL,
+		ns:   string(ns),
+		name: name,
+		cl:   tstime.DefaultClock{},
 		client: &http.Client{
 			Transport: &http.Transport{
 				TLSClientConfig: &tls.Config{
@@ -97,7 +118,10 @@ func New() (Client, error) {
 				},
 			},
 		},
-	}, nil
+	}
+	c.kubeAPIRequest = newKubeAPIRequest(c)
+	c.setEventPerms()
+	return c, nil
 }
 
 // SetURL sets the URL to use for the Kubernetes API.
@@ -115,14 +139,14 @@ func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string
 func (c *client) expireToken() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.tokenExpiry = time.Now()
+	c.tokenExpiry = c.cl.Now()
 }
 
 func (c *client) getOrRenewToken() (string, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	tk, te := c.token, c.tokenExpiry
-	if time.Now().Before(te) {
+	if c.cl.Now().Before(te) {
 		return tk, nil
 	}
 
@@ -131,17 +155,10 @@ func (c *client) getOrRenewToken() (string, error) {
 		return "", err
 	}
 	c.token = string(tkb)
-	c.tokenExpiry = time.Now().Add(30 * time.Minute)
+	c.tokenExpiry = c.cl.Now().Add(30 * time.Minute)
 	return c.token, nil
 }
 
-func (c *client) secretURL(name string) string {
-	if name == "" {
-		return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns)
-	}
-	return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets/%s", c.url, c.ns, name)
-}
-
 func getError(resp *http.Response) error {
 	if resp.StatusCode == 200 || resp.StatusCode == 201 {
 		// These are the only success codes returned by the Kubernetes API.
@@ -161,36 +178,41 @@ func setHeader(key, value string) func(*http.Request) {
 	}
 }
 
-// doRequest performs an HTTP request to the Kubernetes API.
-// If in is not nil, it is expected to be a JSON-encodable object and will be
-// sent as the request body.
-// If out is not nil, it is expected to be a pointer to an object that can be
-// decoded from JSON.
-// If the request fails with a 401, the token is expired and a new one is
-// requested.
-func (c *client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
-	req, err := c.newRequest(ctx, method, url, in)
-	if err != nil {
-		return err
-	}
-	for _, opt := range opts {
-		opt(req)
-	}
-	resp, err := c.client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if err := getError(resp); err != nil {
-		if st, ok := err.(*kubeapi.Status); ok && st.Code == 401 {
-			c.expireToken()
+type kubeAPIRequestFunc func(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error
+
+// newKubeAPIRequest returns a function that can perform an HTTP request to the Kubernetes API.
+func newKubeAPIRequest(c *client) kubeAPIRequestFunc {
+	// If in is not nil, it is expected to be a JSON-encodable object and will be
+	// sent as the request body.
+	// If out is not nil, it is expected to be a pointer to an object that can be
+	// decoded from JSON.
+	// If the request fails with a 401, the token is expired and a new one is
+	// requested.
+	f := func(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
+		req, err := c.newRequest(ctx, method, url, in)
+		if err != nil {
+			return err
 		}
-		return err
+		for _, opt := range opts {
+			opt(req)
+		}
+		resp, err := c.client.Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		if err := getError(resp); err != nil {
+			if st, ok := err.(*kubeapi.Status); ok && st.Code == 401 {
+				c.expireToken()
+			}
+			return err
+		}
+		if out != nil {
+			return json.NewDecoder(resp.Body).Decode(out)
+		}
+		return nil
 	}
-	if out != nil {
-		return json.NewDecoder(resp.Body).Decode(out)
-	}
-	return nil
+	return f
 }
 
 func (c *client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) {
@@ -226,7 +248,7 @@ func (c *client) newRequest(ctx context.Context, method, url string, in any) (*h
 // GetSecret fetches the secret from the Kubernetes API.
 func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) {
 	s := &kubeapi.Secret{Data: make(map[string][]byte)}
-	if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil {
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets), nil, s); err != nil {
 		return nil, err
 	}
 	return s, nil
@@ -235,16 +257,16 @@ func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, e
 // CreateSecret creates a secret in the Kubernetes API.
 func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error {
 	s.Namespace = c.ns
-	return c.doRequest(ctx, "POST", c.secretURL(""), s, nil)
+	return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets), s, nil)
 }
 
 // UpdateSecret updates a secret in the Kubernetes API.
 func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error {
-	return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil)
+	return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets), s, nil)
 }
 
 // JSONPatch is a JSON patch operation.
-// It currently (2023-03-02) only supports "add" and "remove" operations.
+// It currently (2024-11-15) only supports "add", "remove" and "replace" operations.
 //
 // https://tools.ietf.org/html/rfc6902
 type JSONPatch struct {
@@ -253,22 +275,22 @@ type JSONPatch struct {
 	Value any    `json:"value,omitempty"`
 }
 
-// JSONPatchSecret updates a secret in the Kubernetes API using a JSON patch.
-// It currently (2023-03-02) only supports "add" and "remove" operations.
-func (c *client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error {
-	for _, p := range patch {
+// JSONPatchResource updates a resource in the Kubernetes API using a JSON patch.
+// It currently (2024-11-15) only supports "add", "remove" and "replace" operations.
+func (c *client) JSONPatchResource(ctx context.Context, name, typ string, patches []JSONPatch) error {
+	for _, p := range patches {
 		if p.Op != "remove" && p.Op != "add" && p.Op != "replace" {
 			return fmt.Errorf("unsupported JSON patch operation: %q", p.Op)
 		}
 	}
-	return c.doRequest(ctx, "PATCH", c.secretURL(name), patch, nil, setHeader("Content-Type", "application/json-patch+json"))
+	return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
 }
 
 // StrategicMergePatchSecret updates a secret in the Kubernetes API using a
 // strategic merge patch.
 // If a fieldManager is provided, it will be used to track the patch.
 func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error {
-	surl := c.secretURL(name)
+	surl := c.resourceURL(name, TypeSecrets)
 	if fieldManager != "" {
 		uv := url.Values{
 			"fieldManager": {fieldManager},
@@ -277,7 +299,66 @@ func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *
 	}
 	s.Namespace = c.ns
 	s.Name = name
-	return c.doRequest(ctx, "PATCH", surl, s, nil, setHeader("Content-Type", "application/strategic-merge-patch+json"))
+	return c.kubeAPIRequest(ctx, "PATCH", surl, s, nil, setHeader("Content-Type", "application/strategic-merge-patch+json"))
+}
+
+// Event tries to ensure an Event associated with the Pod in which we are running. It is best effort - the event will be
+// created if the kube client on startup was able to determine the name and UID of this Pod from POD_NAME,POD_UID env
+// vars and if permissions check for event creation succeeded. Events are keyed on opts.Reason- if an Event for the
+// current Pod with that reason already exists, its count and first timestamp will be updated, else a new Event will be
+// created.
+func (c *client) Event(ctx context.Context, typ, reason, msg string) error {
+	if !c.hasEventsPerms {
+		return nil
+	}
+	name := c.nameForEvent(reason)
+	ev, err := c.getEvent(ctx, name)
+	now := c.cl.Now()
+	if err != nil {
+		if !IsNotFoundErr(err) {
+			return err
+		}
+		// Event not found - create it
+		ev := kubeapi.Event{
+			ObjectMeta: kubeapi.ObjectMeta{
+				Name:      name,
+				Namespace: c.ns,
+			},
+			Type:    typ,
+			Reason:  reason,
+			Message: msg,
+			Source: kubeapi.EventSource{
+				Component: c.name,
+			},
+			InvolvedObject: kubeapi.ObjectReference{
+				Name:       c.podName,
+				Namespace:  c.ns,
+				UID:        c.podUID,
+				Kind:       "Pod",
+				APIVersion: "v1",
+			},
+
+			FirstTimestamp: now,
+			LastTimestamp:  now,
+			Count:          1,
+		}
+		return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents), &ev, nil)
+	}
+	// If the Event already exists, we patch its count and last timestamp. This ensures that when users run 'kubectl
+	// describe pod...', they see the event just once (but with a message of how many times it has appeared over
+	// last timestamp - first timestamp period of time).
+	count := ev.Count + 1
+	countPatch := JSONPatch{
+		Op:    "replace",
+		Value: count,
+		Path:  "/count",
+	}
+	tsPatch := JSONPatch{
+		Op:    "replace",
+		Value: now,
+		Path:  "/lastTimestamp",
+	}
+	return c.JSONPatchResource(ctx, name, typeEvents, []JSONPatch{countPatch, tsPatch})
 }
 
 // CheckSecretPermissions checks the secret access permissions of the current
@@ -293,7 +374,7 @@ func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *
 func (c *client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch, canCreate bool, err error) {
 	var errs []error
 	for _, verb := range []string{"get", "update"} {
-		ok, err := c.checkPermission(ctx, verb, secretName)
+		ok, err := c.checkPermission(ctx, verb, TypeSecrets, secretName)
 		if err != nil {
 			log.Printf("error checking %s permission on secret %s: %v", verb, secretName, err)
 		} else if !ok {
@@ -303,12 +384,12 @@ func (c *client) CheckSecretPermissions(ctx context.Context, secretName string)
 	if len(errs) > 0 {
 		return false, false, multierr.New(errs...)
 	}
-	canPatch, err = c.checkPermission(ctx, "patch", secretName)
+	canPatch, err = c.checkPermission(ctx, "patch", TypeSecrets, secretName)
 	if err != nil {
 		log.Printf("error checking patch permission on secret %s: %v", secretName, err)
 		return false, false, nil
 	}
-	canCreate, err = c.checkPermission(ctx, "create", secretName)
+	canCreate, err = c.checkPermission(ctx, "create", TypeSecrets, secretName)
 	if err != nil {
 		log.Printf("error checking create permission on secret %s: %v", secretName, err)
 		return false, false, nil
@@ -316,36 +397,98 @@ func (c *client) CheckSecretPermissions(ctx context.Context, secretName string)
 	return canPatch, canCreate, nil
 }
 
-// checkPermission reports whether the current pod has permission to use the
-// given verb (e.g. get, update, patch, create) on secretName.
-func (c *client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
-	sar := map[string]any{
-		"apiVersion": "authorization.k8s.io/v1",
-		"kind":       "SelfSubjectAccessReview",
-		"spec": map[string]any{
-			"resourceAttributes": map[string]any{
-				"namespace": c.ns,
-				"verb":      verb,
-				"resource":  "secrets",
-				"name":      secretName,
-			},
-		},
-	}
-	var res struct {
-		Status struct {
-			Allowed bool `json:"allowed"`
-		} `json:"status"`
-	}
-	url := c.url + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews"
-	if err := c.doRequest(ctx, "POST", url, sar, &res); err != nil {
-		return false, err
-	}
-	return res.Status.Allowed, nil
-}
-
 func IsNotFoundErr(err error) bool {
 	if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 {
 		return true
 	}
 	return false
 }
+
+// setEventPerms checks whether this client will be able to write tailscaled Events to its Pod and updates the state
+// accordingly. If it determines that the client can not write Events, any subsequent calls to client.Event will be a
+// no-op.
+func (c *client) setEventPerms() {
+	name := os.Getenv("POD_NAME")
+	uid := os.Getenv("POD_UID")
+	hasPerms := false
+	defer func() {
+		c.podName = name
+		c.podUID = uid
+		c.hasEventsPerms = hasPerms
+		if !hasPerms {
+			log.Printf(`kubeclient: this client is not able to write tailscaled Events to the Pod in which it is running.
+			To help with future debugging you can make it able write Events by giving it get,create,patch permissions for Events in the Pod namespace
+			and setting POD_NAME, POD_UID env vars for the Pod.`)
+		}
+	}()
+	if name == "" || uid == "" {
+		return
+	}
+	for _, verb := range []string{"get", "create", "patch"} {
+		can, err := c.checkPermission(context.Background(), verb, typeEvents, "")
+		if err != nil {
+			log.Printf("kubeclient: error checking Events permissions: %v", err)
+			return
+		}
+		if !can {
+			return
+		}
+	}
+	hasPerms = true
+	return
+}
+
+// checkPermission reports whether the current pod has permission to use the given verb (e.g. get, update, patch,
+// create) on the given resource type. If name is not an empty string, will check the check will be for resource with
+// the given name only.
+func (c *client) checkPermission(ctx context.Context, verb, typ, name string) (bool, error) {
+	ra := map[string]any{
+		"namespace": c.ns,
+		"verb":      verb,
+		"resource":  typ,
+	}
+	if name != "" {
+		ra["name"] = name
+	}
+	sar := map[string]any{
+		"apiVersion": "authorization.k8s.io/v1",
+		"kind":       "SelfSubjectAccessReview",
+		"spec": map[string]any{
+			"resourceAttributes": ra,
+		},
+	}
+	var res struct {
+		Status struct {
+			Allowed bool `json:"allowed"`
+		} `json:"status"`
+	}
+	url := c.url + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews"
+	if err := c.kubeAPIRequest(ctx, "POST", url, sar, &res); err != nil {
+		return false, err
+	}
+	return res.Status.Allowed, nil
+}
+
+// resourceURL returns a URL that can be used to interact with the given resource type and, if name is not empty string,
+// the named resource of that type.
+// Note that this only works for core/v1 resource types.
+func (c *client) resourceURL(name, typ string) string {
+	if name == "" {
+		return fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
+	}
+	return fmt.Sprintf("%s/api/v1/namespaces/%s/%s/%s", c.url, c.ns, typ, name)
+}
+
+// nameForEvent returns a name for the Event that uniquely identifies Event with that reason for the current Pod.
+func (c *client) nameForEvent(reason string) string {
+	return fmt.Sprintf("%s.%s.%s", c.podName, c.podUID, strings.ToLower(reason))
+}
+
+// getEvent fetches the event from the Kubernetes API.
+func (c *client) getEvent(ctx context.Context, name string) (*kubeapi.Event, error) {
+	e := &kubeapi.Event{}
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents), nil, e); err != nil {
+		return nil, err
+	}
+	return e, nil
+}

kube/kubeclient/client_test.go

@@ -0,0 +1,151 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package kubeclient
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/kube/kubeapi"
+	"tailscale.com/tstest"
+)
+
+func Test_client_Event(t *testing.T) {
+	cl := &tstest.Clock{}
+	tests := []struct {
+		name    string
+		typ     string
+		reason  string
+		msg     string
+		argSets []args
+		wantErr bool
+	}{
+		{
+			name:   "new_event_gets_created",
+			typ:    "Normal",
+			reason: "TestReason",
+			msg:    "TestMessage",
+			argSets: []args{
+				{ // request to GET event returns not found
+					wantsMethod: "GET",
+					wantsURL:    "test-apiserver/api/v1/namespaces/test-ns/events/test-pod.test-uid.testreason",
+					setErr:      &kubeapi.Status{Code: 404},
+				},
+				{ // sends POST request to create event
+					wantsMethod: "POST",
+					wantsURL:    "test-apiserver/api/v1/namespaces/test-ns/events",
+					wantsIn: &kubeapi.Event{
+						ObjectMeta: kubeapi.ObjectMeta{
+							Name:      "test-pod.test-uid.testreason",
+							Namespace: "test-ns",
+						},
+						Type:    "Normal",
+						Reason:  "TestReason",
+						Message: "TestMessage",
+						Source: kubeapi.EventSource{
+							Component: "test-client",
+						},
+						InvolvedObject: kubeapi.ObjectReference{
+							Name:       "test-pod",
+							UID:        "test-uid",
+							Namespace:  "test-ns",
+							APIVersion: "v1",
+							Kind:       "Pod",
+						},
+						FirstTimestamp: cl.Now(),
+						LastTimestamp:  cl.Now(),
+						Count:          1,
+					},
+				},
+			},
+		},
+		{
+			name:   "existing_event_gets_patched",
+			typ:    "Warning",
+			reason: "TestReason",
+			msg:    "TestMsg",
+			argSets: []args{
+				{ // request to GET event does not error - this is enough to assume that event exists
+					wantsMethod: "GET",
+					wantsURL:    "test-apiserver/api/v1/namespaces/test-ns/events/test-pod.test-uid.testreason",
+					setOut:      []byte(`{"count":2}`),
+				},
+				{ // sends PATCH request to update the event
+					wantsMethod: "PATCH",
+					wantsURL:    "test-apiserver/api/v1/namespaces/test-ns/events/test-pod.test-uid.testreason",
+					wantsIn: []JSONPatch{
+						{Op: "replace", Path: "/count", Value: int32(3)},
+						{Op: "replace", Path: "/lastTimestamp", Value: cl.Now()},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &client{
+				cl:             cl,
+				name:           "test-client",
+				podName:        "test-pod",
+				podUID:         "test-uid",
+				url:            "test-apiserver",
+				ns:             "test-ns",
+				kubeAPIRequest: fakeKubeAPIRequest(t, tt.argSets),
+				hasEventsPerms: true,
+			}
+			if err := c.Event(context.Background(), tt.typ, tt.reason, tt.msg); (err != nil) != tt.wantErr {
+				t.Errorf("client.Event() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// args is a set of values for testing a single call to client.kubeAPIRequest.
+type args struct {
+	// wantsMethod is the expected value of 'method' arg.
+	wantsMethod string
+	// wantsURL is the expected value of 'url' arg.
+	wantsURL string
+	// wantsIn is the expected value of 'in' arg.
+	wantsIn any
+	// setOut can be set to a byte slice representing valid JSON. If set 'out' arg will get set to the unmarshalled
+	// JSON object.
+	setOut []byte
+	// setErr is the error that kubeAPIRequest will return.
+	setErr error
+}
+
+// fakeKubeAPIRequest can be used to test that a series of calls to client.kubeAPIRequest gets called with expected
+// values and to set these calls to return preconfigured values. 'argSets' should be set to a slice of expected
+// arguments and should-be return values of a series of kubeAPIRequest calls.
+func fakeKubeAPIRequest(t *testing.T, argSets []args) kubeAPIRequestFunc {
+	count := 0
+	f := func(ctx context.Context, gotMethod, gotUrl string, gotIn, gotOut any, opts ...func(*http.Request)) error {
+		t.Helper()
+		if count >= len(argSets) {
+			t.Fatalf("unexpected call to client.kubeAPIRequest, expected %d calls, but got a %dth call", len(argSets), count+1)
+		}
+		a := argSets[count]
+		if gotMethod != a.wantsMethod {
+			t.Errorf("[%d] got method %q, wants method %q", count, gotMethod, a.wantsMethod)
+		}
+		if gotUrl != a.wantsURL {
+			t.Errorf("[%d] got URL %q, wants URL %q", count, gotUrl, a.wantsURL)
+		}
+		if d := cmp.Diff(gotIn, a.wantsIn); d != "" {
+			t.Errorf("[%d] unexpected payload (-want + got):\n%s", count, d)
+		}
+		if len(a.setOut) != 0 {
+			if err := json.Unmarshal(a.setOut, gotOut); err != nil {
+				t.Fatalf("[%d] error unmarshalling output: %v", count, err)
+			}
+		}
+		count++
+		return a.setErr
+	}
+	return f
+}

kube/kubeclient/fake_client.go

@@ -29,7 +29,11 @@ func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr s
 func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error {
 	return nil
 }
-func (fc *FakeClient) JSONPatchSecret(context.Context, string, []JSONPatch) error {
+func (fc *FakeClient) Event(context.Context, string, string, string) error {
+	return nil
+}
+
+func (fc *FakeClient) JSONPatchResource(context.Context, string, string, []JSONPatch) error {
 	return nil
 }
 func (fc *FakeClient) UpdateSecret(context.Context, *kubeapi.Secret) error { return nil }

kube/kubetypes/metrics.go

@@ -21,6 +21,7 @@ const (
 	MetricConnectorResourceCount         = "k8s_connector_resources"
 	MetricConnectorWithSubnetRouterCount = "k8s_connector_subnetrouter_resources"
 	MetricConnectorWithExitNodeCount     = "k8s_connector_exitnode_resources"
+	MetricConnectorWithAppConnectorCount = "k8s_connector_appconnector_resources"
 	MetricNameserverCount                = "k8s_nameserver_resources"
 	MetricRecorderCount                  = "k8s_recorder_resources"
 	MetricEgressServiceCount             = "k8s_egress_service_resources"

licenses/apple.md

@@ -12,24 +12,23 @@ See also the dependencies in the [Tailscale CLI][].
 
 
  - [filippo.io/edwards25519](https://pkg.go.dev/filippo.io/edwards25519) ([BSD-3-Clause](https://github.com/FiloSottile/edwards25519/blob/v1.1.0/LICENSE))
- - [github.com/aws/aws-sdk-go-v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/v1.30.4/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/v1.32.4/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/config](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/config) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.27.28/config/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/credentials](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/credentials) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.17.28/credentials/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/feature/ec2/imds](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/feature/ec2/imds) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.16.12/feature/ec2/imds/LICENSE.txt))
- - [github.com/aws/aws-sdk-go-v2/internal/configsources](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/configsources) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.3.16/internal/configsources/LICENSE.txt))
- - [github.com/aws/aws-sdk-go-v2/internal/endpoints/v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.6.16/internal/endpoints/v2/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2/internal/configsources](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/configsources) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.3.23/internal/configsources/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2/internal/endpoints/v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.6.23/internal/endpoints/v2/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/internal/ini](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/ini) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.8.1/internal/ini/LICENSE.txt))
- - [github.com/aws/aws-sdk-go-v2/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/aws-sdk-go-v2/blob/v1.30.4/internal/sync/singleflight/LICENSE))
- - [github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/internal/accept-encoding/v1.11.4/service/internal/accept-encoding/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/aws-sdk-go-v2/blob/v1.32.4/internal/sync/singleflight/LICENSE))
+ - [github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/internal/accept-encoding/v1.12.0/service/internal/accept-encoding/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/internal/presigned-url](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.11.18/service/internal/presigned-url/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/ssm](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ssm) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/ssm/v1.45.0/service/ssm/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/sso](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/sso) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.22.5/service/sso/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/ssooidc](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ssooidc) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/ssooidc/v1.26.5/service/ssooidc/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/sts](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/sts) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.30.4/service/sts/LICENSE.txt))
- - [github.com/aws/smithy-go](https://pkg.go.dev/github.com/aws/smithy-go) ([Apache-2.0](https://github.com/aws/smithy-go/blob/v1.20.4/LICENSE))
- - [github.com/aws/smithy-go/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/smithy-go/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/smithy-go/blob/v1.20.4/internal/sync/singleflight/LICENSE))
+ - [github.com/aws/smithy-go](https://pkg.go.dev/github.com/aws/smithy-go) ([Apache-2.0](https://github.com/aws/smithy-go/blob/v1.22.0/LICENSE))
+ - [github.com/aws/smithy-go/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/smithy-go/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/smithy-go/blob/v1.22.0/internal/sync/singleflight/LICENSE))
  - [github.com/bits-and-blooms/bitset](https://pkg.go.dev/github.com/bits-and-blooms/bitset) ([BSD-3-Clause](https://github.com/bits-and-blooms/bitset/blob/v1.13.0/LICENSE))
- - [github.com/coder/websocket](https://pkg.go.dev/github.com/coder/websocket) ([ISC](https://github.com/coder/websocket/blob/v1.8.12/LICENSE.txt))
  - [github.com/coreos/go-iptables/iptables](https://pkg.go.dev/github.com/coreos/go-iptables/iptables) ([Apache-2.0](https://github.com/coreos/go-iptables/blob/65c67c9f46e6/LICENSE))
  - [github.com/digitalocean/go-smbios/smbios](https://pkg.go.dev/github.com/digitalocean/go-smbios/smbios) ([Apache-2.0](https://github.com/digitalocean/go-smbios/blob/390a4f403a8e/LICENSE.md))
  - [github.com/djherbis/times](https://pkg.go.dev/github.com/djherbis/times) ([MIT](https://github.com/djherbis/times/blob/v1.6.0/LICENSE))
@@ -48,9 +47,9 @@ See also the dependencies in the [Tailscale CLI][].
  - [github.com/jmespath/go-jmespath](https://pkg.go.dev/github.com/jmespath/go-jmespath) ([Apache-2.0](https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE))
  - [github.com/josharian/native](https://pkg.go.dev/github.com/josharian/native) ([MIT](https://github.com/josharian/native/blob/5c7d0dd6ab86/license))
  - [github.com/jsimonetti/rtnetlink](https://pkg.go.dev/github.com/jsimonetti/rtnetlink) ([MIT](https://github.com/jsimonetti/rtnetlink/blob/v1.4.1/LICENSE.md))
- - [github.com/klauspost/compress](https://pkg.go.dev/github.com/klauspost/compress) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.17.8/LICENSE))
- - [github.com/klauspost/compress/internal/snapref](https://pkg.go.dev/github.com/klauspost/compress/internal/snapref) ([BSD-3-Clause](https://github.com/klauspost/compress/blob/v1.17.8/internal/snapref/LICENSE))
- - [github.com/klauspost/compress/zstd/internal/xxhash](https://pkg.go.dev/github.com/klauspost/compress/zstd/internal/xxhash) ([MIT](https://github.com/klauspost/compress/blob/v1.17.8/zstd/internal/xxhash/LICENSE.txt))
+ - [github.com/klauspost/compress](https://pkg.go.dev/github.com/klauspost/compress) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.17.11/LICENSE))
+ - [github.com/klauspost/compress/internal/snapref](https://pkg.go.dev/github.com/klauspost/compress/internal/snapref) ([BSD-3-Clause](https://github.com/klauspost/compress/blob/v1.17.11/internal/snapref/LICENSE))
+ - [github.com/klauspost/compress/zstd/internal/xxhash](https://pkg.go.dev/github.com/klauspost/compress/zstd/internal/xxhash) ([MIT](https://github.com/klauspost/compress/blob/v1.17.11/zstd/internal/xxhash/LICENSE.txt))
  - [github.com/kortschak/wol](https://pkg.go.dev/github.com/kortschak/wol) ([BSD-3-Clause](https://github.com/kortschak/wol/blob/da482cc4850a/LICENSE))
  - [github.com/mdlayher/genetlink](https://pkg.go.dev/github.com/mdlayher/genetlink) ([MIT](https://github.com/mdlayher/genetlink/blob/v1.3.2/LICENSE.md))
  - [github.com/mdlayher/netlink](https://pkg.go.dev/github.com/mdlayher/netlink) ([MIT](https://github.com/mdlayher/netlink/blob/v1.7.2/LICENSE.md))
@@ -74,12 +73,12 @@ See also the dependencies in the [Tailscale CLI][].
  - [go4.org/mem](https://pkg.go.dev/go4.org/mem) ([Apache-2.0](https://github.com/go4org/mem/blob/ae6ca9944745/LICENSE))
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.28.0:LICENSE))
- - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/fe59bbe5:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.27.0:LICENSE))
- - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.8.0:LICENSE))
- - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.26.0:LICENSE))
+ - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/fc45aab8:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.30.0:LICENSE))
+ - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.9.0:LICENSE))
+ - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.27.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.25.0:LICENSE))
- - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.19.0:LICENSE))
+ - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.20.0:LICENSE))
  - [golang.org/x/time/rate](https://pkg.go.dev/golang.org/x/time/rate) ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE))
  - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/64c016c92987/LICENSE))
  - [tailscale.com](https://pkg.go.dev/tailscale.com) ([BSD-3-Clause](https://github.com/tailscale/tailscale/blob/HEAD/LICENSE))

licenses/tailscale.md

@@ -58,9 +58,9 @@ Some packages may only be included on certain architectures or operating systems
  - [github.com/jmespath/go-jmespath](https://pkg.go.dev/github.com/jmespath/go-jmespath) ([Apache-2.0](https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE))
  - [github.com/josharian/native](https://pkg.go.dev/github.com/josharian/native) ([MIT](https://github.com/josharian/native/blob/5c7d0dd6ab86/license))
  - [github.com/kballard/go-shellquote](https://pkg.go.dev/github.com/kballard/go-shellquote) ([MIT](https://github.com/kballard/go-shellquote/blob/95032a82bc51/LICENSE))
- - [github.com/klauspost/compress](https://pkg.go.dev/github.com/klauspost/compress) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.17.4/LICENSE))
- - [github.com/klauspost/compress/internal/snapref](https://pkg.go.dev/github.com/klauspost/compress/internal/snapref) ([BSD-3-Clause](https://github.com/klauspost/compress/blob/v1.17.4/internal/snapref/LICENSE))
- - [github.com/klauspost/compress/zstd/internal/xxhash](https://pkg.go.dev/github.com/klauspost/compress/zstd/internal/xxhash) ([MIT](https://github.com/klauspost/compress/blob/v1.17.4/zstd/internal/xxhash/LICENSE.txt))
+ - [github.com/klauspost/compress](https://pkg.go.dev/github.com/klauspost/compress) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.17.11/LICENSE))
+ - [github.com/klauspost/compress/internal/snapref](https://pkg.go.dev/github.com/klauspost/compress/internal/snapref) ([BSD-3-Clause](https://github.com/klauspost/compress/blob/v1.17.11/internal/snapref/LICENSE))
+ - [github.com/klauspost/compress/zstd/internal/xxhash](https://pkg.go.dev/github.com/klauspost/compress/zstd/internal/xxhash) ([MIT](https://github.com/klauspost/compress/blob/v1.17.11/zstd/internal/xxhash/LICENSE.txt))
  - [github.com/kortschak/wol](https://pkg.go.dev/github.com/kortschak/wol) ([BSD-3-Clause](https://github.com/kortschak/wol/blob/da482cc4850a/LICENSE))
  - [github.com/kr/fs](https://pkg.go.dev/github.com/kr/fs) ([BSD-3-Clause](https://github.com/kr/fs/blob/v0.1.0/LICENSE))
  - [github.com/mattn/go-colorable](https://pkg.go.dev/github.com/mattn/go-colorable) ([MIT](https://github.com/mattn/go-colorable/blob/v0.1.13/LICENSE))
@@ -84,7 +84,7 @@ Some packages may only be included on certain architectures or operating systems
  - [github.com/tailscale/peercred](https://pkg.go.dev/github.com/tailscale/peercred) ([BSD-3-Clause](https://github.com/tailscale/peercred/blob/b535050b2aa4/LICENSE))
  - [github.com/tailscale/web-client-prebuilt](https://pkg.go.dev/github.com/tailscale/web-client-prebuilt) ([BSD-3-Clause](https://github.com/tailscale/web-client-prebuilt/blob/5db17b287bf1/LICENSE))
  - [github.com/tailscale/wf](https://pkg.go.dev/github.com/tailscale/wf) ([BSD-3-Clause](https://github.com/tailscale/wf/blob/6fbb0a674ee6/LICENSE))
- - [github.com/tailscale/wireguard-go](https://pkg.go.dev/github.com/tailscale/wireguard-go) ([MIT](https://github.com/tailscale/wireguard-go/blob/799c1978fafc/LICENSE))
+ - [github.com/tailscale/wireguard-go](https://pkg.go.dev/github.com/tailscale/wireguard-go) ([MIT](https://github.com/tailscale/wireguard-go/blob/4e883d38c8d3/LICENSE))
  - [github.com/tailscale/xnet/webdav](https://pkg.go.dev/github.com/tailscale/xnet/webdav) ([BSD-3-Clause](https://github.com/tailscale/xnet/blob/8497ac4dab2e/LICENSE))
  - [github.com/tcnksm/go-httpstat](https://pkg.go.dev/github.com/tcnksm/go-httpstat) ([MIT](https://github.com/tcnksm/go-httpstat/blob/v0.2.0/LICENSE))
  - [github.com/toqueteos/webbrowser](https://pkg.go.dev/github.com/toqueteos/webbrowser) ([MIT](https://github.com/toqueteos/webbrowser/blob/v1.2.0/LICENSE.md))
@@ -98,8 +98,8 @@ Some packages may only be included on certain architectures or operating systems
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/1b970713:LICENSE))
  - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.27.0:LICENSE))
  - [golang.org/x/oauth2](https://pkg.go.dev/golang.org/x/oauth2) ([BSD-3-Clause](https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE))
- - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.7.0:LICENSE))
- - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.22.0:LICENSE))
+ - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.9.0:LICENSE))
+ - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.27.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.22.0:LICENSE))
  - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.16.0:LICENSE))
  - [golang.org/x/time/rate](https://pkg.go.dev/golang.org/x/time/rate) ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE))

licenses/windows.md

@@ -13,22 +13,22 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/alexbrainman/sspi](https://pkg.go.dev/github.com/alexbrainman/sspi) ([BSD-3-Clause](https://github.com/alexbrainman/sspi/blob/1a75b4708caa/LICENSE))
  - [github.com/apenwarr/fixconsole](https://pkg.go.dev/github.com/apenwarr/fixconsole) ([Apache-2.0](https://github.com/apenwarr/fixconsole/blob/5a9f6489cc29/LICENSE))
  - [github.com/apenwarr/w32](https://pkg.go.dev/github.com/apenwarr/w32) ([BSD-3-Clause](https://github.com/apenwarr/w32/blob/aa00fece76ab/LICENSE))
- - [github.com/aws/aws-sdk-go-v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/v1.30.4/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/v1.32.4/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/config](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/config) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.27.28/config/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/credentials](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/credentials) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.17.28/credentials/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/feature/ec2/imds](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/feature/ec2/imds) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.16.12/feature/ec2/imds/LICENSE.txt))
- - [github.com/aws/aws-sdk-go-v2/internal/configsources](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/configsources) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.3.16/internal/configsources/LICENSE.txt))
- - [github.com/aws/aws-sdk-go-v2/internal/endpoints/v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.6.16/internal/endpoints/v2/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2/internal/configsources](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/configsources) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.3.23/internal/configsources/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2/internal/endpoints/v2](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.6.23/internal/endpoints/v2/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/internal/ini](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/ini) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.8.1/internal/ini/LICENSE.txt))
- - [github.com/aws/aws-sdk-go-v2/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/aws-sdk-go-v2/blob/v1.30.4/internal/sync/singleflight/LICENSE))
- - [github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/internal/accept-encoding/v1.11.4/service/internal/accept-encoding/LICENSE.txt))
+ - [github.com/aws/aws-sdk-go-v2/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/aws-sdk-go-v2/blob/v1.32.4/internal/sync/singleflight/LICENSE))
+ - [github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/internal/accept-encoding/v1.12.0/service/internal/accept-encoding/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/internal/presigned-url](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.11.18/service/internal/presigned-url/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/ssm](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ssm) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/ssm/v1.45.0/service/ssm/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/sso](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/sso) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.22.5/service/sso/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/ssooidc](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/ssooidc) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/ssooidc/v1.26.5/service/ssooidc/LICENSE.txt))
  - [github.com/aws/aws-sdk-go-v2/service/sts](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/sts) ([Apache-2.0](https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.30.4/service/sts/LICENSE.txt))
- - [github.com/aws/smithy-go](https://pkg.go.dev/github.com/aws/smithy-go) ([Apache-2.0](https://github.com/aws/smithy-go/blob/v1.20.4/LICENSE))
- - [github.com/aws/smithy-go/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/smithy-go/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/smithy-go/blob/v1.20.4/internal/sync/singleflight/LICENSE))
+ - [github.com/aws/smithy-go](https://pkg.go.dev/github.com/aws/smithy-go) ([Apache-2.0](https://github.com/aws/smithy-go/blob/v1.22.0/LICENSE))
+ - [github.com/aws/smithy-go/internal/sync/singleflight](https://pkg.go.dev/github.com/aws/smithy-go/internal/sync/singleflight) ([BSD-3-Clause](https://github.com/aws/smithy-go/blob/v1.22.0/internal/sync/singleflight/LICENSE))
  - [github.com/coreos/go-iptables/iptables](https://pkg.go.dev/github.com/coreos/go-iptables/iptables) ([Apache-2.0](https://github.com/coreos/go-iptables/blob/65c67c9f46e6/LICENSE))
  - [github.com/dblohm7/wingoes](https://pkg.go.dev/github.com/dblohm7/wingoes) ([BSD-3-Clause](https://github.com/dblohm7/wingoes/blob/b75a8a7d7eb0/LICENSE))
  - [github.com/djherbis/times](https://pkg.go.dev/github.com/djherbis/times) ([MIT](https://github.com/djherbis/times/blob/v1.6.0/LICENSE))
@@ -44,9 +44,9 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/jmespath/go-jmespath](https://pkg.go.dev/github.com/jmespath/go-jmespath) ([Apache-2.0](https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE))
  - [github.com/josharian/native](https://pkg.go.dev/github.com/josharian/native) ([MIT](https://github.com/josharian/native/blob/5c7d0dd6ab86/license))
  - [github.com/jsimonetti/rtnetlink](https://pkg.go.dev/github.com/jsimonetti/rtnetlink) ([MIT](https://github.com/jsimonetti/rtnetlink/blob/v1.4.1/LICENSE.md))
- - [github.com/klauspost/compress](https://pkg.go.dev/github.com/klauspost/compress) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.17.8/LICENSE))
- - [github.com/klauspost/compress/internal/snapref](https://pkg.go.dev/github.com/klauspost/compress/internal/snapref) ([BSD-3-Clause](https://github.com/klauspost/compress/blob/v1.17.8/internal/snapref/LICENSE))
- - [github.com/klauspost/compress/zstd/internal/xxhash](https://pkg.go.dev/github.com/klauspost/compress/zstd/internal/xxhash) ([MIT](https://github.com/klauspost/compress/blob/v1.17.8/zstd/internal/xxhash/LICENSE.txt))
+ - [github.com/klauspost/compress](https://pkg.go.dev/github.com/klauspost/compress) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.17.11/LICENSE))
+ - [github.com/klauspost/compress/internal/snapref](https://pkg.go.dev/github.com/klauspost/compress/internal/snapref) ([BSD-3-Clause](https://github.com/klauspost/compress/blob/v1.17.11/internal/snapref/LICENSE))
+ - [github.com/klauspost/compress/zstd/internal/xxhash](https://pkg.go.dev/github.com/klauspost/compress/zstd/internal/xxhash) ([MIT](https://github.com/klauspost/compress/blob/v1.17.11/zstd/internal/xxhash/LICENSE.txt))
  - [github.com/mdlayher/netlink](https://pkg.go.dev/github.com/mdlayher/netlink) ([MIT](https://github.com/mdlayher/netlink/blob/v1.7.2/LICENSE.md))
  - [github.com/mdlayher/socket](https://pkg.go.dev/github.com/mdlayher/socket) ([MIT](https://github.com/mdlayher/socket/blob/v0.5.0/LICENSE.md))
  - [github.com/miekg/dns](https://pkg.go.dev/github.com/miekg/dns) ([BSD-3-Clause](https://github.com/miekg/dns/blob/v1.1.58/LICENSE))
@@ -66,14 +66,14 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [go4.org/mem](https://pkg.go.dev/go4.org/mem) ([Apache-2.0](https://github.com/go4org/mem/blob/ae6ca9944745/LICENSE))
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.28.0:LICENSE))
- - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/fe59bbe5:LICENSE))
+ - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/fc45aab8:LICENSE))
  - [golang.org/x/image/bmp](https://pkg.go.dev/golang.org/x/image/bmp) ([BSD-3-Clause](https://cs.opensource.google/go/x/image/+/v0.18.0:LICENSE))
  - [golang.org/x/mod](https://pkg.go.dev/golang.org/x/mod) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.19.0:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.27.0:LICENSE))
- - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.8.0:LICENSE))
- - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.26.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.30.0:LICENSE))
+ - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.9.0:LICENSE))
+ - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.27.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.25.0:LICENSE))
- - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.19.0:LICENSE))
+ - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.20.0:LICENSE))
  - [golang.zx2c4.com/wintun](https://pkg.go.dev/golang.zx2c4.com/wintun) ([MIT](https://git.zx2c4.com/wintun-go/tree/LICENSE?id=0fa3db229ce2))
  - [golang.zx2c4.com/wireguard/windows/tunnel/winipcfg](https://pkg.go.dev/golang.zx2c4.com/wireguard/windows/tunnel/winipcfg) ([MIT](https://git.zx2c4.com/wireguard-windows/tree/COPYING?h=v0.5.3))
  - [gopkg.in/Knetic/govaluate.v3](https://pkg.go.dev/gopkg.in/Knetic/govaluate.v3) ([MIT](https://github.com/Knetic/govaluate/blob/v3.0.0/LICENSE))

net/captivedetection/captivedetection.go

@@ -136,26 +136,31 @@ func interfaceNameDoesNotNeedCaptiveDetection(ifName string, goos string) bool {
 func (d *Detector) detectOnInterface(ctx context.Context, ifIndex int, endpoints []Endpoint) bool {
 	defer d.httpClient.CloseIdleConnections()
 
-	d.logf("[v2] %d available captive portal detection endpoints: %v", len(endpoints), endpoints)
+	use := min(len(endpoints), 5)
+	endpoints = endpoints[:use]
+	d.logf("[v2] %d available captive portal detection endpoints; trying %v", len(endpoints), use)
 
 	// We try to detect the captive portal more quickly by making requests to multiple endpoints concurrently.
 	var wg sync.WaitGroup
 	resultCh := make(chan bool, len(endpoints))
 
-	for i, e := range endpoints {
-		if i >= 5 {
-			// Try a maximum of 5 endpoints, break out (returning false) if we run of attempts.
-			break
-		}
+	// Once any goroutine detects a captive portal, we shut down the others.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	for _, e := range endpoints {
 		wg.Add(1)
 		go func(endpoint Endpoint) {
 			defer wg.Done()
 			found, err := d.verifyCaptivePortalEndpoint(ctx, endpoint, ifIndex)
 			if err != nil {
-				d.logf("[v1] checkCaptivePortalEndpoint failed with endpoint %v: %v", endpoint, err)
+				if ctx.Err() == nil {
+					d.logf("[v1] checkCaptivePortalEndpoint failed with endpoint %v: %v", endpoint, err)
+				}
 				return
 			}
 			if found {
+				cancel() // one match is good enough
 				resultCh <- true
 			}
 		}(e)

net/captivedetection/captivedetection_test.go

@@ -7,10 +7,12 @@ import (
 	"context"
 	"runtime"
 	"sync"
+	"sync/atomic"
 	"testing"
 
-	"tailscale.com/cmd/testwrapper/flakytest"
 	"tailscale.com/net/netmon"
+	"tailscale.com/syncs"
+	"tailscale.com/tstest/nettest"
 )
 
 func TestAvailableEndpointsAlwaysAtLeastTwo(t *testing.T) {
@@ -36,25 +38,46 @@ func TestDetectCaptivePortalReturnsFalse(t *testing.T) {
 	}
 }
 
-func TestAllEndpointsAreUpAndReturnExpectedResponse(t *testing.T) {
-	flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/13019")
+func TestEndpointsAreUpAndReturnExpectedResponse(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
+
 	d := NewDetector(t.Logf)
 	endpoints := availableEndpoints(nil, 0, t.Logf, runtime.GOOS)
+	t.Logf("testing %d endpoints", len(endpoints))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	var good atomic.Bool
 
 	var wg sync.WaitGroup
+	sem := syncs.NewSemaphore(5)
 	for _, e := range endpoints {
 		wg.Add(1)
 		go func(endpoint Endpoint) {
 			defer wg.Done()
-			found, err := d.verifyCaptivePortalEndpoint(context.Background(), endpoint, 0)
-			if err != nil {
-				t.Errorf("verifyCaptivePortalEndpoint failed with endpoint %v: %v", endpoint, err)
+
+			if !sem.AcquireContext(ctx) {
+				return
+			}
+			defer sem.Release()
+
+			found, err := d.verifyCaptivePortalEndpoint(ctx, endpoint, 0)
+			if err != nil && ctx.Err() == nil {
+				t.Logf("verifyCaptivePortalEndpoint failed with endpoint %v: %v", endpoint, err)
 			}
 			if found {
-				t.Errorf("verifyCaptivePortalEndpoint with endpoint %v says we're behind a captive portal, but we aren't", endpoint)
+				t.Logf("verifyCaptivePortalEndpoint with endpoint %v says we're behind a captive portal, but we aren't", endpoint)
+				return
 			}
+			good.Store(true)
+			t.Logf("endpoint good: %v", endpoint)
+			cancel()
 		}(e)
 	}
 
 	wg.Wait()
+
+	if !good.Load() {
+		t.Errorf("no good endpoints found")
+	}
 }

net/ipset/ipset.go

@@ -82,8 +82,8 @@ func NewContainsIPFunc(addrs views.Slice[netip.Prefix]) func(ip netip.Addr) bool
 		pathForTest("bart")
 		// Built a bart table.
 		t := &bart.Table[struct{}]{}
-		for i := range addrs.Len() {
-			t.Insert(addrs.At(i), struct{}{})
+		for _, p := range addrs.All() {
+			t.Insert(p, struct{}{})
 		}
 		return bartLookup(t)
 	}
@@ -99,8 +99,8 @@ func NewContainsIPFunc(addrs views.Slice[netip.Prefix]) func(ip netip.Addr) bool
 	// General case:
 	pathForTest("ip-map")
 	m := set.Set[netip.Addr]{}
-	for i := range addrs.Len() {
-		m.Add(addrs.At(i).Addr())
+	for _, p := range addrs.All() {
+		m.Add(p.Addr())
 	}
 	return ipInMap(m)
 }

net/netmon/netmon_darwin.go

@@ -56,7 +56,18 @@ func (m *darwinRouteMon) Receive() (message, error) {
 		if err != nil {
 			return nil, err
 		}
-		msgs, err := route.ParseRIB(route.RIBTypeRoute, m.buf[:n])
+		msgs, err := func() (msgs []route.Message, err error) {
+			defer func() {
+				// TODO(raggi,#14201): remove once we've got a fix from
+				// golang/go#70528.
+				msg := recover()
+				if msg != nil {
+					msgs = nil
+					err = fmt.Errorf("panic in route.ParseRIB: %s", msg)
+				}
+			}()
+			return route.ParseRIB(route.RIBTypeRoute, m.buf[:n])
+		}()
 		if err != nil {
 			if debugRouteMessages {
 				m.logf("read %d bytes (% 02x), failed to parse RIB: %v", n, m.buf[:n], err)

net/tsaddr/tsaddr.go

@@ -66,15 +66,21 @@ const (
 	TailscaleServiceIPv6String = "fd7a:115c:a1e0::53"
 )
 
-// IsTailscaleIP reports whether ip is an IP address in a range that
+// IsTailscaleIP reports whether IP is an IP address in a range that
 // Tailscale assigns from.
 func IsTailscaleIP(ip netip.Addr) bool {
 	if ip.Is4() {
-		return CGNATRange().Contains(ip) && !ChromeOSVMRange().Contains(ip)
+		return IsTailscaleIPv4(ip)
 	}
 	return TailscaleULARange().Contains(ip)
 }
 
+// IsTailscaleIPv4 reports whether an IPv4 IP is an IP address that
+// Tailscale assigns from.
+func IsTailscaleIPv4(ip netip.Addr) bool {
+	return CGNATRange().Contains(ip) && !ChromeOSVMRange().Contains(ip)
+}
+
 // TailscaleULARange returns the IPv6 Unique Local Address range that
 // is the superset range that Tailscale assigns out of.
 func TailscaleULARange() netip.Prefix {
@@ -180,8 +186,7 @@ func PrefixIs6(p netip.Prefix) bool { return p.Addr().Is6() }
 // IPv6 /0 route.
 func ContainsExitRoutes(rr views.Slice[netip.Prefix]) bool {
 	var v4, v6 bool
-	for i := range rr.Len() {
-		r := rr.At(i)
+	for _, r := range rr.All() {
 		if r == allIPv4 {
 			v4 = true
 		} else if r == allIPv6 {
@@ -194,8 +199,8 @@ func ContainsExitRoutes(rr views.Slice[netip.Prefix]) bool {
 // ContainsExitRoute reports whether rr contains at least one of IPv4 or
 // IPv6 /0 (exit) routes.
 func ContainsExitRoute(rr views.Slice[netip.Prefix]) bool {
-	for i := range rr.Len() {
-		if rr.At(i).Bits() == 0 {
+	for _, r := range rr.All() {
+		if r.Bits() == 0 {
 			return true
 		}
 	}
@@ -205,8 +210,8 @@ func ContainsExitRoute(rr views.Slice[netip.Prefix]) bool {
 // ContainsNonExitSubnetRoutes reports whether v contains Subnet
 // Routes other than ExitNode Routes.
 func ContainsNonExitSubnetRoutes(rr views.Slice[netip.Prefix]) bool {
-	for i := range rr.Len() {
-		if rr.At(i).Bits() != 0 {
+	for _, r := range rr.All() {
+		if r.Bits() != 0 {
 			return true
 		}
 	}

net/tsaddr/tsaddr_test.go

@@ -222,3 +222,71 @@ func TestContainsExitRoute(t *testing.T) {
 		}
 	}
 }
+
+func TestIsTailscaleIPv4(t *testing.T) {
+	tests := []struct {
+		in   netip.Addr
+		want bool
+	}{
+		{
+			in:   netip.MustParseAddr("100.67.19.57"),
+			want: true,
+		},
+		{
+			in:   netip.MustParseAddr("10.10.10.10"),
+			want: false,
+		},
+		{
+
+			in:   netip.MustParseAddr("fd7a:115c:a1e0:3f2b:7a1d:4e88:9c2b:7f01"),
+			want: false,
+		},
+		{
+			in:   netip.MustParseAddr("bc9d:0aa0:1f0a:69ab:eb5c:28e0:5456:a518"),
+			want: false,
+		},
+		{
+			in:   netip.MustParseAddr("100.115.92.157"),
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		if got := IsTailscaleIPv4(tt.in); got != tt.want {
+			t.Errorf("IsTailscaleIPv4(%v) = %v, want %v", tt.in, got, tt.want)
+		}
+	}
+}
+
+func TestIsTailscaleIP(t *testing.T) {
+	tests := []struct {
+		in   netip.Addr
+		want bool
+	}{
+		{
+			in:   netip.MustParseAddr("100.67.19.57"),
+			want: true,
+		},
+		{
+			in:   netip.MustParseAddr("10.10.10.10"),
+			want: false,
+		},
+		{
+
+			in:   netip.MustParseAddr("fd7a:115c:a1e0:3f2b:7a1d:4e88:9c2b:7f01"),
+			want: true,
+		},
+		{
+			in:   netip.MustParseAddr("bc9d:0aa0:1f0a:69ab:eb5c:28e0:5456:a518"),
+			want: false,
+		},
+		{
+			in:   netip.MustParseAddr("100.115.92.157"),
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		if got := IsTailscaleIP(tt.in); got != tt.want {
+			t.Errorf("IsTailscaleIP(%v) = %v, want %v", tt.in, got, tt.want)
+		}
+	}
+}

net/tsdial/dnsmap.go

@@ -42,8 +42,8 @@ func dnsMapFromNetworkMap(nm *netmap.NetworkMap) dnsMap {
 		if dnsname.HasSuffix(nm.Name, suffix) {
 			ret[canonMapKey(dnsname.TrimSuffix(nm.Name, suffix))] = ip
 		}
-		for i := range addrs.Len() {
-			if addrs.At(i).Addr().Is4() {
+		for _, p := range addrs.All() {
+			if p.Addr().Is4() {
 				have4 = true
 			}
 		}
@@ -52,9 +52,8 @@ func dnsMapFromNetworkMap(nm *netmap.NetworkMap) dnsMap {
 		if p.Name() == "" {
 			continue
 		}
-		for i := range p.Addresses().Len() {
-			a := p.Addresses().At(i)
-			ip := a.Addr()
+		for _, pfx := range p.Addresses().All() {
+			ip := pfx.Addr()
 			if ip.Is4() && !have4 {
 				continue
 			}

prober/derp.go

@@ -45,6 +45,9 @@ type derpProber struct {
 	bwInterval  time.Duration
 	bwProbeSize int64
 
+	// Optionally restrict probes to a single regionCode.
+	regionCode string
+
 	// Probe class for fetching & updating the DERP map.
 	ProbeMap ProbeClass
 
@@ -97,6 +100,14 @@ func WithTLSProbing(interval time.Duration) DERPOpt {
 	}
 }
 
+// WithRegion restricts probing to the specified region identified by its code
+// (e.g. "lax"). This is case sensitive.
+func WithRegion(regionCode string) DERPOpt {
+	return func(d *derpProber) {
+		d.regionCode = regionCode
+	}
+}
+
 // DERP creates a new derpProber.
 //
 // If derpMapURL is "local", the DERPMap is fetched via
@@ -135,6 +146,10 @@ func (d *derpProber) probeMapFn(ctx context.Context) error {
 	defer d.Unlock()
 
 	for _, region := range d.lastDERPMap.Regions {
+		if d.skipRegion(region) {
+			continue
+		}
+
 		for _, server := range region.Nodes {
 			labels := Labels{
 				"region":    region.RegionCode,
@@ -316,6 +331,10 @@ func (d *derpProber) updateMap(ctx context.Context) error {
 	d.lastDERPMapAt = time.Now()
 	d.nodes = make(map[string]*tailcfg.DERPNode)
 	for _, reg := range d.lastDERPMap.Regions {
+		if d.skipRegion(reg) {
+			continue
+		}
+
 		for _, n := range reg.Nodes {
 			if existing, ok := d.nodes[n.Name]; ok {
 				return fmt.Errorf("derpmap has duplicate nodes: %+v and %+v", existing, n)
@@ -338,6 +357,10 @@ func (d *derpProber) ProbeUDP(ipaddr string, port int) ProbeClass {
 	}
 }
 
+func (d *derpProber) skipRegion(region *tailcfg.DERPRegion) bool {
+	return d.regionCode != "" && region.RegionCode != d.regionCode
+}
+
 func derpProbeUDP(ctx context.Context, ipStr string, port int) error {
 	pc, err := net.ListenPacket("udp", ":0")
 	if err != nil {

prober/derp_test.go

@@ -44,6 +44,19 @@ func TestDerpProber(t *testing.T) {
 					},
 				},
 			},
+			1: {
+				RegionID:   1,
+				RegionCode: "one",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:     "n3",
+						RegionID: 0,
+						HostName: "derpn3.tailscale.test",
+						IPv4:     "1.1.1.1",
+						IPv6:     "::1",
+					},
+				},
+			},
 		},
 	}
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -68,6 +81,7 @@ func TestDerpProber(t *testing.T) {
 		meshProbeFn:  func(_, _ string) ProbeClass { return FuncProbe(func(context.Context) error { return nil }) },
 		nodes:        make(map[string]*tailcfg.DERPNode),
 		probes:       make(map[string]*Probe),
+		regionCode:   "zero",
 	}
 	if err := dp.probeMapFn(context.Background()); err != nil {
 		t.Errorf("unexpected probeMapFn() error: %s", err)
@@ -84,9 +98,9 @@ func TestDerpProber(t *testing.T) {
 
 	// Add one more node and check that probes got created.
 	dm.Regions[0].Nodes = append(dm.Regions[0].Nodes, &tailcfg.DERPNode{
-		Name:     "n3",
+		Name:     "n4",
 		RegionID: 0,
-		HostName: "derpn3.tailscale.test",
+		HostName: "derpn4.tailscale.test",
 		IPv4:     "1.1.1.1",
 		IPv6:     "::1",
 	})
@@ -113,6 +127,19 @@ func TestDerpProber(t *testing.T) {
 	if len(dp.probes) != 4 {
 		t.Errorf("unexpected probes: %+v", dp.probes)
 	}
+
+	// Stop filtering regions.
+	dp.regionCode = ""
+	if err := dp.probeMapFn(context.Background()); err != nil {
+		t.Errorf("unexpected probeMapFn() error: %s", err)
+	}
+	if len(dp.nodes) != 2 {
+		t.Errorf("unexpected nodes: %+v", dp.nodes)
+	}
+	// 6 regular probes + 2 mesh probe
+	if len(dp.probes) != 8 {
+		t.Errorf("unexpected probes: %+v", dp.probes)
+	}
 }
 
 func TestRunDerpProbeNodePair(t *testing.T) {

sessionrecording/connect.go

@@ -7,6 +7,8 @@ package sessionrecording
 
 import (
 	"context"
+	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,12 +16,33 @@ import (
 	"net/http"
 	"net/http/httptrace"
 	"net/netip"
+	"sync/atomic"
 	"time"
 
+	"golang.org/x/net/http2"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/httpm"
 	"tailscale.com/util/multierr"
 )
 
+const (
+	// Timeout for an individual DialFunc call for a single recorder address.
+	perDialAttemptTimeout = 5 * time.Second
+	// Timeout for the V2 API HEAD probe request (supportsV2).
+	http2ProbeTimeout = 10 * time.Second
+	// Maximum timeout for trying all available recorders, including V2 API
+	// probes and dial attempts.
+	allDialAttemptsTimeout = 30 * time.Second
+)
+
+// uploadAckWindow is the period of time to wait for an ackFrame from recorder
+// before terminating the connection. This is a variable to allow overriding it
+// in tests.
+var uploadAckWindow = 30 * time.Second
+
+// DialFunc is a function for dialing the recorder.
+type DialFunc func(ctx context.Context, network, host string) (net.Conn, error)
+
 // ConnectToRecorder connects to the recorder at any of the provided addresses.
 // It returns the first successful response, or a multierr if all attempts fail.
 //
@@ -32,19 +55,15 @@ import (
 // attempts are in order the recorder(s) was attempted. If successful a
 // successful connection is made, the last attempt in the slice is the
 // attempt for connected recorder.
-func ConnectToRecorder(ctx context.Context, recs []netip.AddrPort, dial func(context.Context, string, string) (net.Conn, error)) (io.WriteCloser, []*tailcfg.SSHRecordingAttempt, <-chan error, error) {
+func ConnectToRecorder(ctx context.Context, recs []netip.AddrPort, dial DialFunc) (io.WriteCloser, []*tailcfg.SSHRecordingAttempt, <-chan error, error) {
 	if len(recs) == 0 {
 		return nil, nil, nil, errors.New("no recorders configured")
 	}
 	// We use a special context for dialing the recorder, so that we can
 	// limit the time we spend dialing to 30 seconds and still have an
 	// unbounded context for the upload.
-	dialCtx, dialCancel := context.WithTimeout(ctx, 30*time.Second)
+	dialCtx, dialCancel := context.WithTimeout(ctx, allDialAttemptsTimeout)
 	defer dialCancel()
-	hc, err := SessionRecordingClientForDialer(dialCtx, dial)
-	if err != nil {
-		return nil, nil, nil, err
-	}
 
 	var errs []error
 	var attempts []*tailcfg.SSHRecordingAttempt
@@ -54,74 +73,230 @@ func ConnectToRecorder(ctx context.Context, recs []netip.AddrPort, dial func(con
 		}
 		attempts = append(attempts, attempt)
 
-		// We dial the recorder and wait for it to send a 100-continue
-		// response before returning from this function. This ensures that
-		// the recorder is ready to accept the recording.
-
-		// got100 is closed when we receive the 100-continue response.
-		got100 := make(chan struct{})
-		ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
-			Got100Continue: func() {
-				close(got100)
-			},
-		})
-
-		pr, pw := io.Pipe()
-		req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://%s:%d/record", ap.Addr(), ap.Port()), pr)
+		var pw io.WriteCloser
+		var errChan <-chan error
+		var err error
+		hc := clientHTTP2(dialCtx, dial)
+		// We need to probe V2 support using a separate HEAD request. Sending
+		// an HTTP/2 POST request to a HTTP/1 server will just "hang" until the
+		// request body is closed (instead of returning a 404 as one would
+		// expect). Sending a HEAD request without a body does not have that
+		// problem.
+		if supportsV2(ctx, hc, ap) {
+			pw, errChan, err = connectV2(ctx, hc, ap)
+		} else {
+			pw, errChan, err = connectV1(ctx, clientHTTP1(dialCtx, dial), ap)
+		}
 		if err != nil {
-			err = fmt.Errorf("recording: error starting recording: %w", err)
+			err = fmt.Errorf("recording: error starting recording on %q: %w", ap, err)
 			attempt.FailureMessage = err.Error()
 			errs = append(errs, err)
 			continue
 		}
-		// We set the Expect header to 100-continue, so that the recorder
-		// will send a 100-continue response before it starts reading the
-		// request body.
-		req.Header.Set("Expect", "100-continue")
-
-		// errChan is used to indicate the result of the request.
-		errChan := make(chan error, 1)
-		go func() {
-			resp, err := hc.Do(req)
-			if err != nil {
-				errChan <- fmt.Errorf("recording: error starting recording: %w", err)
-				return
-			}
-			if resp.StatusCode != 200 {
-				errChan <- fmt.Errorf("recording: unexpected status: %v", resp.Status)
-				return
-			}
-			errChan <- nil
-		}()
-		select {
-		case <-got100:
-		case err := <-errChan:
-			// If we get an error before we get the 100-continue response,
-			// we need to try another recorder.
-			if err == nil {
-				// If the error is nil, we got a 200 response, which
-				// is unexpected as we haven't sent any data yet.
-				err = errors.New("recording: unexpected EOF")
-			}
-			attempt.FailureMessage = err.Error()
-			errs = append(errs, err)
-			continue // try the next recorder
-		}
 		return pw, attempts, errChan, nil
 	}
 	return nil, attempts, nil, multierr.New(errs...)
 }
 
-// SessionRecordingClientForDialer returns an http.Client that uses a clone of
-// the provided Dialer's PeerTransport to dial connections. This is used to make
-// requests to the session recording server to upload session recordings. It
-// uses the provided dialCtx to dial connections, and limits a single dial to 5
-// seconds.
-func SessionRecordingClientForDialer(dialCtx context.Context, dial func(context.Context, string, string) (net.Conn, error)) (*http.Client, error) {
-	tr := http.DefaultTransport.(*http.Transport).Clone()
+// supportsV2 checks whether a recorder instance supports the /v2/record
+// endpoint.
+func supportsV2(ctx context.Context, hc *http.Client, ap netip.AddrPort) bool {
+	ctx, cancel := context.WithTimeout(ctx, http2ProbeTimeout)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, httpm.HEAD, fmt.Sprintf("http://%s/v2/record", ap), nil)
+	if err != nil {
+		return false
+	}
+	resp, err := hc.Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	return resp.StatusCode == http.StatusOK && resp.ProtoMajor > 1
+}
 
+// connectV1 connects to the legacy /record endpoint on the recorder. It is
+// used for backwards-compatibility with older tsrecorder instances.
+//
+// On success, it returns a WriteCloser that can be used to upload the
+// recording, and a channel that will be sent an error (or nil) when the upload
+// fails or completes.
+func connectV1(ctx context.Context, hc *http.Client, ap netip.AddrPort) (io.WriteCloser, <-chan error, error) {
+	// We dial the recorder and wait for it to send a 100-continue
+	// response before returning from this function. This ensures that
+	// the recorder is ready to accept the recording.
+
+	// got100 is closed when we receive the 100-continue response.
+	got100 := make(chan struct{})
+	ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
+		Got100Continue: func() {
+			close(got100)
+		},
+	})
+
+	pr, pw := io.Pipe()
+	req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://%s/record", ap), pr)
+	if err != nil {
+		return nil, nil, err
+	}
+	// We set the Expect header to 100-continue, so that the recorder
+	// will send a 100-continue response before it starts reading the
+	// request body.
+	req.Header.Set("Expect", "100-continue")
+
+	// errChan is used to indicate the result of the request.
+	errChan := make(chan error, 1)
+	go func() {
+		defer close(errChan)
+		resp, err := hc.Do(req)
+		if err != nil {
+			errChan <- err
+			return
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			errChan <- fmt.Errorf("recording: unexpected status: %v", resp.Status)
+			return
+		}
+	}()
+	select {
+	case <-got100:
+		return pw, errChan, nil
+	case err := <-errChan:
+		// If we get an error before we get the 100-continue response,
+		// we need to try another recorder.
+		if err == nil {
+			// If the error is nil, we got a 200 response, which
+			// is unexpected as we haven't sent any data yet.
+			err = errors.New("recording: unexpected EOF")
+		}
+		return nil, nil, err
+	}
+}
+
+// connectV2 connects to the /v2/record endpoint on the recorder over HTTP/2.
+// It explicitly tracks ack frames sent in the response and terminates the
+// connection if sent recording data is un-acked for uploadAckWindow.
+//
+// On success, it returns a WriteCloser that can be used to upload the
+// recording, and a channel that will be sent an error (or nil) when the upload
+// fails or completes.
+func connectV2(ctx context.Context, hc *http.Client, ap netip.AddrPort) (io.WriteCloser, <-chan error, error) {
+	pr, pw := io.Pipe()
+	upload := &readCounter{r: pr}
+	req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://%s/v2/record", ap), upload)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// With HTTP/2, hc.Do will not block while the request body is being sent.
+	// It will return immediately and allow us to consume the response body at
+	// the same time.
+	resp, err := hc.Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, nil, fmt.Errorf("recording: unexpected status: %v", resp.Status)
+	}
+
+	errChan := make(chan error, 1)
+	acks := make(chan int64)
+	// Read acks from the response and send them to the acks channel.
+	go func() {
+		defer close(errChan)
+		defer close(acks)
+		defer resp.Body.Close()
+		defer pw.Close()
+		dec := json.NewDecoder(resp.Body)
+		for {
+			var frame v2ResponseFrame
+			if err := dec.Decode(&frame); err != nil {
+				if !errors.Is(err, io.EOF) {
+					errChan <- fmt.Errorf("recording: unexpected error receiving acks: %w", err)
+				}
+				return
+			}
+			if frame.Error != "" {
+				errChan <- fmt.Errorf("recording: received error from the recorder: %q", frame.Error)
+				return
+			}
+			select {
+			case acks <- frame.Ack:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	// Track acks from the acks channel.
+	go func() {
+		// Hack for tests: some tests modify uploadAckWindow and reset it when
+		// the test ends. This can race with t.Reset call below. Making a copy
+		// here is a lazy workaround to not wait for this goroutine to exit in
+		// the test cases.
+		uploadAckWindow := uploadAckWindow
+		// This timer fires if we didn't receive an ack for too long.
+		t := time.NewTimer(uploadAckWindow)
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				// Close the pipe which terminates the connection and cleans up
+				// other goroutines. Note that tsrecorder will send us ack
+				// frames even if there is no new data to ack. This helps
+				// detect broken recorder connection if the session is idle.
+				pr.CloseWithError(errNoAcks)
+				resp.Body.Close()
+				return
+			case _, ok := <-acks:
+				if !ok {
+					// acks channel closed means that the goroutine reading them
+					// finished, which means that the request has ended.
+					return
+				}
+				// TODO(awly): limit how far behind the received acks can be. This
+				// should handle scenarios where a session suddenly dumps a lot of
+				// output.
+				t.Reset(uploadAckWindow)
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return pw, errChan, nil
+}
+
+var errNoAcks = errors.New("did not receive ack frames from the recorder in 30s")
+
+type v2ResponseFrame struct {
+	// Ack is the number of bytes received from the client so far. The bytes
+	// are not guaranteed to be durably stored yet.
+	Ack int64 `json:"ack,omitempty"`
+	// Error is an error encountered while storing the recording. Error is only
+	// ever set as the last frame in the response.
+	Error string `json:"error,omitempty"`
+}
+
+// readCounter is an io.Reader that counts how many bytes were read.
+type readCounter struct {
+	r    io.Reader
+	sent atomic.Int64
+}
+
+func (u *readCounter) Read(buf []byte) (int, error) {
+	n, err := u.r.Read(buf)
+	u.sent.Add(int64(n))
+	return n, err
+}
+
+// clientHTTP1 returns a claassic http.Client with a per-dial context. It uses
+// dialCtx and adds a 5s timeout to it.
+func clientHTTP1(dialCtx context.Context, dial DialFunc) *http.Client {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		perAttemptCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		perAttemptCtx, cancel := context.WithTimeout(ctx, perDialAttemptTimeout)
 		defer cancel()
 		go func() {
 			select {
@@ -132,7 +307,32 @@ func SessionRecordingClientForDialer(dialCtx context.Context, dial func(context.
 		}()
 		return dial(perAttemptCtx, network, addr)
 	}
-	return &http.Client{
-		Transport: tr,
-	}, nil
+	return &http.Client{Transport: tr}
+}
+
+// clientHTTP2 is like clientHTTP1 but returns an http.Client suitable for h2c
+// requests (HTTP/2 over plaintext). Unfortunately the same client does not
+// work for HTTP/1 so we need to split these up.
+func clientHTTP2(dialCtx context.Context, dial DialFunc) *http.Client {
+	return &http.Client{
+		Transport: &http2.Transport{
+			// Allow "http://" scheme in URLs.
+			AllowHTTP: true,
+			// Pretend like we're using TLS, but actually use the provided
+			// DialFunc underneath. This is necessary to convince the transport
+			// to actually dial.
+			DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
+				perAttemptCtx, cancel := context.WithTimeout(ctx, perDialAttemptTimeout)
+				defer cancel()
+				go func() {
+					select {
+					case <-perAttemptCtx.Done():
+					case <-dialCtx.Done():
+						cancel()
+					}
+				}()
+				return dial(perAttemptCtx, network, addr)
+			},
+		},
+	}
 }

sessionrecording/connect_test.go

@@ -0,0 +1,189 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package sessionrecording
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+)
+
+func TestConnectToRecorder(t *testing.T) {
+	tests := []struct {
+		desc  string
+		http2 bool
+		// setup returns a recorder server mux, and a channel which sends the
+		// hash of the recording uploaded to it. The channel is expected to
+		// fire only once.
+		setup   func(t *testing.T) (*http.ServeMux, <-chan []byte)
+		wantErr bool
+	}{
+		{
+			desc: "v1 recorder",
+			setup: func(t *testing.T) (*http.ServeMux, <-chan []byte) {
+				uploadHash := make(chan []byte, 1)
+				mux := http.NewServeMux()
+				mux.HandleFunc("POST /record", func(w http.ResponseWriter, r *http.Request) {
+					hash := sha256.New()
+					if _, err := io.Copy(hash, r.Body); err != nil {
+						t.Error(err)
+					}
+					uploadHash <- hash.Sum(nil)
+				})
+				return mux, uploadHash
+			},
+		},
+		{
+			desc:  "v2 recorder",
+			http2: true,
+			setup: func(t *testing.T) (*http.ServeMux, <-chan []byte) {
+				uploadHash := make(chan []byte, 1)
+				mux := http.NewServeMux()
+				mux.HandleFunc("POST /record", func(w http.ResponseWriter, r *http.Request) {
+					t.Error("received request to v1 endpoint")
+					http.Error(w, "not found", http.StatusNotFound)
+				})
+				mux.HandleFunc("POST /v2/record", func(w http.ResponseWriter, r *http.Request) {
+					// Force the status to send to unblock the client waiting
+					// for it.
+					w.WriteHeader(http.StatusOK)
+					w.(http.Flusher).Flush()
+
+					body := &readCounter{r: r.Body}
+					hash := sha256.New()
+					ctx, cancel := context.WithCancel(r.Context())
+					go func() {
+						defer cancel()
+						if _, err := io.Copy(hash, body); err != nil {
+							t.Error(err)
+						}
+					}()
+
+					// Send acks for received bytes.
+					tick := time.NewTicker(time.Millisecond)
+					defer tick.Stop()
+					enc := json.NewEncoder(w)
+				outer:
+					for {
+						select {
+						case <-ctx.Done():
+							break outer
+						case <-tick.C:
+							if err := enc.Encode(v2ResponseFrame{Ack: body.sent.Load()}); err != nil {
+								t.Errorf("writing ack frame: %v", err)
+								break outer
+							}
+						}
+					}
+
+					uploadHash <- hash.Sum(nil)
+				})
+				// Probing HEAD endpoint which always returns 200 OK.
+				mux.HandleFunc("HEAD /v2/record", func(http.ResponseWriter, *http.Request) {})
+				return mux, uploadHash
+			},
+		},
+		{
+			desc:    "v2 recorder no acks",
+			http2:   true,
+			wantErr: true,
+			setup: func(t *testing.T) (*http.ServeMux, <-chan []byte) {
+				// Make the client no-ack timeout quick for the test.
+				oldAckWindow := uploadAckWindow
+				uploadAckWindow = 100 * time.Millisecond
+				t.Cleanup(func() { uploadAckWindow = oldAckWindow })
+
+				uploadHash := make(chan []byte, 1)
+				mux := http.NewServeMux()
+				mux.HandleFunc("POST /record", func(w http.ResponseWriter, r *http.Request) {
+					t.Error("received request to v1 endpoint")
+					http.Error(w, "not found", http.StatusNotFound)
+				})
+				mux.HandleFunc("POST /v2/record", func(w http.ResponseWriter, r *http.Request) {
+					// Force the status to send to unblock the client waiting
+					// for it.
+					w.WriteHeader(http.StatusOK)
+					w.(http.Flusher).Flush()
+
+					// Consume the whole request body but don't send any acks
+					// back.
+					hash := sha256.New()
+					if _, err := io.Copy(hash, r.Body); err != nil {
+						t.Error(err)
+					}
+					// Goes in the channel buffer, non-blocking.
+					uploadHash <- hash.Sum(nil)
+
+					// Block until the parent test case ends to prevent the
+					// request termination. We want to exercise the ack
+					// tracking logic specifically.
+					ctx, cancel := context.WithCancel(r.Context())
+					t.Cleanup(cancel)
+					<-ctx.Done()
+				})
+				mux.HandleFunc("HEAD /v2/record", func(http.ResponseWriter, *http.Request) {})
+				return mux, uploadHash
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			mux, uploadHash := tt.setup(t)
+
+			srv := httptest.NewUnstartedServer(mux)
+			if tt.http2 {
+				// Wire up h2c-compatible HTTP/2 server. This is optional
+				// because the v1 recorder didn't support HTTP/2 and we try to
+				// mimic that.
+				h2s := &http2.Server{}
+				srv.Config.Handler = h2c.NewHandler(mux, h2s)
+				if err := http2.ConfigureServer(srv.Config, h2s); err != nil {
+					t.Errorf("configuring HTTP/2 support in server: %v", err)
+				}
+			}
+			srv.Start()
+			t.Cleanup(srv.Close)
+
+			d := new(net.Dialer)
+
+			ctx := context.Background()
+			w, _, errc, err := ConnectToRecorder(ctx, []netip.AddrPort{netip.MustParseAddrPort(srv.Listener.Addr().String())}, d.DialContext)
+			if err != nil {
+				t.Fatalf("ConnectToRecorder: %v", err)
+			}
+
+			// Send some random data and hash it to compare with the recorded
+			// data hash.
+			hash := sha256.New()
+			const numBytes = 1 << 20 // 1MB
+			if _, err := io.CopyN(io.MultiWriter(w, hash), rand.Reader, numBytes); err != nil {
+				t.Fatalf("writing recording data: %v", err)
+			}
+			if err := w.Close(); err != nil {
+				t.Fatalf("closing recording stream: %v", err)
+			}
+			if err := <-errc; err != nil && !tt.wantErr {
+				t.Fatalf("error from the channel: %v", err)
+			} else if err == nil && tt.wantErr {
+				t.Fatalf("did not receive expected error from the channel")
+			}
+
+			if recv, sent := <-uploadHash, hash.Sum(nil); !bytes.Equal(recv, sent) {
+				t.Errorf("mismatch in recording data hash, sent %x, received %x", sent, recv)
+			}
+		})
+	}
+}

ssh/tailssh/tailssh.go

@@ -1170,7 +1170,7 @@ func (ss *sshSession) run() {
 		if err != nil && !errors.Is(err, io.EOF) {
 			isErrBecauseProcessExited := processDone.Load() && errors.Is(err, syscall.EIO)
 			if !isErrBecauseProcessExited {
-				logf("stdout copy: %v, %T", err)
+				logf("stdout copy: %v", err)
 				ss.cancelCtx(err)
 			}
 		}
@@ -1520,9 +1520,14 @@ func (ss *sshSession) startNewRecording() (_ *recording, err error) {
 		go func() {
 			err := <-errChan
 			if err == nil {
-				// Success.
-				ss.logf("recording: finished uploading recording")
-				return
+				select {
+				case <-ss.ctx.Done():
+					// Success.
+					ss.logf("recording: finished uploading recording")
+					return
+				default:
+					err = errors.New("recording upload ended before the SSH session")
+				}
 			}
 			if onFailure != nil && onFailure.NotifyURL != "" && len(attempts) > 0 {
 				lastAttempt := attempts[len(attempts)-1]