control/controlhttp, util/multierr: deduplicate context errors

Add a new function to the util/multierr package that allows
deduplicating the `context.Canceled` and `context.DeadlineExceeded`
errors from a slice, and then use that to reduce log spam in
controlhttp.

Updates #14233

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I5a121124fe27c4449705ba35de6faf83665db5fe

.github/workflows/codeql-analysis.yml

@@ -49,13 +49,13 @@ jobs:
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1

.github/workflows/golangci-lint.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
         with:
           go-version-file: go.mod
           cache: false

.github/workflows/test.yml

@@ -80,7 +80,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -153,13 +153,13 @@ jobs:
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -260,7 +260,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -319,7 +319,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -367,7 +367,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -461,7 +461,7 @@ jobs:
       run: |
         echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts

build_docker.sh

@@ -17,12 +17,20 @@ eval "$(./build_dist.sh shellvars)"
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
 DEFAULT_BASE="tailscale/alpine-base:3.18"
+# Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
+# Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
+# annotations defined here will be overriden by release scripts that call this script.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+DEFAULT_ANNOTATIONS="org.opencontainers.image.source=https://github.com/tailscale/tailscale/blob/main/build_docker.sh,org.opencontainers.image.vendor=Tailscale"
 
 PUSH="${PUSH:-false}"
 TARGET="${TARGET:-${DEFAULT_TARGET}}"
 TAGS="${TAGS:-${DEFAULT_TAGS}}"
 BASE="${BASE:-${DEFAULT_BASE}}"
 PLATFORM="${PLATFORM:-}" # default to all platforms
+# OCI annotations that will be added to the image.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
+ANNOTATIONS="${ANNOTATIONS:-${DEFAULT_ANNOTATIONS}}"
 
 case "$TARGET" in
   client)
@@ -43,6 +51,7 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/containerboot
     ;;
   operator)
@@ -60,6 +69,7 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/operator
     ;;
   k8s-nameserver)
@@ -77,6 +87,7 @@ case "$TARGET" in
       --repos="${REPOS}" \
       --push="${PUSH}" \
       --target="${PLATFORM}" \
+      --annotations="${ANNOTATIONS}" \
       /usr/local/bin/k8s-nameserver
     ;;
   *)

client/tailscale/localclient.go

@@ -40,6 +40,7 @@ import (
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
+	"tailscale.com/util/syspolicy/setting"
 )
 
 // defaultLocalClient is the default LocalClient when using the legacy
@@ -814,6 +815,33 @@ func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn
 	return decodeJSON[*ipn.Prefs](body)
 }
 
+// GetEffectivePolicy returns the effective policy for the specified scope.
+func (lc *LocalClient) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+	scopeID, err := scope.MarshalText()
+	if err != nil {
+		return nil, err
+	}
+	body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID))
+	if err != nil {
+		return nil, err
+	}
+	return decodeJSON[*setting.Snapshot](body)
+}
+
+// ReloadEffectivePolicy reloads the effective policy for the specified scope
+// by reading and merging policy settings from all applicable policy sources.
+func (lc *LocalClient) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) {
+	scopeID, err := scope.MarshalText()
+	if err != nil {
+		return nil, err
+	}
+	body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody)
+	if err != nil {
+		return nil, err
+	}
+	return decodeJSON[*setting.Snapshot](body)
+}
+
 // GetDNSOSConfig returns the system DNS configuration for the current device.
 // That is, it returns the DNS configuration that the system would use if Tailscale weren't being used.
 func (lc *LocalClient) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) {
@@ -1299,6 +1327,17 @@ func (lc *LocalClient) SetServeConfig(ctx context.Context, config *ipn.ServeConf
 	return nil
 }
 
+// DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be
+// run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over
+// to another replica whilst there is still some grace period for the existing connections to terminate.
+func (lc *LocalClient) DisconnectControl(ctx context.Context) error {
+	_, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil)
+	if err != nil {
+		return fmt.Errorf("error disconnecting control: %w", err)
+	}
+	return nil
+}
+
 // NetworkLockDisable shuts down network-lock across the tailnet.
 func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) error {
 	if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil {

cmd/containerboot/healthz.go

@@ -39,7 +39,7 @@ func runHealthz(addr string, h *healthz) {
 		log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
 	}
 	mux := http.NewServeMux()
-	mux.Handle("/healthz", h)
+	mux.Handle("GET /healthz", h)
 	log.Printf("Running healthcheck endpoint at %s/healthz", addr)
 	hs := &http.Server{Handler: mux}
 

cmd/containerboot/kube.go

@@ -61,7 +61,7 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 			Path: "/data/authkey",
 		},
 	}
-	if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil {
+	if err := kc.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
 		if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
 			// This is kubernetes-ese for "the field you asked to
 			// delete already doesn't exist", aka no-op.
@@ -81,7 +81,7 @@ func initKubeClient(root string) {
 		kubeclient.SetRootPathForTesting(root)
 	}
 	var err error
-	kc, err = kubeclient.New()
+	kc, err = kubeclient.New("tailscale-container")
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}

cmd/containerboot/main.go

@@ -102,7 +102,6 @@ import (
 	"net/netip"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -179,6 +178,14 @@ func main() {
 	}
 	defer killTailscaled()
 
+	if cfg.LocalAddrPort != "" && cfg.MetricsEnabled {
+		m := &metrics{
+			lc:            client,
+			debugEndpoint: cfg.DebugAddrPort,
+		}
+		runMetrics(cfg.LocalAddrPort, m)
+	}
+
 	if cfg.EnableForwardingOptimizations {
 		if err := client.SetUDPGROForwarding(bootCtx); err != nil {
 			log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
@@ -731,7 +738,6 @@ func tailscaledConfigFilePath() string {
 		}
 		cv, err := kubeutils.CapVerFromFileName(e.Name())
 		if err != nil {
-			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
 			continue
 		}
 		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
@@ -739,8 +745,9 @@ func tailscaledConfigFilePath() string {
 		}
 	}
 	if maxCompatVer == -1 {
-		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
+		log.Fatalf("no tailscaled config file found in %q for current capability version %d", dir, tailcfg.CurrentCapabilityVersion)
 	}
-	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	filePath := filepath.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	log.Printf("Using tailscaled config file %q to match current capability version %d", filePath, tailcfg.CurrentCapabilityVersion)
+	return filePath
 }

cmd/containerboot/metrics.go

@@ -0,0 +1,91 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
+)
+
+// metrics is a simple metrics HTTP server, if enabled it forwards requests to
+// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
+type metrics struct {
+	debugEndpoint string
+	lc            *tailscale.LocalClient
+}
+
+func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+
+	resp, err := do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	for key, val := range resp.Header {
+		for _, v := range val {
+			w.Header().Add(key, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
+	localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
+	proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
+}
+
+func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
+	if m.debugEndpoint == "" {
+		http.Error(w, "debug endpoint not configured", http.StatusNotFound)
+		return
+	}
+
+	debugURL := "http://" + m.debugEndpoint + r.URL.Path
+	proxy(w, r, debugURL, http.DefaultClient.Do)
+}
+
+// runMetrics runs a simple HTTP metrics endpoint at <addr>/metrics, forwarding
+// requests to tailscaled's /localapi/v0/usermetrics API.
+//
+// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
+// endpoint if configured to ease migration for a breaking change serving user
+// metrics instead of debug metrics on the "metrics" port.
+func runMetrics(addr string, m *metrics) {
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("error listening on the provided metrics endpoint address %q: %v", addr, err)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /metrics", m.handleMetrics)
+	mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
+
+	log.Printf("Running metrics endpoint at %s/metrics", addr)
+	ms := &http.Server{Handler: mux}
+
+	go func() {
+		if err := ms.Serve(ln); err != nil {
+			log.Fatalf("failed running metrics endpoint: %v", err)
+		}
+	}()
+}

cmd/containerboot/services.go

@@ -389,7 +389,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta
 		Path:  fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
 		Value: bs,
 	}
-	if err := ep.kc.JSONPatchSecret(ctx, ep.stateSecret, []kubeclient.JSONPatch{patch}); err != nil {
+	if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil {
 		return fmt.Errorf("error patching state Secret: %w", err)
 	}
 	ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()

cmd/containerboot/settings.go

@@ -67,11 +67,18 @@ type settings struct {
 	PodIP               string
 	PodIPv4             string
 	PodIPv6             string
-	HealthCheckAddrPort string
+	HealthCheckAddrPort string // TODO(tomhjp): use the local addr/port instead.
+	LocalAddrPort       string
+	MetricsEnabled      bool
+	DebugAddrPort       string
 	EgressSvcsCfgPath   string
 }
 
 func configFromEnv() (*settings, error) {
+	defaultLocalAddrPort := ""
+	if v, ok := os.LookupEnv("POD_IP"); ok && v != "" {
+		defaultLocalAddrPort = fmt.Sprintf("%s:9002", v)
+	}
 	cfg := &settings{
 		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
 		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
@@ -98,6 +105,9 @@ func configFromEnv() (*settings, error) {
 		PodIP:                                 defaultEnv("POD_IP", ""),
 		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
 		HealthCheckAddrPort:                   defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
+		LocalAddrPort:                         defaultEnv("TS_LOCAL_ADDR_PORT", defaultLocalAddrPort),
+		MetricsEnabled:                        defaultBool("TS_METRICS_ENABLED", false),
+		DebugAddrPort:                         defaultEnv("TS_DEBUG_ADDR_PORT", ""),
 		EgressSvcsCfgPath:                     defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
 	}
 	podIPs, ok := os.LookupEnv("POD_IPS")
@@ -175,6 +185,16 @@ func (s *settings) validate() error {
 			return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
 		}
 	}
+	if s.LocalAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
+		}
+	}
+	if s.DebugAddrPort != "" {
+		if _, err := netip.ParseAddrPort(s.DebugAddrPort); err != nil {
+			return fmt.Errorf("error parsing TS_DEBUG_ADDR_PORT value %q: %w", s.DebugAddrPort, err)
+		}
+	}
 	return nil
 }
 

cmd/containerboot/tailscaled.go

@@ -90,6 +90,12 @@ func tailscaledArgs(cfg *settings) []string {
 	if cfg.TailscaledConfigFilePath != "" {
 		args = append(args, "--config="+cfg.TailscaledConfigFilePath)
 	}
+	// Once enough proxy versions have been released for all the supported
+	// versions to understand this cfg setting, the operator can stop
+	// setting TS_TAILSCALED_EXTRA_ARGS for the debug flag.
+	if cfg.DebugAddrPort != "" && !strings.Contains(cfg.DaemonExtraArgs, cfg.DebugAddrPort) {
+		args = append(args, "--debug="+cfg.DebugAddrPort)
+	}
 	if cfg.DaemonExtraArgs != "" {
 		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
 	}

cmd/derper/depaware.txt

@@ -27,7 +27,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
@@ -116,7 +115,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
-        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
+        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper
         tailscale.com/paths                                          from tailscale.com/client/tailscale
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
@@ -140,6 +139,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/ipn
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/ipn+
@@ -151,24 +151,29 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/set                                       from tailscale.com/derp+
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
         tailscale.com/util/slicesx                                   from tailscale.com/cmd/derper+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/testenv                                   from tailscale.com/util/syspolicy+
         tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
@@ -189,7 +194,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
-        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -238,7 +243,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         crypto/tls                                                   from golang.org/x/crypto/acme+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -250,7 +254,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper
+        flag                                                         from tailscale.com/cmd/derper+
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -258,6 +262,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
+        html/template                                                from tailscale.com/cmd/derper
         io                                                           from bufio+
         io/fs                                                        from crypto/x509+
         io/ioutil                                                    from github.com/mitchellh/go-ps+
@@ -269,7 +274,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -284,7 +289,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/derper
-   W    os/user                                                      from tailscale.com/util/winutil
+   W    os/user                                                      from tailscale.com/util/winutil+
         path                                                         from github.com/prometheus/client_golang/prometheus/internal+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
@@ -302,6 +307,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
         text/tabwriter                                               from runtime/pprof
+        text/template                                                from html/template
+        text/template/parse                                          from html/template+
         time                                                         from compress/gzip+
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+

cmd/derper/derper.go

@@ -19,6 +19,7 @@ import (
 	"expvar"
 	"flag"
 	"fmt"
+	"html/template"
 	"io"
 	"log"
 	"math"
@@ -212,25 +213,16 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
 		w.WriteHeader(200)
-		io.WriteString(w, `<html><body>
-<h1>DERP</h1>
-<p>
-  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
-</p>
-<p>
-  Documentation:
-</p>
-<ul>
-  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
-  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
-  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
-</ul>
-`)
-		if !*runDERP {
-			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
-		}
-		if tsweb.AllowDebugAccess(r) {
-			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
+		err := homePageTemplate.Execute(w, templateData{
+			ShowAbuseInfo: validProdHostname.MatchString(*hostname),
+			Disabled:      !*runDERP,
+			AllowDebug:    tsweb.AllowDebugAccess(r),
+		})
+		if err != nil {
+			if r.Context().Err() == nil {
+				log.Printf("homePageTemplate.Execute: %v", err)
+			}
+			return
 		}
 	}))
 	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -468,3 +460,52 @@ func init() {
 		return 0
 	}))
 }
+
+type templateData struct {
+	ShowAbuseInfo bool
+	Disabled      bool
+	AllowDebug    bool
+}
+
+// homePageTemplate renders the home page using [templateData].
+var homePageTemplate = template.Must(template.New("home").Parse(`<html><body>
+<h1>DERP</h1>
+<p>
+  This is a <a href="https://tailscale.com/">Tailscale</a> DERP server.
+</p>
+
+<p>
+  It provides STUN, interactive connectivity establishment, and relaying of end-to-end encrypted traffic
+  for Tailscale clients.
+</p>
+
+{{if .ShowAbuseInfo }}
+<p>
+  If you suspect abuse, please contact <a href="mailto:security@tailscale.com">security@tailscale.com</a>.
+</p>
+{{end}}
+
+<p>
+  Documentation:
+</p>
+
+<ul>
+{{if .ShowAbuseInfo }}
+  <li><a href="https://tailscale.com/security-policies">Tailscale Security Policies</a></li>
+  <li><a href="https://tailscale.com/tailscale-aup">Tailscale Acceptable Use Policies</a></li>
+{{end}}
+  <li><a href="https://tailscale.com/kb/1232/derp-servers">About DERP</a></li>
+  <li><a href="https://pkg.go.dev/tailscale.com/derp">Protocol & Go docs</a></li>
+  <li><a href="https://github.com/tailscale/tailscale/tree/main/cmd/derper#derp">How to run a DERP server</a></li>
+</ul>
+
+{{if .Disabled}}
+<p>Status: <b>disabled</b></p>
+{{end}}
+
+{{if .AllowDebug}}
+<p>Debug info at <a href='/debug/'>/debug/</a>.</p>
+{{end}}
+</body>
+</html>
+`))

cmd/derper/derper_test.go

@@ -4,7 +4,9 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -107,6 +109,34 @@ func TestDeps(t *testing.T) {
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
 			"tailscale.com/net/packet":           "not needed in derper",
 			"github.com/gaissmai/bart":           "not needed in derper",
+			"database/sql/driver":                "not needed in derper", // previously came in via github.com/google/uuid
 		},
 	}.Check(t)
 }
+
+func TestTemplate(t *testing.T) {
+	buf := &bytes.Buffer{}
+	err := homePageTemplate.Execute(buf, templateData{
+		ShowAbuseInfo: true,
+		Disabled:      true,
+		AllowDebug:    true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	str := buf.String()
+	if !strings.Contains(str, "If you suspect abuse") {
+		t.Error("Output is missing abuse mailto")
+	}
+	if !strings.Contains(str, "Tailscale Security Policies") {
+		t.Error("Output is missing Tailscale Security Policies link")
+	}
+	if !strings.Contains(str, "Status:") {
+		t.Error("Output is missing disabled status")
+	}
+	if !strings.Contains(str, "Debug info") {
+		t.Error("Output is missing debug info")
+	}
+	fmt.Println(buf.String())
+}

cmd/derpprobe/derpprobe.go

@@ -29,6 +29,7 @@ var (
 	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
 	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
 	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	regionCode   = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -47,6 +48,9 @@ func main() {
 	if *bwInterval > 0 {
 		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
 	}
+	if *regionCode != "" {
+		opts = append(opts, prober.WithRegion(*regionCode))
+	}
 	dp, err := prober.DERP(p, *derpMapURL, opts...)
 	if err != nil {
 		log.Fatal(err)

cmd/k8s-operator/connector.go

@@ -13,7 +13,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pkg/errors"
+	"errors"
+
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
@@ -58,6 +59,7 @@ type ConnectorReconciler struct {
 
 	subnetRouters set.Slice[types.UID] // for subnet routers gauge
 	exitNodes     set.Slice[types.UID] // for exit nodes gauge
+	appConnectors set.Slice[types.UID] // for app connectors gauge
 }
 
 var (
@@ -67,6 +69,8 @@ var (
 	gaugeConnectorSubnetRouterResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithSubnetRouterCount)
 	// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
 	gaugeConnectorExitNodeResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithExitNodeCount)
+	// gaugeConnectorAppConnectorResources tracks the number of Connectors currently managed by this operator instance that are app connectors.
+	gaugeConnectorAppConnectorResources = clientmetric.NewGauge(kubetypes.MetricConnectorWithAppConnectorCount)
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
@@ -108,13 +112,12 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	oldCnStatus := cn.Status.DeepCopy()
 	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
+		var updateErr error
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
-			if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
-				err = errors.Wrap(err, updateErr.Error())
-			}
+			updateErr = a.Client.Status().Update(ctx, cn)
 		}
-		return res, err
+		return res, errors.Join(err, updateErr)
 	}
 
 	if !slices.Contains(cn.Finalizers, FinalizerName) {
@@ -150,6 +153,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 		return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 	}
+	if cn.Spec.AppConnector != nil {
+		cn.Status.IsAppConnector = true
+	}
 	cn.Status.SubnetRoutes = ""
 	return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
 }
@@ -189,23 +195,37 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
 	}
 
+	if cn.Spec.AppConnector != nil {
+		sts.Connector.isAppConnector = true
+		if len(cn.Spec.AppConnector.Routes) != 0 {
+			sts.Connector.routes = cn.Spec.AppConnector.Routes.Stringify()
+		}
+	}
+
 	a.mu.Lock()
-	if sts.Connector.isExitNode {
+	if cn.Spec.ExitNode {
 		a.exitNodes.Add(cn.UID)
 	} else {
 		a.exitNodes.Remove(cn.UID)
 	}
-	if sts.Connector.routes != "" {
+	if cn.Spec.SubnetRouter != nil {
 		a.subnetRouters.Add(cn.GetUID())
 	} else {
 		a.subnetRouters.Remove(cn.GetUID())
 	}
+	if cn.Spec.AppConnector != nil {
+		a.appConnectors.Add(cn.GetUID())
+	} else {
+		a.appConnectors.Remove(cn.GetUID())
+	}
 	a.mu.Unlock()
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
@@ -248,12 +268,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
 	a.mu.Lock()
 	a.subnetRouters.Remove(cn.UID)
 	a.exitNodes.Remove(cn.UID)
+	a.appConnectors.Remove(cn.UID)
 	a.mu.Unlock()
 	gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
 	gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
+	gaugeConnectorAppConnectorResources.Set(int64(a.appConnectors.Len()))
 	var connectors set.Slice[types.UID]
 	connectors.AddSlice(a.exitNodes.Slice())
 	connectors.AddSlice(a.subnetRouters.Slice())
+	connectors.AddSlice(a.appConnectors.Slice())
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 	return true, nil
 }
@@ -262,8 +285,14 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 	// Connector fields are already validated at apply time with CEL validation
 	// on custom resource fields. The checks here are a backup in case the
 	// CEL validation breaks without us noticing.
-	if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
-		return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
+	if cn.Spec.SubnetRouter == nil && !cn.Spec.ExitNode && cn.Spec.AppConnector == nil {
+		return errors.New("invalid spec: a Connector must be configured as at least one of subnet router, exit node or app connector")
+	}
+	if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
+		return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
+	}
+	if cn.Spec.AppConnector != nil {
+		return validateAppConnector(cn.Spec.AppConnector)
 	}
 	if cn.Spec.SubnetRouter == nil {
 		return nil
@@ -272,19 +301,27 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
 }
 
 func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
-	if len(sb.AdvertiseRoutes) < 1 {
+	if len(sb.AdvertiseRoutes) == 0 {
 		return errors.New("invalid subnet router spec: no routes defined")
 	}
-	var err error
-	for _, route := range sb.AdvertiseRoutes {
+	return validateRoutes(sb.AdvertiseRoutes)
+}
+
+func validateAppConnector(ac *tsapi.AppConnector) error {
+	return validateRoutes(ac.Routes)
+}
+
+func validateRoutes(routes tsapi.Routes) error {
+	var errs []error
+	for _, route := range routes {
 		pfx, e := netip.ParsePrefix(string(route))
 		if e != nil {
-			err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err))
+			errs = append(errs, fmt.Errorf("route %v is invalid: %v", route, e))
 			continue
 		}
 		if pfx.Masked() != pfx {
-			err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
+			errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked()))
 		}
 	}
-	return err
+	return errors.Join(errs...)
 }

cmd/k8s-operator/connector_test.go

@@ -8,12 +8,14 @@ package main
 import (
 	"context"
 	"testing"
+	"time"
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -296,3 +298,100 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	expectReconciled(t, cr, "", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 }
+
+func TestConnectorWithAppConnector(t *testing.T) {
+	// Setup
+	cn := &tsapi.Connector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  types.UID("1234-UID"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       tsapi.ConnectorKind,
+			APIVersion: "tailscale.io/v1alpha1",
+		},
+		Spec: tsapi.ConnectorSpec{
+			AppConnector: &tsapi.AppConnector{},
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(cn).
+		WithStatusSubresource(cn).
+		Build()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	fr := record.NewFakeRecorder(1)
+	cr := &ConnectorReconciler{
+		Client: fc,
+		clock:  cl,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		recorder: fr,
+	}
+
+	// 1. Connector with app connnector is created and becomes ready
+	expectReconciled(t, cr, "", "test")
+	fullName, shortName := findGenName(t, fc, "", "test", "connector")
+	opts := configOpts{
+		stsName:        shortName,
+		secretName:     fullName,
+		parentType:     "connector",
+		hostname:       "test-connector",
+		app:            kubetypes.AppConnector,
+		isAppConnector: true,
+	}
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
+	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	// Connector's ready condition should be set to true
+
+	cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
+	cn.Status.IsAppConnector = true
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectEqual(t, fc, cn, nil)
+
+	// 2. Connector with invalid app connector routes has status set to invalid
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
+	expectReconciled(t, cr, "", "test")
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionFalse,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorInvalid,
+		Message:            "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
+	}}
+	expectEqual(t, fc, cn, nil)
+
+	// 3. Connector with valid app connnector routes becomes ready
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	})
+	cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
+	cn.Status.Conditions = []metav1.Condition{{
+		Type:               string(tsapi.ConnectorReady),
+		Status:             metav1.ConditionTrue,
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		Reason:             reasonConnectorCreated,
+		Message:            reasonConnectorCreated,
+	}}
+	expectReconciled(t, cr, "", "test")
+}

cmd/k8s-operator/depaware.txt

@@ -80,10 +80,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -658,6 +654,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/ipn/localapi+
@@ -740,7 +737,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/tsd+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -775,6 +771,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/cmd/k8s-operator+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/appc+
@@ -792,7 +789,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/appc+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlclient+
@@ -812,8 +809,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/slicesx                                   from tailscale.com/appc+
         tailscale.com/util/syspolicy                                 from tailscale.com/control/controlclient+
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
@@ -823,7 +823,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -81,6 +81,14 @@ spec:
             - name: PROXY_DEFAULT_CLASS
               value: {{ .Values.proxyConfig.defaultProxyClass }}
             {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.ingressClass.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: IngressClass
 metadata:
@@ -6,3 +7,4 @@ metadata:
 spec:
   controller: tailscale.com/ts-ingress # controller name currently can not be changed
   # parameters: {} # currently no parameters are supported
+{{- end }}

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -16,6 +16,9 @@ rules:
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -54,6 +54,9 @@ operatorConfig:
   # - name: EXTRA_VAR2
   #   value: "value2"
 
+# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
+ingressClass:
+  enabled: true
 
 # proxyConfig contains configuraton that will be applied to any ingress/egress
 # proxies created by the operator.

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -24,6 +24,10 @@ spec:
           jsonPath: .status.isExitNode
           name: IsExitNode
           type: string
+        - description: Whether this Connector instance is an app connector.
+          jsonPath: .status.isAppConnector
+          name: IsAppConnector
+          type: string
         - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
@@ -66,10 +70,40 @@ spec:
                 https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
+                appConnector:
+                  description: |-
+                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                    Connector does not act as an app connector.
+                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                    Admin panel.
+                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                    tested or optimised for.
+                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                    device with a static IP address.
+                    https://tailscale.com/kb/1281/app-connectors
+                  type: object
+                  properties:
+                    routes:
+                      description: |-
+                        Routes are optional preconfigured routes for the domains routed via the app connector.
+                        If not set, routes for the domains will be discovered dynamically.
+                        If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                        also dynamically discover other routes.
+                        https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                      type: array
+                      minItems: 1
+                      items:
+                        type: string
+                        format: cidr
                 exitNode:
                   description: |-
-                    ExitNode defines whether the Connector node should act as a
-                    Tailscale exit node. Defaults to false.
+                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                    This field is mutually exclusive with the appConnector field.
                     https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
@@ -90,9 +124,11 @@ spec:
                   type: string
                 subnetRouter:
                   description: |-
-                    SubnetRouter defines subnet routes that the Connector node should
-                    expose to tailnet. If unset, none are exposed.
+                    SubnetRouter defines subnet routes that the Connector device should
+                    expose to tailnet as a Tailscale subnet router.
                     https://tailscale.com/kb/1019/subnets/
+                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                    This field is mutually exclusive with the appConnector field.
                   type: object
                   required:
                     - advertiseRoutes
@@ -125,8 +161,10 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
               x-kubernetes-validations:
-                - rule: has(self.subnetRouter) || self.exitNode == true
-                  message: A Connector needs to be either an exit node or a subnet router, or both.
+                - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                  message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                - rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
+                  message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
             status:
               description: |-
                 ConnectorStatus describes the status of the Connector. This is set
@@ -200,6 +238,9 @@ spec:
                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                     node.
                   type: string
+                isAppConnector:
+                  description: IsAppConnector is set to true if the Connector acts as an app connector.
+                  type: boolean
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -73,7 +73,12 @@ spec:
                     enable:
                       description: |-
                         Setting enable to true will make the proxy serve Tailscale metrics
-                        at <pod-ip>:9001/debug/metrics.
+                        at <pod-ip>:9002/metrics.
+
+                        In 1.78.x and 1.80.x, this field also serves as the default value for
+                        .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                        fields will independently default to false.
+
                         Defaults to false.
                       type: boolean
                 statefulSet:
@@ -1249,6 +1254,25 @@ spec:
                           description: Configuration for the proxy container running tailscale.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1553,6 +1577,25 @@ spec:
                           description: Configuration for the proxy init container that enables forwarding.
                           type: object
                           properties:
+                            debug:
+                              description: |-
+                                Configuration for enabling extra debug information in the container.
+                                Not recommended for production use.
+                              type: object
+                              properties:
+                                enable:
+                                  description: |-
+                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                    9001 is a container port named "debug". The endpoints and their responses
+                                    may change in backwards incompatible ways in the future, and should not
+                                    be considered stable.
+
+                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                    this setting will default to false, and no requests will be proxied.
+                                  type: boolean
                             env:
                               description: |-
                                 List of environment variables to set in the container.
@@ -1896,6 +1939,182 @@ spec:
                                   Value is the taint value the toleration matches to.
                                   If the operator is Exists, the value should be empty, otherwise just a regular string.
                                 type: string
+                        topologySpreadConstraints:
+                          description: |-
+                            Proxy Pod's topology spread constraints.
+                            By default Tailscale Kubernetes operator does not apply any topology spread constraints.
+                            https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
+                          type: array
+                          items:
+                            description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                            type: object
+                            required:
+                              - maxSkew
+                              - topologyKey
+                              - whenUnsatisfiable
+                            properties:
+                              labelSelector:
+                                description: |-
+                                  LabelSelector is used to find matching pods.
+                                  Pods that match this label selector are counted to determine the number of pods
+                                  in their corresponding topology domain.
+                                type: object
+                                properties:
+                                  matchExpressions:
+                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                    type: array
+                                    items:
+                                      description: |-
+                                        A label selector requirement is a selector that contains values, a key, and an operator that
+                                        relates the key and values.
+                                      type: object
+                                      required:
+                                        - key
+                                        - operator
+                                      properties:
+                                        key:
+                                          description: key is the label key that the selector applies to.
+                                          type: string
+                                        operator:
+                                          description: |-
+                                            operator represents a key's relationship to a set of values.
+                                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                                          type: string
+                                        values:
+                                          description: |-
+                                            values is an array of string values. If the operator is In or NotIn,
+                                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                            the values array must be empty. This array is replaced during a strategic
+                                            merge patch.
+                                          type: array
+                                          items:
+                                            type: string
+                                          x-kubernetes-list-type: atomic
+                                    x-kubernetes-list-type: atomic
+                                  matchLabels:
+                                    description: |-
+                                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                    type: object
+                                    additionalProperties:
+                                      type: string
+                                x-kubernetes-map-type: atomic
+                              matchLabelKeys:
+                                description: |-
+                                  MatchLabelKeys is a set of pod label keys to select the pods over which
+                                  spreading will be calculated. The keys are used to lookup values from the
+                                  incoming pod labels, those key-value labels are ANDed with labelSelector
+                                  to select the group of existing pods over which spreading will be calculated
+                                  for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
+                                  MatchLabelKeys cannot be set when LabelSelector isn't set.
+                                  Keys that don't exist in the incoming pod labels will
+                                  be ignored. A null or empty list means only match against labelSelector.
+
+                                  This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+                                type: array
+                                items:
+                                  type: string
+                                x-kubernetes-list-type: atomic
+                              maxSkew:
+                                description: |-
+                                  MaxSkew describes the degree to which pods may be unevenly distributed.
+                                  When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
+                                  between the number of matching pods in the target topology and the global minimum.
+                                  The global minimum is the minimum number of matching pods in an eligible domain
+                                  or zero if the number of eligible domains is less than MinDomains.
+                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                  labelSelector spread as 2/2/1:
+                                  In this case, the global minimum is 1.
+                                  | zone1 | zone2 | zone3 |
+                                  |  P P  |  P P  |   P   |
+                                  - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
+                                  scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
+                                  violate MaxSkew(1).
+                                  - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
+                                  When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
+                                  to topologies that satisfy it.
+                                  It's a required field. Default value is 1 and 0 is not allowed.
+                                type: integer
+                                format: int32
+                              minDomains:
+                                description: |-
+                                  MinDomains indicates a minimum number of eligible domains.
+                                  When the number of eligible domains with matching topology keys is less than minDomains,
+                                  Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
+                                  And when the number of eligible domains with matching topology keys equals or greater than minDomains,
+                                  this value has no effect on scheduling.
+                                  As a result, when the number of eligible domains is less than minDomains,
+                                  scheduler won't schedule more than maxSkew Pods to those domains.
+                                  If value is nil, the constraint behaves as if MinDomains is equal to 1.
+                                  Valid values are integers greater than 0.
+                                  When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
+
+                                  For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
+                                  labelSelector spread as 2/2/2:
+                                  | zone1 | zone2 | zone3 |
+                                  |  P P  |  P P  |  P P  |
+                                  The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
+                                  In this situation, new pod with the same labelSelector cannot be scheduled,
+                                  because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
+                                  it will violate MaxSkew.
+                                type: integer
+                                format: int32
+                              nodeAffinityPolicy:
+                                description: |-
+                                  NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
+                                  when calculating pod topology spread skew. Options are:
+                                  - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
+                                  - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
+
+                                  If this value is nil, the behavior is equivalent to the Honor policy.
+                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                type: string
+                              nodeTaintsPolicy:
+                                description: |-
+                                  NodeTaintsPolicy indicates how we will treat node taints when calculating
+                                  pod topology spread skew. Options are:
+                                  - Honor: nodes without taints, along with tainted nodes for which the incoming pod
+                                  has a toleration, are included.
+                                  - Ignore: node taints are ignored. All nodes are included.
+
+                                  If this value is nil, the behavior is equivalent to the Ignore policy.
+                                  This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                type: string
+                              topologyKey:
+                                description: |-
+                                  TopologyKey is the key of node labels. Nodes that have a label with this key
+                                  and identical values are considered to be in the same topology.
+                                  We consider each <key, value> as a "bucket", and try to put balanced number
+                                  of pods into each bucket.
+                                  We define a domain as a particular instance of a topology.
+                                  Also, we define an eligible domain as a domain whose nodes meet the requirements of
+                                  nodeAffinityPolicy and nodeTaintsPolicy.
+                                  e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
+                                  And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
+                                  It's a required field.
+                                type: string
+                              whenUnsatisfiable:
+                                description: |-
+                                  WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
+                                  the spread constraint.
+                                  - DoNotSchedule (default) tells the scheduler not to schedule it.
+                                  - ScheduleAnyway tells the scheduler to schedule the pod in any location,
+                                    but giving higher precedence to topologies that would help reduce the
+                                    skew.
+                                  A constraint is considered "Unsatisfiable" for an incoming pod
+                                  if and only if every possible node assignment for that pod would violate
+                                  "MaxSkew" on some topology.
+                                  For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                  labelSelector spread as 3/1/1:
+                                  | zone1 | zone2 | zone3 |
+                                  | P P P |   P   |   P   |
+                                  If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
+                                  to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
+                                  MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
+                                  won't make it *more* imbalanced.
+                                  It's a required field.
+                                type: string
                 tailscale:
                   description: |-
                     TailscaleConfig contains options to configure the tailscale-specific

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -53,6 +53,10 @@ spec:
               jsonPath: .status.isExitNode
               name: IsExitNode
               type: string
+            - description: Whether this Connector instance is an app connector.
+              jsonPath: .status.isAppConnector
+              name: IsAppConnector
+              type: string
             - description: Status of the deployed Connector resources.
               jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
               name: Status
@@ -91,10 +95,40 @@ spec:
                             More info:
                             https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
+                            appConnector:
+                                description: |-
+                                    AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
+                                    configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
+                                    Connector does not act as an app connector.
+                                    Note that you will need to manually configure the permissions and the domains for the app connector via the
+                                    Admin panel.
+                                    Note also that the main tested and supported use case of this config option is to deploy an app connector on
+                                    Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
+                                    cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
+                                    tested or optimised for.
+                                    If you are using the app connector to access SaaS applications because you need a predictable egress IP that
+                                    can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
+                                    via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
+                                    device with a static IP address.
+                                    https://tailscale.com/kb/1281/app-connectors
+                                properties:
+                                    routes:
+                                        description: |-
+                                            Routes are optional preconfigured routes for the domains routed via the app connector.
+                                            If not set, routes for the domains will be discovered dynamically.
+                                            If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may
+                                            also dynamically discover other routes.
+                                            https://tailscale.com/kb/1332/apps-best-practices#preconfiguration
+                                        items:
+                                            format: cidr
+                                            type: string
+                                        minItems: 1
+                                        type: array
+                                type: object
                             exitNode:
                                 description: |-
-                                    ExitNode defines whether the Connector node should act as a
-                                    Tailscale exit node. Defaults to false.
+                                    ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
+                                    This field is mutually exclusive with the appConnector field.
                                     https://tailscale.com/kb/1103/exit-nodes
                                 type: boolean
                             hostname:
@@ -115,9 +149,11 @@ spec:
                                 type: string
                             subnetRouter:
                                 description: |-
-                                    SubnetRouter defines subnet routes that the Connector node should
-                                    expose to tailnet. If unset, none are exposed.
+                                    SubnetRouter defines subnet routes that the Connector device should
+                                    expose to tailnet as a Tailscale subnet router.
                                     https://tailscale.com/kb/1019/subnets/
+                                    If this field is unset, the device does not get configured as a Tailscale subnet router.
+                                    This field is mutually exclusive with the appConnector field.
                                 properties:
                                     advertiseRoutes:
                                         description: |-
@@ -151,8 +187,10 @@ spec:
                                 type: array
                         type: object
                         x-kubernetes-validations:
-                            - message: A Connector needs to be either an exit node or a subnet router, or both.
-                              rule: has(self.subnetRouter) || self.exitNode == true
+                            - message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
+                              rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
+                            - message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
+                              rule: '!((has(self.subnetRouter) || (has(self.exitNode)  && self.exitNode == true)) && has(self.appConnector))'
                     status:
                         description: |-
                             ConnectorStatus describes the status of the Connector. This is set
@@ -225,6 +263,9 @@ spec:
                                     If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                                     node.
                                 type: string
+                            isAppConnector:
+                                description: IsAppConnector is set to true if the Connector acts as an app connector.
+                                type: boolean
                             isExitNode:
                                 description: IsExitNode is set to true if the Connector acts as an exit node.
                                 type: boolean
@@ -499,7 +540,12 @@ spec:
                                     enable:
                                         description: |-
                                             Setting enable to true will make the proxy serve Tailscale metrics
-                                            at <pod-ip>:9001/debug/metrics.
+                                            at <pod-ip>:9002/metrics.
+
+                                            In 1.78.x and 1.80.x, this field also serves as the default value for
+                                            .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
+                                            fields will independently default to false.
+
                                             Defaults to false.
                                         type: boolean
                                 required:
@@ -1675,6 +1721,25 @@ spec:
                                             tailscaleContainer:
                                                 description: Configuration for the proxy container running tailscale.
                                                 properties:
+                                                    debug:
+                                                        description: |-
+                                                            Configuration for enabling extra debug information in the container.
+                                                            Not recommended for production use.
+                                                        properties:
+                                                            enable:
+                                                                description: |-
+                                                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                                                    9001 is a container port named "debug". The endpoints and their responses
+                                                                    may change in backwards incompatible ways in the future, and should not
+                                                                    be considered stable.
+
+                                                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                                                    this setting will default to false, and no requests will be proxied.
+                                                                type: boolean
+                                                        type: object
                                                     env:
                                                         description: |-
                                                             List of environment variables to set in the container.
@@ -1979,6 +2044,25 @@ spec:
                                             tailscaleInitContainer:
                                                 description: Configuration for the proxy init container that enables forwarding.
                                                 properties:
+                                                    debug:
+                                                        description: |-
+                                                            Configuration for enabling extra debug information in the container.
+                                                            Not recommended for production use.
+                                                        properties:
+                                                            enable:
+                                                                description: |-
+                                                                    Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
+                                                                    and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
+                                                                    9001 is a container port named "debug". The endpoints and their responses
+                                                                    may change in backwards incompatible ways in the future, and should not
+                                                                    be considered stable.
+
+                                                                    In 1.78.x and 1.80.x, this setting will default to the value of
+                                                                    .spec.metrics.enable, and requests to the "metrics" port matching the
+                                                                    mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
+                                                                    this setting will default to false, and no requests will be proxied.
+                                                                type: boolean
+                                                        type: object
                                                     env:
                                                         description: |-
                                                             List of environment variables to set in the container.
@@ -2323,6 +2407,182 @@ spec:
                                                             type: string
                                                     type: object
                                                 type: array
+                                            topologySpreadConstraints:
+                                                description: |-
+                                                    Proxy Pod's topology spread constraints.
+                                                    By default Tailscale Kubernetes operator does not apply any topology spread constraints.
+                                                    https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
+                                                items:
+                                                    description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                                                    properties:
+                                                        labelSelector:
+                                                            description: |-
+                                                                LabelSelector is used to find matching pods.
+                                                                Pods that match this label selector are counted to determine the number of pods
+                                                                in their corresponding topology domain.
+                                                            properties:
+                                                                matchExpressions:
+                                                                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                                                    items:
+                                                                        description: |-
+                                                                            A label selector requirement is a selector that contains values, a key, and an operator that
+                                                                            relates the key and values.
+                                                                        properties:
+                                                                            key:
+                                                                                description: key is the label key that the selector applies to.
+                                                                                type: string
+                                                                            operator:
+                                                                                description: |-
+                                                                                    operator represents a key's relationship to a set of values.
+                                                                                    Valid operators are In, NotIn, Exists and DoesNotExist.
+                                                                                type: string
+                                                                            values:
+                                                                                description: |-
+                                                                                    values is an array of string values. If the operator is In or NotIn,
+                                                                                    the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                                                                    the values array must be empty. This array is replaced during a strategic
+                                                                                    merge patch.
+                                                                                items:
+                                                                                    type: string
+                                                                                type: array
+                                                                                x-kubernetes-list-type: atomic
+                                                                        required:
+                                                                            - key
+                                                                            - operator
+                                                                        type: object
+                                                                    type: array
+                                                                    x-kubernetes-list-type: atomic
+                                                                matchLabels:
+                                                                    additionalProperties:
+                                                                        type: string
+                                                                    description: |-
+                                                                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                                                                        map is equivalent to an element of matchExpressions, whose key field is "key", the
+                                                                        operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                                                    type: object
+                                                            type: object
+                                                            x-kubernetes-map-type: atomic
+                                                        matchLabelKeys:
+                                                            description: |-
+                                                                MatchLabelKeys is a set of pod label keys to select the pods over which
+                                                                spreading will be calculated. The keys are used to lookup values from the
+                                                                incoming pod labels, those key-value labels are ANDed with labelSelector
+                                                                to select the group of existing pods over which spreading will be calculated
+                                                                for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.
+                                                                MatchLabelKeys cannot be set when LabelSelector isn't set.
+                                                                Keys that don't exist in the incoming pod labels will
+                                                                be ignored. A null or empty list means only match against labelSelector.
+
+                                                                This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+                                                            items:
+                                                                type: string
+                                                            type: array
+                                                            x-kubernetes-list-type: atomic
+                                                        maxSkew:
+                                                            description: |-
+                                                                MaxSkew describes the degree to which pods may be unevenly distributed.
+                                                                When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference
+                                                                between the number of matching pods in the target topology and the global minimum.
+                                                                The global minimum is the minimum number of matching pods in an eligible domain
+                                                                or zero if the number of eligible domains is less than MinDomains.
+                                                                For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                                                labelSelector spread as 2/2/1:
+                                                                In this case, the global minimum is 1.
+                                                                | zone1 | zone2 | zone3 |
+                                                                |  P P  |  P P  |   P   |
+                                                                - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;
+                                                                scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)
+                                                                violate MaxSkew(1).
+                                                                - if MaxSkew is 2, incoming pod can be scheduled onto any zone.
+                                                                When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence
+                                                                to topologies that satisfy it.
+                                                                It's a required field. Default value is 1 and 0 is not allowed.
+                                                            format: int32
+                                                            type: integer
+                                                        minDomains:
+                                                            description: |-
+                                                                MinDomains indicates a minimum number of eligible domains.
+                                                                When the number of eligible domains with matching topology keys is less than minDomains,
+                                                                Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed.
+                                                                And when the number of eligible domains with matching topology keys equals or greater than minDomains,
+                                                                this value has no effect on scheduling.
+                                                                As a result, when the number of eligible domains is less than minDomains,
+                                                                scheduler won't schedule more than maxSkew Pods to those domains.
+                                                                If value is nil, the constraint behaves as if MinDomains is equal to 1.
+                                                                Valid values are integers greater than 0.
+                                                                When value is not nil, WhenUnsatisfiable must be DoNotSchedule.
+
+                                                                For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same
+                                                                labelSelector spread as 2/2/2:
+                                                                | zone1 | zone2 | zone3 |
+                                                                |  P P  |  P P  |  P P  |
+                                                                The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0.
+                                                                In this situation, new pod with the same labelSelector cannot be scheduled,
+                                                                because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,
+                                                                it will violate MaxSkew.
+                                                            format: int32
+                                                            type: integer
+                                                        nodeAffinityPolicy:
+                                                            description: |-
+                                                                NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector
+                                                                when calculating pod topology spread skew. Options are:
+                                                                - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.
+                                                                - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.
+
+                                                                If this value is nil, the behavior is equivalent to the Honor policy.
+                                                                This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                                            type: string
+                                                        nodeTaintsPolicy:
+                                                            description: |-
+                                                                NodeTaintsPolicy indicates how we will treat node taints when calculating
+                                                                pod topology spread skew. Options are:
+                                                                - Honor: nodes without taints, along with tainted nodes for which the incoming pod
+                                                                has a toleration, are included.
+                                                                - Ignore: node taints are ignored. All nodes are included.
+
+                                                                If this value is nil, the behavior is equivalent to the Ignore policy.
+                                                                This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+                                                            type: string
+                                                        topologyKey:
+                                                            description: |-
+                                                                TopologyKey is the key of node labels. Nodes that have a label with this key
+                                                                and identical values are considered to be in the same topology.
+                                                                We consider each <key, value> as a "bucket", and try to put balanced number
+                                                                of pods into each bucket.
+                                                                We define a domain as a particular instance of a topology.
+                                                                Also, we define an eligible domain as a domain whose nodes meet the requirements of
+                                                                nodeAffinityPolicy and nodeTaintsPolicy.
+                                                                e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology.
+                                                                And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology.
+                                                                It's a required field.
+                                                            type: string
+                                                        whenUnsatisfiable:
+                                                            description: |-
+                                                                WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy
+                                                                the spread constraint.
+                                                                - DoNotSchedule (default) tells the scheduler not to schedule it.
+                                                                - ScheduleAnyway tells the scheduler to schedule the pod in any location,
+                                                                  but giving higher precedence to topologies that would help reduce the
+                                                                  skew.
+                                                                A constraint is considered "Unsatisfiable" for an incoming pod
+                                                                if and only if every possible node assignment for that pod would violate
+                                                                "MaxSkew" on some topology.
+                                                                For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same
+                                                                labelSelector spread as 3/1/1:
+                                                                | zone1 | zone2 | zone3 |
+                                                                | P P P |   P   |   P   |
+                                                                If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled
+                                                                to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
+                                                                MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler
+                                                                won't make it *more* imbalanced.
+                                                                It's a required field.
+                                                            type: string
+                                                    required:
+                                                        - maxSkew
+                                                        - topologyKey
+                                                        - whenUnsatisfiable
+                                                    type: object
+                                                type: array
                                         type: object
                                 type: object
                             tailscale:
@@ -4486,6 +4746,14 @@ rules:
         - patch
         - update
         - watch
+    - apiGroups:
+        - ""
+      resources:
+        - events
+      verbs:
+        - create
+        - patch
+        - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -4558,6 +4826,14 @@ spec:
                       value: "false"
                     - name: PROXY_FIREWALL_MODE
                       value: auto
+                    - name: POD_NAME
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.name
+                    - name: POD_UID
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: metadata.uid
                   image: tailscale/k8s-operator:unstable
                   imagePullPolicy: Always
                   name: operator

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -30,6 +30,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
           securityContext:
             capabilities:
               add:

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -24,3 +24,11 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid

cmd/k8s-operator/operator.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -150,6 +151,13 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 		Hostname: hostname,
 		Logf:     zlog.Named("tailscaled").Debugf,
 	}
+	if p := os.Getenv("TS_PORT"); p != "" {
+		port, err := strconv.ParseUint(p, 10, 16)
+		if err != nil {
+			startlog.Fatalf("TS_PORT %q cannot be parsed as uint16: %v", p, err)
+		}
+		s.Port = uint16(port)
+	}
 	if kubeSecret != "" {
 		st, err := kubestore.New(logger.Discard, kubeSecret)
 		if err != nil {

cmd/k8s-operator/operator_test.go

@@ -432,6 +432,148 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 }
 
+func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		clock:    clock,
+		recorder: record.NewFakeRecorder(100),
+	}
+	tailnetTargetIP := "invalid-ip"
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	t0 := conditionTime(clock)
+
+	want := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyReady),
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: t0,
+				Reason:             reasonProxyInvalid,
+				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "invalid-ip" could not be parsed as a valid IP Address, error: ParseAddr("invalid-ip"): unable to parse IP`,
+			}},
+		},
+	}
+
+	expectEqual(t, fc, want, nil)
+}
+
+func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:   zl.Sugar(),
+		clock:    clock,
+		recorder: record.NewFakeRecorder(100),
+	}
+	tailnetTargetIP := "999.999.999.999"
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	t0 := conditionTime(clock)
+
+	want := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyReady),
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: t0,
+				Reason:             reasonProxyInvalid,
+				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "999.999.999.999" could not be parsed as a valid IP Address, error: ParseAddr("999.999.999.999"): IPv4 field has value >255`,
+			}},
+		},
+	}
+
+	expectEqual(t, fc, want, nil)
+}
+
 func TestAnnotations(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
@@ -1246,7 +1388,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
+		confFileHash:    "a67b5ad3ff605531c822327e8f1a23dd0846e1075b722c13402f7d5d0ba32ba2",
 		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
@@ -1257,7 +1399,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684"
+	o.confFileHash = "888a993ebee20ad6be99623b45015339de117946850cf1252bede0b570e04293"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }

cmd/k8s-operator/proxyclass.go

@@ -160,6 +160,10 @@ func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations fiel
 						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
 					}
 				}
+
+				if tc.Debug != nil {
+					violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "debug"), tc.Debug, "debug settings cannot be configured on the init container"))
+				}
 			}
 		}
 	}

cmd/k8s-operator/proxyclass_test.go

@@ -135,3 +135,56 @@ func TestProxyClass(t *testing.T) {
 	expectReconciled(t, pcr, "", "test")
 	expectEvents(t, fr, expectedEvents)
 }
+
+func TestValidateProxyClass(t *testing.T) {
+	for name, tc := range map[string]struct {
+		pc    *tsapi.ProxyClass
+		valid bool
+	}{
+		"empty": {
+			valid: true,
+			pc:    &tsapi.ProxyClass{},
+		},
+		"debug_enabled_for_main_container": {
+			valid: true,
+			pc: &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{
+									Enable: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"debug_enabled_for_init_container": {
+			valid: false,
+			pc: &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleInitContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{
+									Enable: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			pcr := &ProxyClassReconciler{}
+			err := pcr.validate(tc.pc)
+			valid := err == nil
+			if valid != tc.valid {
+				t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
+			}
+		})
+	}
+}

cmd/k8s-operator/proxygroup.go

@@ -47,7 +47,7 @@ const (
 	reasonProxyGroupInvalid        = "ProxyGroupInvalid"
 )
 
-var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupCount)
+var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
 
 // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
 type ProxyGroupReconciler struct {
@@ -353,7 +353,7 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc
 
 func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) {
 	logger := r.logger(pg.Name)
-	var allConfigs []tailscaledConfigs
+	var configSHA256Sum string
 	for i := range pgReplicas(pg) {
 		cfgSecret := &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
@@ -389,7 +389,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		if err != nil {
 			return "", fmt.Errorf("error creating tailscaled config: %w", err)
 		}
-		allConfigs = append(allConfigs, configs)
 
 		for cap, cfg := range configs {
 			cfgJSON, err := json.Marshal(cfg)
@@ -399,6 +398,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
 		}
 
+		// The config sha256 sum is a value for a hash annotation used to trigger
+		// pod restarts when tailscaled config changes. Any config changes apply
+		// to all replicas, so it is sufficient to only hash the config for the
+		// first replica.
+		//
+		// In future, we're aiming to eliminate restarts altogether and have
+		// pods dynamically reload their config when it changes.
+		if i == 0 {
+			sum := sha256.New()
+			for _, cfg := range configs {
+				// Zero out the auth key so it doesn't affect the sha256 hash when we
+				// remove it from the config after the pods have all authed. Otherwise
+				// all the pods will need to restart immediately after authing.
+				cfg.AuthKey = nil
+				b, err := json.Marshal(cfg)
+				if err != nil {
+					return "", err
+				}
+				if _, err := sum.Write(b); err != nil {
+					return "", err
+				}
+			}
+
+			configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil))
+		}
+
 		if existingCfgSecret != nil {
 			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
 			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
@@ -412,16 +437,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		}
 	}
 
-	sum := sha256.New()
-	b, err := json.Marshal(allConfigs)
-	if err != nil {
-		return "", err
-	}
-	if _, err := sum.Write(b); err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%x", sum.Sum(nil)), nil
+	return configSHA256Sum, nil
 }
 
 func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {

cmd/k8s-operator/proxygroup_specs.go

@@ -15,6 +15,7 @@ import (
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
 
@@ -92,6 +93,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 	c.Image = image
 	c.VolumeMounts = func() []corev1.VolumeMount {
 		var mounts []corev1.VolumeMount
+
+		// TODO(tomhjp): Read config directly from the secret instead. The
+		// mounts change on scaling up/down which causes unnecessary restarts
+		// for pods that haven't meaningfully changed.
 		for i := range pgReplicas(pg) {
 			mounts = append(mounts, corev1.VolumeMount{
 				Name:      fmt.Sprintf("tailscaledconfig-%d", i),
@@ -121,15 +126,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 					},
 				},
 			},
-			{
-				Name: "POD_NAME",
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						// Secret is named after the pod.
-						FieldPath: "metadata.name",
-					},
-				},
-			},
 			{
 				Name:  "TS_KUBE_SECRET",
 				Value: "$(POD_NAME)",
@@ -143,8 +139,8 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				Value: "/etc/tsconfig/$(POD_NAME)",
 			},
 			{
-				Name:  "TS_USERSPACE",
-				Value: "false",
+				Name:  "TS_INTERNAL_APP",
+				Value: kubetypes.AppProxyGroupEgress,
 			},
 		}
 
@@ -162,7 +158,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 			})
 		}
 
-		return envs
+		return append(c.Env, envs...)
 	}()
 
 	return ss, nil
@@ -206,6 +202,15 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 					return secrets
 				}(),
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"create",
+					"patch",
+					"get",
+				},
+			},
 		},
 	}
 }

cmd/k8s-operator/proxygroup_test.go

@@ -35,6 +35,8 @@ var defaultProxyClassAnnotations = map[string]string{
 }
 
 func TestProxyGroup(t *testing.T) {
+	const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196"
+
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "default-pc",
@@ -80,6 +82,7 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, false, "")
 	})
 
 	t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) {
@@ -100,10 +103,11 @@ func TestProxyGroup(t *testing.T) {
 
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		if expected := 1; reconciler.proxyGroups.Len() != expected {
 			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
 		}
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		keyReq := tailscale.KeyCapabilities{
 			Devices: tailscale.KeyDeviceCapabilities{
 				Create: tailscale.KeyDeviceCreateCapabilities{
@@ -135,7 +139,7 @@ func TestProxyGroup(t *testing.T) {
 		}
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 	})
 
 	t.Run("scale_up_to_3", func(t *testing.T) {
@@ -146,6 +150,7 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 
 		addNodeIDToStateSecrets(t, fc, pg)
 		expectReconciled(t, reconciler, "", pg.Name)
@@ -155,7 +160,7 @@ func TestProxyGroup(t *testing.T) {
 			TailnetIPs: []string{"1.2.3.4", "::1"},
 		})
 		expectEqual(t, fc, pg, nil)
-		expectProxyGroupResources(t, fc, pg, true)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 	})
 
 	t.Run("scale_down_to_1", func(t *testing.T) {
@@ -163,11 +168,26 @@ func TestProxyGroup(t *testing.T) {
 		mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
 			p.Spec = pg.Spec
 		})
+
 		expectReconciled(t, reconciler, "", pg.Name)
+
 		pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
 		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
+	})
 
-		expectProxyGroupResources(t, fc, pg, true)
+	t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) {
+		pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{
+			AcceptRoutes: true,
+		}
+		mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
+			p.Spec = pc.Spec
+		})
+
+		expectReconciled(t, reconciler, "", pg.Name)
+
+		expectEqual(t, fc, pg, nil)
+		expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74")
 	})
 
 	t.Run("delete_and_cleanup", func(t *testing.T) {
@@ -191,13 +211,13 @@ func TestProxyGroup(t *testing.T) {
 	})
 }
 
-func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool) {
+func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
 	t.Helper()
 
 	role := pgRole(pg, tsNamespace)
 	roleBinding := pgRoleBinding(pg, tsNamespace)
 	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", cfgHash)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -207,9 +227,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 		expectEqual(t, fc, role, nil)
 		expectEqual(t, fc, roleBinding, nil)
 		expectEqual(t, fc, serviceAccount, nil)
-		expectEqual(t, fc, statefulSet, func(ss *appsv1.StatefulSet) {
-			ss.Spec.Template.Annotations[podAnnotationLastSetConfigFileHash] = ""
-		})
+		expectEqual(t, fc, statefulSet, nil)
 	} else {
 		expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
 		expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
@@ -218,11 +236,13 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 	}
 
 	var expectedSecrets []string
-	for i := range pgReplicas(pg) {
-		expectedSecrets = append(expectedSecrets,
-			fmt.Sprintf("%s-%d", pg.Name, i),
-			fmt.Sprintf("%s-%d-config", pg.Name, i),
-		)
+	if shouldExist {
+		for i := range pgReplicas(pg) {
+			expectedSecrets = append(expectedSecrets,
+				fmt.Sprintf("%s-%d", pg.Name, i),
+				fmt.Sprintf("%s-%d-config", pg.Name, i),
+			)
+		}
 	}
 	expectSecrets(t, fc, expectedSecrets)
 }

cmd/k8s-operator/sts.go

@@ -132,10 +132,13 @@ type tailscaleSTSConfig struct {
 }
 
 type connector struct {
-	// routes is a list of subnet routes that this Connector should expose.
+	// routes is a list of routes that this Connector should advertise either as a subnet router or as an app
+	// connector.
 	routes string
 	// isExitNode defines whether this Connector should act as an exit node.
 	isExitNode bool
+	// isAppConnector defines whether this Connector should act as an app connector.
+	isAppConnector bool
 }
 type tsnetServer interface {
 	CertDomains() []string
@@ -473,7 +476,7 @@ var proxyYaml []byte
 //go:embed deploy/manifests/userspace-proxy.yaml
 var userspaceProxyYaml []byte
 
-func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, _ map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -518,11 +521,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_KUBE_SECRET",
 			Value: proxySecret,
 		},
-		corev1.EnvVar{
-			// Old tailscaled config key is still used for backwards compatibility.
-			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
-			Value: "/etc/tsconfig/tailscaled",
-		},
 		corev1.EnvVar{
 			// New style is in the form of cap-<capability-version>.hujson.
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
@@ -668,24 +666,42 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [
 	return custom
 }
 
+func debugSetting(pc *tsapi.ProxyClass) bool {
+	if pc == nil ||
+		pc.Spec.StatefulSet == nil ||
+		pc.Spec.StatefulSet.Pod == nil ||
+		pc.Spec.StatefulSet.Pod.TailscaleContainer == nil ||
+		pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug == nil {
+		// This default will change to false in 1.82.0.
+		return pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
+	}
+
+	return pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug.Enable
+}
+
 func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet {
 	if pc == nil || ss == nil {
 		return ss
 	}
-	if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
-		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
-			enableMetrics(ss, pc)
-		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+
+	metricsEnabled := pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
+	debugEnabled := debugSetting(pc)
+	if metricsEnabled || debugEnabled {
+		isEgress := stsCfg != nil && (stsCfg.TailnetTargetFQDN != "" || stsCfg.TailnetTargetIP != "")
+		isForwardingL7Ingress := stsCfg != nil && stsCfg.ForwardClusterTrafficViaL7IngressProxy
+		if isEgress {
 			// TODO (irbekrm): fix this
 			// For Ingress proxies that have been configured with
 			// tailscale.com/experimental-forward-cluster-traffic-via-ingress
 			// annotation, all cluster traffic is forwarded to the
 			// Ingress backend(s).
-			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
-		} else {
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for egress proxies.")
+		} else if isForwardingL7Ingress {
 			// TODO (irbekrm): fix this
 			// For egress proxies, currently all cluster traffic is forwarded to the tailnet target.
 			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		} else {
+			enableEndpoints(ss, metricsEnabled, debugEnabled)
 		}
 	}
 
@@ -718,6 +734,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
 	ss.Spec.Template.Spec.Affinity = wantsPod.Affinity
 	ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
+	ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints
 
 	// Update containers.
 	updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container {
@@ -762,16 +779,58 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	return ss
 }
 
-func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) {
+func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
 	for i, c := range ss.Spec.Template.Spec.Containers {
 		if c.Name == "tailscale" {
-			// Serve metrics on on <pod-ip>:9001/debug/metrics. If
-			// we didn't specify Pod IP here, the proxy would, in
-			// some cases, also listen to its Tailscale IP- we don't
-			// want folks to start relying on this side-effect as a
-			// feature.
-			ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
-			ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001})
+			if debug {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
+					// Serve tailscaled's debug metrics on on
+					// <pod-ip>:9001/debug/metrics. If we didn't specify Pod IP
+					// here, the proxy would, in some cases, also listen to its
+					// Tailscale IP- we don't want folks to start relying on this
+					// side-effect as a feature.
+					corev1.EnvVar{
+						Name:  "TS_DEBUG_ADDR_PORT",
+						Value: "$(POD_IP):9001",
+					},
+					// TODO(tomhjp): Can remove this env var once 1.76.x is no
+					// longer supported.
+					corev1.EnvVar{
+						Name:  "TS_TAILSCALED_EXTRA_ARGS",
+						Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
+					},
+				)
+
+				ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
+					corev1.ContainerPort{
+						Name:          "debug",
+						Protocol:      "TCP",
+						ContainerPort: 9001,
+					},
+				)
+			}
+
+			if metrics {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
+					// Serve client metrics on <pod-ip>:9002/metrics.
+					corev1.EnvVar{
+						Name:  "TS_LOCAL_ADDR_PORT",
+						Value: "$(POD_IP):9002",
+					},
+					corev1.EnvVar{
+						Name:  "TS_METRICS_ENABLED",
+						Value: "true",
+					},
+				)
+				ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
+					corev1.ContainerPort{
+						Name:          "metrics",
+						Protocol:      "TCP",
+						ContainerPort: 9002,
+					},
+				)
+			}
+
 			break
 		}
 	}
@@ -785,15 +844,9 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 	return origConf.AuthKey, nil
 }
 
-// tailscaledConfig takes a proxy config, a newly generated auth key if
-// generated and a Secret with the previous proxy state and auth key and
-// returns tailscaled configuration and a hash of that configuration.
-//
-// As of 2024-05-09 it also returns legacy tailscaled config without the
-// later added NoStatefulFilter field to support proxies older than cap95.
-// TODO (irbekrm): remove the legacy config once we no longer need to support
-// versions older than cap94,
-// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
+// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that
+// configuration.
 func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
@@ -802,11 +855,13 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		Locked:              "false",
 		Hostname:            &stsC.Hostname,
 		NoStatefulFiltering: "false",
+		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
 	}
 
 	// For egress proxies only, we need to ensure that stateful filtering is
 	// not in place so that traffic from cluster can be forwarded via
 	// Tailscale IPs.
+	// TODO (irbekrm): set it to true always as this is now the default in core.
 	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	}
@@ -816,6 +871,9 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
 		conf.AdvertiseRoutes = routes
+		if stsC.Connector.isAppConnector {
+			conf.AppConnector.Advertise = true
+		}
 	}
 	if shouldAcceptRoutes(stsC.ProxyClass) {
 		conf.AcceptRoutes = "true"
@@ -830,11 +888,13 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		}
 		conf.AuthKey = key
 	}
+
 	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[107] = *conf
+
+	// AppConnector config option is only understood by clients of capver 107 and newer.
+	conf.AppConnector = nil
 	capVerConfigs[95] = *conf
-	// legacy config should not contain NoStatefulFiltering field.
-	conf.NoStatefulFiltering.Clear()
-	capVerConfigs[94] = *conf
 	return capVerConfigs, nil
 }
 

cmd/k8s-operator/sts_test.go

@@ -18,6 +18,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/types/ptr"
@@ -73,6 +74,16 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 					NodeSelector:     map[string]string{"beta.kubernetes.io/os": "linux"},
 					Affinity:         &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}},
 					Tolerations:      []corev1.Toleration{{Key: "", Operator: "Exists"}},
+					TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
+						{
+							WhenUnsatisfiable: "DoNotSchedule",
+							TopologyKey:       "kubernetes.io/hostname",
+							MaxSkew:           3,
+							LabelSelector: &metav1.LabelSelector{
+								MatchLabels: map[string]string{"foo": "bar"},
+							},
+						},
+					},
 					TailscaleContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
 							Privileged: ptr.To(true),
@@ -114,10 +125,26 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 			},
 		},
 	}
-	proxyClassMetrics := &tsapi.ProxyClass{
-		Spec: tsapi.ProxyClassSpec{
-			Metrics: &tsapi.Metrics{Enable: true},
-		},
+
+	proxyClassWithMetricsDebug := func(metrics bool, debug *bool) *tsapi.ProxyClass {
+		return &tsapi.ProxyClass{
+			Spec: tsapi.ProxyClassSpec{
+				Metrics: &tsapi.Metrics{Enable: metrics},
+				StatefulSet: func() *tsapi.StatefulSet {
+					if debug == nil {
+						return nil
+					}
+
+					return &tsapi.StatefulSet{
+						Pod: &tsapi.Pod{
+							TailscaleContainer: &tsapi.Container{
+								Debug: &tsapi.Debug{Enable: *debug},
+							},
+						},
+					}
+				}(),
+			},
+		}
 	}
 
 	var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
@@ -159,6 +186,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
+	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
@@ -172,7 +200,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 
 	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 2. Test that a ProxyClass with custom labels and annotations for
@@ -185,7 +213,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 3. Test that a ProxyClass with all fields set gets correctly applied
@@ -201,6 +229,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
 	wantSS.Spec.Template.Spec.Affinity = proxyClassAllOpts.Spec.StatefulSet.Pod.Affinity
 	wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
+	wantSS.Spec.Template.Spec.TopologySpreadConstraints = proxyClassAllOpts.Spec.StatefulSet.Pod.TopologySpreadConstraints
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
@@ -208,7 +237,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
 	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
@@ -220,16 +249,48 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
-	// 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet.
+	// 5. Metrics enabled defaults to enabling both metrics and debug.
 	wantSS = nonUserspaceProxySS.DeepCopy()
-	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
-	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}}
-	gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
+		corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
+		corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
+		corev1.EnvVar{Name: "TS_METRICS_ENABLED", Value: "true"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{
+		{Name: "debug", Protocol: "TCP", ContainerPort: 9001},
+		{Name: "metrics", Protocol: "TCP", ContainerPort: 9002},
+	}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, nil), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
+
+	// 6. Enable _just_ metrics by explicitly disabling debug.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
+		corev1.EnvVar{Name: "TS_METRICS_ENABLED", Value: "true"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9002}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, ptr.To(false)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
+
+	// 7. Enable _just_ debug without metrics.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
+		corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
+		corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
+	)
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "debug", Protocol: "TCP", ContainerPort: 9001}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(false, ptr.To(true)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
 	}
 }
 

cmd/k8s-operator/svc.go

@@ -358,9 +358,14 @@ func validateService(svc *corev1.Service) []string {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
-
-	// TODO(irbekrm): validate that tailscale.com/tailnet-ip annotation is a
-	// valid IP address (tailscale/tailscale#13671).
+	if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" {
+		ip, err := netip.ParseAddr(ipStr)
+		if err != nil {
+			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err))
+		} else if !ip.IsValid() {
+			violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr))
+		}
+	}
 
 	svcName := nameForService(svc)
 	if err := dnsname.ValidLabel(svcName); err != nil {

cmd/k8s-operator/testutils_test.go

@@ -48,6 +48,7 @@ type configOpts struct {
 	clusterTargetDNS                               string
 	subnetRoutes                                   string
 	isExitNode                                     bool
+	isAppConnector                                 bool
 	confFileHash                                   string
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
@@ -69,8 +70,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "false"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
@@ -228,8 +230,9 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
+			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 			{Name: "TS_INTERNAL_APP", Value: opts.app},
@@ -356,6 +359,7 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 		Locked:       "false",
 		AuthKey:      ptr.To("secret-authkey"),
 		AcceptRoutes: "false",
+		AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
 	}
 	if opts.proxyClass != "" {
 		t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
@@ -370,6 +374,9 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 	if opts.shouldRemoveAuthKey {
 		conf.AuthKey = nil
 	}
+	if opts.isAppConnector {
+		conf.AppConnector = &ipn.AppConnectorPrefs{Advertise: true}
+	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -384,22 +391,23 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 			routes = append(routes, prefix)
 		}
 	}
-	conf.AdvertiseRoutes = routes
-	b, err := json.Marshal(conf)
-	if err != nil {
-		t.Fatalf("error marshalling tailscaled config")
-	}
 	if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
 		conf.NoStatefulFiltering = "true"
 	} else {
 		conf.NoStatefulFiltering = "false"
 	}
+	conf.AdvertiseRoutes = routes
+	bnn, err := json.Marshal(conf)
+	if err != nil {
+		t.Fatalf("error marshalling tailscaled config")
+	}
+	conf.AppConnector = nil
 	bn, err := json.Marshal(conf)
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
-	mak.Set(&s.StringData, "tailscaled", string(b))
 	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
+	mak.Set(&s.StringData, "cap-107.hujson", string(bnn))
 	labels := map[string]string{
 		"tailscale.com/managed":              "true",
 		"tailscale.com/parent-resource":      "test",
@@ -650,18 +658,6 @@ func removeTargetPortsFromSvc(svc *corev1.Service) {
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()
-		if len(secret.StringData["tailscaled"]) != 0 {
-			conf := &ipn.ConfigVAlpha{}
-			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
-				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
-			}
-			conf.AuthKey = nil
-			b, err := json.Marshal(conf)
-			if err != nil {
-				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
-			}
-			mak.Set(&secret.StringData, "tailscaled", string(b))
-		}
 		if len(secret.StringData["cap-95.hujson"]) != 0 {
 			conf := &ipn.ConfigVAlpha{}
 			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
@@ -674,5 +670,17 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 			}
 			mak.Set(&secret.StringData, "cap-95.hujson", string(b))
 		}
+		if len(secret.StringData["cap-107.hujson"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["cap-107.hujson"]), conf); err != nil {
+				t.Fatalf("error umarshalling 'cap-107.hujson' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling 'cap-107.huson' contents: %v", err)
+			}
+			mak.Set(&secret.StringData, "cap-107.hujson", string(b))
+		}
 	}
 }

cmd/k8s-operator/tsrecorder_specs.go

@@ -130,6 +130,15 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
 					fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
 				},
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"get",
+					"create",
+					"patch",
+				},
+			},
 		},
 	}
 }
@@ -203,6 +212,14 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar {
 				},
 			},
 		},
+		{
+			Name: "POD_UID",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.uid",
+				},
+			},
+		},
 		{
 			Name:  "TS_STATE",
 			Value: "kube:$(POD_NAME)",

cmd/stund/depaware.txt

@@ -8,7 +8,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
-        github.com/google/uuid                                       from tailscale.com/util/fastuuid
      💣 github.com/prometheus/client_golang/prometheus               from tailscale.com/tsweb/promvarz
         github.com/prometheus/client_golang/prometheus/internal      from github.com/prometheus/client_golang/prometheus
         github.com/prometheus/client_model/go                        from github.com/prometheus/client_golang/prometheus+
@@ -67,15 +66,16 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/types/logger                                   from tailscale.com/tsweb
         tailscale.com/types/opt                                      from tailscale.com/envknob+
         tailscale.com/types/ptr                                      from tailscale.com/tailcfg+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/tailcfg+
         tailscale.com/types/tkatype                                  from tailscale.com/tailcfg+
         tailscale.com/types/views                                    from tailscale.com/net/tsaddr+
         tailscale.com/util/ctxkey                                    from tailscale.com/tsweb+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/tailcfg
-        tailscale.com/util/fastuuid                                  from tailscale.com/tsweb
-        tailscale.com/util/lineread                                  from tailscale.com/version/distro
+        tailscale.com/util/lineiter                                  from tailscale.com/version/distro
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/slicesx                                   from tailscale.com/tailcfg
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
         tailscale.com/version                                        from tailscale.com/envknob+
@@ -132,7 +132,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         crypto/tls                                                   from net/http+
         crypto/x509                                                  from crypto/tls
         crypto/x509/pkix                                             from crypto/x509
-        database/sql/driver                                          from github.com/google/uuid
         embed                                                        from crypto/internal/nistec+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
@@ -163,7 +162,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from math/big+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid+
+        math/rand/v2                                                 from internal/concurrent+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/tailscale/cli/cli.go

@@ -93,8 +93,13 @@ func Run(args []string) (err error) {
 
 	args = CleanUpArgs(args)
 
-	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
-		args = []string{"version"}
+	if len(args) == 1 {
+		switch args[0] {
+		case "-V", "--version":
+			args = []string{"version"}
+		case "help":
+			args = []string{"--help"}
+		}
 	}
 
 	var warnOnce sync.Once
@@ -185,10 +190,12 @@ change in the future.
 			logoutCmd,
 			switchCmd,
 			configureCmd,
+			syspolicyCmd,
 			netcheckCmd,
 			ipCmd,
 			dnsCmd,
 			statusCmd,
+			metricsCmd,
 			pingCmd,
 			ncCmd,
 			sshCmd,

cmd/tailscale/cli/cli_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"net/netip"
 	"reflect"
 	"strings"
@@ -1480,3 +1481,33 @@ func TestParseNLArgs(t *testing.T) {
 		})
 	}
 }
+
+func TestHelpAlias(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	tstest.Replace[io.Writer](t, &Stdout, &stdout)
+	tstest.Replace[io.Writer](t, &Stderr, &stderr)
+
+	gotExit0 := false
+	defer func() {
+		if !gotExit0 {
+			t.Error("expected os.Exit(0) to be called")
+			return
+		}
+		if !strings.Contains(stderr.String(), "SUBCOMMANDS") {
+			t.Errorf("expected help output to contain SUBCOMMANDS; got stderr=%q; stdout=%q", stderr.String(), stdout.String())
+		}
+	}()
+	defer func() {
+		if e := recover(); e != nil {
+			if strings.Contains(fmt.Sprint(e), "unexpected call to os.Exit(0)") {
+				gotExit0 = true
+			} else {
+				t.Errorf("unexpected panic: %v", e)
+			}
+		}
+	}()
+	err := Run([]string{"help"})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}

cmd/tailscale/cli/debug.go

@@ -36,6 +36,7 @@ import (
 	"tailscale.com/hostinfo"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/ipn"
+	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/paths"
@@ -213,6 +214,7 @@ var debugCmd = &ffcli.Command{
 				fs := newFlagSet("watch-ipn")
 				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
 				fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status")
+				fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags")
 				fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap")
 				fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever")
 				return fs
@@ -500,6 +502,7 @@ var watchIPNArgs struct {
 	netmap         bool
 	initial        bool
 	showPrivateKey bool
+	rateLimit      bool
 	count          int
 }
 
@@ -511,6 +514,9 @@ func runWatchIPN(ctx context.Context, args []string) error {
 	if !watchIPNArgs.showPrivateKey {
 		mask |= ipn.NotifyNoPrivateKeys
 	}
+	if watchIPNArgs.rateLimit {
+		mask |= ipn.NotifyRateLimit
+	}
 	watcher, err := localClient.WatchIPNBus(ctx, mask)
 	if err != nil {
 		return err
@@ -845,6 +851,11 @@ func runTS2021(ctx context.Context, args []string) error {
 		logf = log.Printf
 	}
 
+	netMon, err := netmon.New(logger.WithPrefix(logf, "netmon: "))
+	if err != nil {
+		return fmt.Errorf("creating netmon: %w", err)
+	}
+
 	noiseDialer := &controlhttp.Dialer{
 		Hostname:        ts2021Args.host,
 		HTTPPort:        "80",
@@ -854,6 +865,7 @@ func runTS2021(ctx context.Context, args []string) error {
 		ProtocolVersion: uint16(ts2021Args.version),
 		Dialer:          dialFunc,
 		Logf:            logf,
+		NetMon:          netMon,
 	}
 	const tries = 2
 	for i := range tries {

cmd/tailscale/cli/metrics.go

@@ -0,0 +1,88 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/atomicfile"
+)
+
+var metricsCmd = &ffcli.Command{
+	Name:      "metrics",
+	ShortHelp: "Show Tailscale metrics",
+	LongHelp: strings.TrimSpace(`
+
+The 'tailscale metrics' command shows Tailscale user-facing metrics (as opposed
+to internal metrics printed by 'tailscale debug metrics').
+
+For more information about Tailscale metrics, refer to
+https://tailscale.com/s/client-metrics
+
+`),
+	ShortUsage: "tailscale metrics <subcommand> [flags]",
+	UsageFunc:  usageFuncNoDefaultValues,
+	Exec:       runMetricsNoSubcommand,
+	Subcommands: []*ffcli.Command{
+		{
+			Name:       "print",
+			ShortUsage: "tailscale metrics print",
+			Exec:       runMetricsPrint,
+			ShortHelp:  "Prints current metric values in the Prometheus text exposition format",
+		},
+		{
+			Name:       "write",
+			ShortUsage: "tailscale metrics write <path>",
+			Exec:       runMetricsWrite,
+			ShortHelp:  "Writes metric values to a file",
+			LongHelp: strings.TrimSpace(`
+
+The 'tailscale metrics write' command writes metric values to a text file provided as its
+only argument. It's meant to be used alongside Prometheus node exporter, allowing Tailscale
+metrics to be consumed and exported by the textfile collector.
+
+As an example, to export Tailscale metrics on an Ubuntu system running node exporter, you
+can regularly run 'tailscale metrics write /var/lib/prometheus/node-exporter/tailscaled.prom'
+using cron or a systemd timer.
+
+	`),
+		},
+	},
+}
+
+// runMetricsNoSubcommand prints metric values if no subcommand is specified.
+func runMetricsNoSubcommand(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("tailscale metrics: unknown subcommand: %s", args[0])
+	}
+
+	return runMetricsPrint(ctx, args)
+}
+
+// runMetricsPrint prints metric values to stdout.
+func runMetricsPrint(ctx context.Context, args []string) error {
+	out, err := localClient.UserMetrics(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(out)
+	return nil
+}
+
+// runMetricsWrite writes metric values to a file.
+func runMetricsWrite(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: tailscale metrics write <path>")
+	}
+	path := args[0]
+	out, err := localClient.UserMetrics(ctx)
+	if err != nil {
+		return err
+	}
+	return atomicfile.WriteFile(path, out, 0644)
+}

cmd/tailscale/cli/risks.go

@@ -17,11 +17,18 @@ import (
 )
 
 var (
-	riskTypes   []string
-	riskLoseSSH = registerRiskType("lose-ssh")
-	riskAll     = registerRiskType("all")
+	riskTypes           []string
+	riskLoseSSH         = registerRiskType("lose-ssh")
+	riskMacAppConnector = registerRiskType("mac-app-connector")
+	riskAll             = registerRiskType("all")
 )
 
+const riskMacAppConnectorMessage = `
+You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. 
+
+Do not use a macOS app connector for any mission-critical purposes. For the best experience, Linux is the only recommended platform for app connectors.
+`
+
 func registerRiskType(riskType string) string {
 	riskTypes = append(riskTypes, riskType)
 	return riskType

cmd/tailscale/cli/set.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/netip"
 	"os/exec"
+	"runtime"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
@@ -203,6 +204,12 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 		}
 	}
 
+	if runtime.GOOS == "darwin" && maskedPrefs.AppConnector.Advertise {
+		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, setArgs.acceptedRisks); err != nil {
+			return err
+		}
+	}
+
 	if maskedPrefs.RunSSHSet {
 		wantSSH, haveSSH := maskedPrefs.RunSSH, curPrefs.RunSSH
 		if err := presentSSHToggleRisk(wantSSH, haveSSH, setArgs.acceptedRisks); err != nil {

cmd/tailscale/cli/syspolicy.go

@@ -0,0 +1,110 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"slices"
+	"text/tabwriter"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/util/syspolicy/setting"
+)
+
+var syspolicyArgs struct {
+	json bool // JSON output mode
+}
+
+var syspolicyCmd = &ffcli.Command{
+	Name:       "syspolicy",
+	ShortHelp:  "Diagnose the MDM and system policy configuration",
+	LongHelp:   "The 'tailscale syspolicy' command provides tools for diagnosing the MDM and system policy configuration.",
+	ShortUsage: "tailscale syspolicy <subcommand>",
+	UsageFunc:  usageFuncNoDefaultValues,
+	Subcommands: []*ffcli.Command{
+		{
+			Name:       "list",
+			ShortUsage: "tailscale syspolicy list",
+			Exec:       runSysPolicyList,
+			ShortHelp:  "Prints effective policy settings",
+			LongHelp:   "The 'tailscale syspolicy list' subcommand displays the effective policy settings and their sources (e.g., MDM or environment variables).",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("syspolicy list")
+				fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format")
+				return fs
+			})(),
+		},
+		{
+			Name:       "reload",
+			ShortUsage: "tailscale syspolicy reload",
+			Exec:       runSysPolicyReload,
+			ShortHelp:  "Forces a reload of policy settings, even if no changes are detected, and prints the result",
+			LongHelp:   "The 'tailscale syspolicy reload' subcommand forces a reload of policy settings, even if no changes are detected, and prints the result.",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("syspolicy reload")
+				fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format")
+				return fs
+			})(),
+		},
+	},
+}
+
+func runSysPolicyList(ctx context.Context, args []string) error {
+	policy, err := localClient.GetEffectivePolicy(ctx, setting.DefaultScope())
+	if err != nil {
+		return err
+	}
+	printPolicySettings(policy)
+	return nil
+
+}
+
+func runSysPolicyReload(ctx context.Context, args []string) error {
+	policy, err := localClient.ReloadEffectivePolicy(ctx, setting.DefaultScope())
+	if err != nil {
+		return err
+	}
+	printPolicySettings(policy)
+	return nil
+}
+
+func printPolicySettings(policy *setting.Snapshot) {
+	if syspolicyArgs.json {
+		json, err := json.MarshalIndent(policy, "", "\t")
+		if err != nil {
+			errf("syspolicy marshalling error: %v", err)
+		} else {
+			outln(string(json))
+		}
+		return
+	}
+	if policy.Len() == 0 {
+		outln("No policy settings")
+		return
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "Name\tOrigin\tValue\tError")
+	fmt.Fprintln(w, "----\t------\t-----\t-----")
+	for _, k := range slices.Sorted(policy.Keys()) {
+		setting, _ := policy.GetSetting(k)
+		var origin string
+		if o := setting.Origin(); o != nil {
+			origin = o.String()
+		}
+		if err := setting.Error(); err != nil {
+			fmt.Fprintf(w, "%s\t%s\t\t{%v}\n", k, origin, err)
+		} else {
+			fmt.Fprintf(w, "%s\t%s\t%v\t\n", k, origin, setting.Value())
+		}
+	}
+	w.Flush()
+
+	fmt.Println()
+	return
+}

cmd/tailscale/cli/up.go

@@ -379,6 +379,12 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, err
 	}
 
+	if runtime.GOOS == "darwin" && env.upArgs.advertiseConnector {
+		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, env.upArgs.acceptedRisks); err != nil {
+			return false, nil, err
+		}
+	}
+
 	if env.upArgs.forceReauth && isSSHOverTailscale() {
 		if err := presentRiskToUser(riskLoseSSH, `You are connected over Tailscale; this action will result in your SSH session disconnecting.`, env.upArgs.acceptedRisks); err != nil {
 			return false, nil, err

cmd/tailscale/depaware.txt

@@ -5,10 +5,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
@@ -86,6 +82,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/cmd/tailscale/cli/ffcomplete/internal          from tailscale.com/cmd/tailscale/cli/ffcomplete
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlhttp                            from tailscale.com/cmd/tailscale/cli
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
@@ -124,7 +121,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/tlsdial/blockblame                         from tailscale.com/net/tlsdial
         tailscale.com/net/tsaddr                                     from tailscale.com/client/web+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
         tailscale.com/syncs                                          from tailscale.com/cmd/tailscale/cli+
@@ -148,6 +144,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/ptr                                      from tailscale.com/hostinfo+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/types/key+
         tailscale.com/types/views                                    from tailscale.com/tailcfg+
@@ -162,7 +159,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/groupmember                               from tailscale.com/client/web
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
         tailscale.com/util/mak                                       from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlhttp+
@@ -174,14 +171,18 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
-        tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
+        tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/usermetric                                from tailscale.com/health
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/util/syspolicy/source
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
@@ -320,7 +321,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         reflect                                                      from archive/tar+
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
-        runtime/debug                                                from github.com/coder/websocket/internal/xsync+
+        runtime/debug                                                from tailscale.com+
         slices                                                       from tailscale.com/client/web+
         sort                                                         from compress/flate+
         strconv                                                      from archive/tar+

cmd/tailscaled/depaware.txt

@@ -79,10 +79,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
-        github.com/coder/websocket                                   from tailscale.com/control/controlhttp+
-        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
-        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -249,6 +245,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/cmd/tailscaled+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
+        tailscale.com/control/controlhttp/controlhttpcommon          from tailscale.com/control/controlhttp
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/tailscaled+
@@ -327,7 +324,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/tsdial                                     from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
-        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -364,6 +360,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/ptr                                      from tailscale.com/control/controlclient+
+        tailscale.com/types/result                                   from tailscale.com/util/lineiter
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/tka+
         tailscale.com/types/views                                    from tailscale.com/ipn/ipnlocal+
@@ -381,7 +378,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
-        tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/control/controlclient+
         tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
@@ -401,8 +398,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
         tailscale.com/util/syspolicy                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/util/syspolicy/internal                        from tailscale.com/util/syspolicy/setting+
-        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy
-        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
+        tailscale.com/util/syspolicy/internal/loggerx                from tailscale.com/util/syspolicy/internal/metrics+
+        tailscale.com/util/syspolicy/internal/metrics                from tailscale.com/util/syspolicy/source
+        tailscale.com/util/syspolicy/rsop                            from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy+
+        tailscale.com/util/syspolicy/source                          from tailscale.com/util/syspolicy+
         tailscale.com/util/sysresources                              from tailscale.com/wgengine/magicsock
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
@@ -412,7 +412,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
-   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns
+   W 💣 tailscale.com/util/winutil/gp                                from tailscale.com/net/dns+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
    W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+

cmd/tailscaled/deps_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"testing"
+
+	"tailscale.com/tstest/deptest"
+)
+
+func TestOmitSSH(t *testing.T) {
+	const msg = "unexpected with ts_omit_ssh"
+	deptest.DepChecker{
+		GOOS:   "linux",
+		GOARCH: "amd64",
+		Tags:   "ts_omit_ssh",
+		BadDeps: map[string]string{
+			"tailscale.com/ssh/tailssh":            msg,
+			"golang.org/x/crypto/ssh":              msg,
+			"tailscale.com/sessionrecording":       msg,
+			"github.com/anmitsu/go-shlex":          msg,
+			"github.com/creack/pty":                msg,
+			"github.com/kr/fs":                     msg,
+			"github.com/pkg/sftp":                  msg,
+			"github.com/u-root/u-root/pkg/termios": msg,
+			"tempfork/gliderlabs/ssh":              msg,
+		},
+	}.Check(t)
+}

cmd/tailscaled/ssh.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build linux || darwin || freebsd || openbsd
+//go:build (linux || darwin || freebsd || openbsd) && !ts_omit_ssh
 
 package main
 

cmd/tailscaled/tailscaled.go

@@ -788,7 +788,6 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 }
 
 func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
-	tfs, _ := sys.DriveForLocal.GetOK()
 	ret, err := netstack.Create(logf,
 		sys.Tun.Get(),
 		sys.Engine.Get(),
@@ -796,7 +795,6 @@ func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
 		sys.Dialer.Get(),
 		sys.DNSManager.Get(),
 		sys.ProxyMapper(),
-		tfs,
 	)
 	if err != nil {
 		return nil, err

cmd/tailscaled/tailscaled_windows.go

@@ -134,14 +134,13 @@ func runWindowsService(pol *logpolicy.Policy) error {
 		logger.Logf(log.Printf).JSON(1, "SupportInfo", osdiag.SupportInfo(osdiag.LogSupportInfoReasonStartup))
 	}()
 
-	if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions {
-		syslog, err := eventlog.Open(serviceName)
-		if err == nil {
-			syslogf = func(format string, args ...any) {
+	if syslog, err := eventlog.Open(serviceName); err == nil {
+		syslogf = func(format string, args ...any) {
+			if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions {
 				syslog.Info(0, fmt.Sprintf(format, args...))
 			}
-			defer syslog.Close()
 		}
+		defer syslog.Close()
 	}
 
 	syslogf("Service entering svc.Run")
@@ -160,10 +159,7 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	changes <- svc.Status{State: svc.StartPending}
 	syslogf("Service start pending")
 
-	svcAccepts := svc.AcceptStop
-	if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock {
-		svcAccepts |= svc.AcceptSessionChange
-	}
+	svcAccepts := svc.AcceptStop | svc.AcceptSessionChange
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -371,13 +367,15 @@ func handleSessionChange(chgRequest svc.ChangeRequest) {
 		return
 	}
 
-	log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.")
-	go func() {
-		err := dns.Flush()
-		if err != nil {
-			log.Printf("Error flushing DNS on session unlock: %v", err)
-		}
-	}()
+	if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock {
+		log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.")
+		go func() {
+			err := dns.Flush()
+			if err != nil {
+				log.Printf("Error flushing DNS on session unlock: %v", err)
+			}
+		}()
+	}
 }
 
 var (

cmd/testwrapper/testwrapper.go

@@ -42,6 +42,7 @@ type testAttempt struct {
 	testName      string // "TestFoo"
 	outcome       string // "pass", "fail", "skip"
 	logs          bytes.Buffer
+	start, end    time.Time
 	isMarkedFlaky bool   // set if the test is marked as flaky
 	issueURL      string // set if the test is marked as flaky
 
@@ -132,11 +133,17 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 		}
 		pkg := goOutput.Package
 		pkgTests := resultMap[pkg]
+		if pkgTests == nil {
+			pkgTests = make(map[string]*testAttempt)
+			resultMap[pkg] = pkgTests
+		}
 		if goOutput.Test == "" {
 			switch goOutput.Action {
+			case "start":
+				pkgTests[""] = &testAttempt{start: goOutput.Time}
 			case "fail", "pass", "skip":
 				for _, test := range pkgTests {
-					if test.outcome == "" {
+					if test.testName != "" && test.outcome == "" {
 						test.outcome = "fail"
 						ch <- test
 					}
@@ -144,15 +151,13 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 				ch <- &testAttempt{
 					pkg:         goOutput.Package,
 					outcome:     goOutput.Action,
+					start:       pkgTests[""].start,
+					end:         goOutput.Time,
 					pkgFinished: true,
 				}
 			}
 			continue
 		}
-		if pkgTests == nil {
-			pkgTests = make(map[string]*testAttempt)
-			resultMap[pkg] = pkgTests
-		}
 		testName := goOutput.Test
 		if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
 			testName = test
@@ -168,8 +173,10 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
 			pkgTests[testName] = &testAttempt{
 				pkg:      pkg,
 				testName: testName,
+				start:    goOutput.Time,
 			}
 		case "skip", "pass", "fail":
+			pkgTests[testName].end = goOutput.Time
 			pkgTests[testName].outcome = goOutput.Action
 			ch <- pkgTests[testName]
 		case "output":
@@ -213,7 +220,7 @@ func main() {
 		firstRun.tests = append(firstRun.tests, &packageTests{Pattern: pkg})
 	}
 	toRun := []*nextRun{firstRun}
-	printPkgOutcome := func(pkg, outcome string, attempt int) {
+	printPkgOutcome := func(pkg, outcome string, attempt int, runtime time.Duration) {
 		if outcome == "skip" {
 			fmt.Printf("?\t%s [skipped/no tests] \n", pkg)
 			return
@@ -225,10 +232,10 @@ func main() {
 			outcome = "FAIL"
 		}
 		if attempt > 1 {
-			fmt.Printf("%s\t%s [attempt=%d]\n", outcome, pkg, attempt)
+			fmt.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt)
 			return
 		}
-		fmt.Printf("%s\t%s\n", outcome, pkg)
+		fmt.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds())
 	}
 
 	// Check for -coverprofile argument and filter it out
@@ -307,7 +314,7 @@ func main() {
 						// when a package times out.
 						failed = true
 					}
-					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt)
+					printPkgOutcome(tr.pkg, tr.outcome, thisRun.attempt, tr.end.Sub(tr.start))
 					continue
 				}
 				if testingVerbose || tr.outcome == "fail" {

cmd/testwrapper/testwrapper_test.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sync"
 	"testing"
 )
@@ -76,7 +77,10 @@ func TestFlakeRun(t *testing.T) {
 		t.Fatalf("go run . %s: %s with output:\n%s", testfile, err, out)
 	}
 
-	want := []byte("ok\t" + testfile + " [attempt=2]")
+	// Replace the unpredictable timestamp with "0.00s".
+	out = regexp.MustCompile(`\t\d+\.\d\d\ds\t`).ReplaceAll(out, []byte("\t0.00s\t"))
+
+	want := []byte("ok\t" + testfile + "\t0.00s\t[attempt=2]")
 	if !bytes.Contains(out, want) {
 		t.Fatalf("wanted output containing %q but got:\n%s", want, out)
 	}

cmd/tsconnect/common.go

@@ -150,6 +150,7 @@ func runEsbuildServe(buildOptions esbuild.BuildOptions) {
 		log.Fatalf("Cannot start esbuild server: %v", err)
 	}
 	log.Printf("Listening on http://%s:%d\n", result.Host, result.Port)
+	select {}
 }
 
 func runEsbuild(buildOptions esbuild.BuildOptions) esbuild.BuildResult {

cmd/tsconnect/wasm/wasm_js.go

@@ -115,7 +115,7 @@ func newIPN(jsConfig js.Value) map[string]any {
 	}
 	sys.Set(eng)
 
-	ns, err := netstack.Create(logf, sys.Tun.Get(), eng, sys.MagicSock.Get(), dialer, sys.DNSManager.Get(), sys.ProxyMapper(), nil)
+	ns, err := netstack.Create(logf, sys.Tun.Get(), eng, sys.MagicSock.Get(), dialer, sys.DNSManager.Get(), sys.ProxyMapper())
 	if err != nil {
 		log.Fatalf("netstack.Create: %v", err)
 	}
@@ -272,8 +272,8 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 						name = p.Hostinfo().Hostname()
 					}
 					addrs := make([]string, p.Addresses().Len())
-					for i := range p.Addresses().Len() {
-						addrs[i] = p.Addresses().At(i).Addr().String()
+					for i, ap := range p.Addresses().All() {
+						addrs[i] = ap.Addr().String()
 					}
 					return jsNetMapPeerNode{
 						jsNetMapNode: jsNetMapNode{
@@ -589,8 +589,8 @@ func mapSlice[T any, M any](a []T, f func(T) M) []M {
 
 func mapSliceView[T any, M any](a views.Slice[T], f func(T) M) []M {
 	n := make([]M, a.Len())
-	for i := range a.Len() {
-		n[i] = f(a.At(i))
+	for i, v := range a.All() {
+		n[i] = f(v)
 	}
 	return n
 }

control/controlclient/noise.go

@@ -17,7 +17,6 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlhttp"
-	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/dnscache"
@@ -30,7 +29,6 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/multierr"
 	"tailscale.com/util/singleflight"
-	"tailscale.com/util/testenv"
 )
 
 // NoiseClient provides a http.Client to connect to tailcontrol over
@@ -107,11 +105,6 @@ type NoiseOpts struct {
 	DialPlan func() *tailcfg.ControlDialPlan
 }
 
-// controlIsPlaintext is whether we should assume that the controlplane is only accessible
-// over plaintext HTTP (as the first hop, before the ts2021 encryption begins).
-// This is used by some tests which don't have a real TLS certificate.
-var controlIsPlaintext = envknob.RegisterBool("TS_CONTROL_IS_PLAINTEXT_HTTP")
-
 // NewNoiseClient returns a new noiseClient for the provided server and machine key.
 // serverURL is of the form https://<host>:<port> (no trailing slash).
 //
@@ -129,7 +122,7 @@ func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) {
 		if u.Scheme == "http" {
 			httpPort = port
 			httpsPort = "443"
-			if (testenv.InTest() || controlIsPlaintext()) && (u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost") {
+			if u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost" {
 				httpsPort = ""
 			}
 		} else {

control/controlclient/noise_test.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	"golang.org/x/net/http2"
-	"tailscale.com/control/controlhttp"
+	"tailscale.com/control/controlhttp/controlhttpserver"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
@@ -201,7 +201,7 @@ func (up *Upgrader) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return nil
 	}
 
-	cbConn, err := controlhttp.AcceptHTTP(r.Context(), w, r, up.noiseKeyPriv, earlyWriteFn)
+	cbConn, err := controlhttpserver.AcceptHTTP(r.Context(), w, r, up.noiseKeyPriv, earlyWriteFn)
 	if err != nil {
 		up.logf("controlhttp: Accept: %v", err)
 		return

control/controlclient/sign_supported.go

@@ -13,7 +13,6 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"sync"
 	"time"
 
 	"github.com/tailscale/certstore"
@@ -22,11 +21,6 @@ import (
 	"tailscale.com/util/syspolicy"
 )
 
-var getMachineCertificateSubjectOnce struct {
-	sync.Once
-	v string // Subject of machine certificate to search for
-}
-
 // getMachineCertificateSubject returns the exact name of a Subject that needs
 // to be present in an identity's certificate chain to sign a RegisterRequest,
 // formatted as per pkix.Name.String(). The Subject may be that of the identity
@@ -37,11 +31,8 @@ var getMachineCertificateSubjectOnce struct {
 //
 // Example: "CN=Tailscale Inc Test Root CA,OU=Tailscale Inc Test Certificate Authority,O=Tailscale Inc,ST=ON,C=CA"
 func getMachineCertificateSubject() string {
-	getMachineCertificateSubjectOnce.Do(func() {
-		getMachineCertificateSubjectOnce.v, _ = syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
-	})
-
-	return getMachineCertificateSubjectOnce.v
+	machineCertSubject, _ := syspolicy.GetString(syspolicy.MachineCertificateSubject, "")
+	return machineCertSubject
 }
 
 var (

control/controlhttp/client.go

@@ -38,6 +38,7 @@ import (
 	"time"
 
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
@@ -245,7 +246,7 @@ func (a *Dialer) dial(ctx context.Context) (*ClientConn, error) {
 		results[i].conn = nil // so we don't close it in the defer
 		return conn, nil
 	}
-	merr := multierr.New(errs...)
+	merr := multierr.New(multierr.DeduplicateContextErrors(errs)...)
 
 	// If we get here, then we didn't get anywhere with our dial plan; fall back to just using DNS.
 	a.logf("controlhttp: failed dialing using DialPlan, falling back to DNS; errs=%s", merr.Error())
@@ -571,9 +572,9 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad
 		Method: "POST",
 		URL:    u,
 		Header: http.Header{
-			"Upgrade":           []string{upgradeHeaderValue},
-			"Connection":        []string{"upgrade"},
-			handshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
+			"Upgrade":                             []string{controlhttpcommon.UpgradeHeaderValue},
+			"Connection":                          []string{"upgrade"},
+			controlhttpcommon.HandshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
 		},
 	}
 	req = req.WithContext(ctx)
@@ -597,7 +598,7 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad
 		return nil, fmt.Errorf("httptrace didn't provide a connection")
 	}
 
-	if next := resp.Header.Get("Upgrade"); next != upgradeHeaderValue {
+	if next := resp.Header.Get("Upgrade"); next != controlhttpcommon.UpgradeHeaderValue {
 		resp.Body.Close()
 		return nil, fmt.Errorf("server switched to unexpected protocol %q", next)
 	}

control/controlhttp/client_js.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/net/wsconn"
 )
 
@@ -42,11 +43,11 @@ func (d *Dialer) Dial(ctx context.Context) (*ClientConn, error) {
 		// Can't set HTTP headers on the websocket request, so we have to to send
 		// the handshake via an HTTP header.
 		RawQuery: url.Values{
-			handshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
+			controlhttpcommon.HandshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
 		}.Encode(),
 	}
 	wsConn, _, err := websocket.Dial(ctx, wsURL.String(), &websocket.DialOptions{
-		Subprotocols: []string{upgradeHeaderValue},
+		Subprotocols: []string{controlhttpcommon.UpgradeHeaderValue},
 	})
 	if err != nil {
 		return nil, err

control/controlhttp/constants.go

@@ -18,15 +18,6 @@ import (
 )
 
 const (
-	// upgradeHeader is the value of the Upgrade HTTP header used to
-	// indicate the Tailscale control protocol.
-	upgradeHeaderValue = "tailscale-control-protocol"
-
-	// handshakeHeaderName is the HTTP request header that can
-	// optionally contain base64-encoded initial handshake
-	// payload, to save an RTT.
-	handshakeHeaderName = "X-Tailscale-Handshake"
-
 	// serverUpgradePath is where the server-side HTTP handler to
 	// to do the protocol switch is located.
 	serverUpgradePath = "/ts2021"
@@ -85,6 +76,8 @@ type Dialer struct {
 	// dropped.
 	Logf logger.Logf
 
+	// NetMon is the [netmon.Monitor] to use for this Dialer. It must be
+	// non-nil.
 	NetMon *netmon.Monitor
 
 	// HealthTracker, if non-nil, is the health tracker to use.

control/controlhttp/controlhttpcommon/controlhttpcommon.go

@@ -0,0 +1,15 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package controlhttpcommon contains common constants for used
+// by the controlhttp client and controlhttpserver packages.
+package controlhttpcommon
+
+// UpgradeHeader is the value of the Upgrade HTTP header used to
+// indicate the Tailscale control protocol.
+const UpgradeHeaderValue = "tailscale-control-protocol"
+
+// handshakeHeaderName is the HTTP request header that can
+// optionally contain base64-encoded initial handshake
+// payload, to save an RTT.
+const HandshakeHeaderName = "X-Tailscale-Handshake"

control/controlhttp/controlhttpserver/controlhttpserver.go

@@ -3,7 +3,8 @@
 
 //go:build !ios
 
-package controlhttp
+// Package controlhttpserver contains the HTTP server side of the ts2021 control protocol.
+package controlhttpserver
 
 import (
 	"context"
@@ -18,6 +19,7 @@ import (
 
 	"github.com/coder/websocket"
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/wsconn"
 	"tailscale.com/types/key"
@@ -45,12 +47,12 @@ func acceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, pri
 	if next == "websocket" {
 		return acceptWebsocket(ctx, w, r, private)
 	}
-	if next != upgradeHeaderValue {
+	if next != controlhttpcommon.UpgradeHeaderValue {
 		http.Error(w, "unknown next protocol", http.StatusBadRequest)
 		return nil, fmt.Errorf("client requested unhandled next protocol %q", next)
 	}
 
-	initB64 := r.Header.Get(handshakeHeaderName)
+	initB64 := r.Header.Get(controlhttpcommon.HandshakeHeaderName)
 	if initB64 == "" {
 		http.Error(w, "missing Tailscale handshake header", http.StatusBadRequest)
 		return nil, errors.New("no tailscale handshake header in HTTP request")
@@ -67,7 +69,7 @@ func acceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, pri
 		return nil, errors.New("can't hijack client connection")
 	}
 
-	w.Header().Set("Upgrade", upgradeHeaderValue)
+	w.Header().Set("Upgrade", controlhttpcommon.UpgradeHeaderValue)
 	w.Header().Set("Connection", "upgrade")
 	w.WriteHeader(http.StatusSwitchingProtocols)
 
@@ -117,7 +119,7 @@ func acceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, pri
 // speak HTTP) to a Tailscale control protocol base transport connection.
 func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request, private key.MachinePrivate) (*controlbase.Conn, error) {
 	c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
-		Subprotocols:   []string{upgradeHeaderValue},
+		Subprotocols:   []string{controlhttpcommon.UpgradeHeaderValue},
 		OriginPatterns: []string{"*"},
 		// Disable compression because we transmit Noise messages that are not
 		// compressible.
@@ -129,7 +131,7 @@ func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request
 	if err != nil {
 		return nil, fmt.Errorf("Could not accept WebSocket connection %v", err)
 	}
-	if c.Subprotocol() != upgradeHeaderValue {
+	if c.Subprotocol() != controlhttpcommon.UpgradeHeaderValue {
 		c.Close(websocket.StatusPolicyViolation, "client must speak the control subprotocol")
 		return nil, fmt.Errorf("Unexpected subprotocol %q", c.Subprotocol())
 	}
@@ -137,7 +139,7 @@ func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request
 		c.Close(websocket.StatusPolicyViolation, "Could not parse parameters")
 		return nil, fmt.Errorf("parse query parameters: %v", err)
 	}
-	initB64 := r.Form.Get(handshakeHeaderName)
+	initB64 := r.Form.Get(controlhttpcommon.HandshakeHeaderName)
 	if initB64 == "" {
 		c.Close(websocket.StatusPolicyViolation, "missing Tailscale handshake parameter")
 		return nil, errors.New("no tailscale handshake parameter in HTTP request")

control/controlhttp/http_test.go

@@ -23,12 +23,16 @@ import (
 	"time"
 
 	"tailscale.com/control/controlbase"
+	"tailscale.com/control/controlhttp/controlhttpcommon"
+	"tailscale.com/control/controlhttp/controlhttpserver"
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/socks5"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
+	"tailscale.com/tstest/deptest"
 	"tailscale.com/tstime"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -158,7 +162,7 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 				return err
 			}
 		}
-		conn, err := AcceptHTTP(context.Background(), w, r, server, earlyWriteFn)
+		conn, err := controlhttpserver.AcceptHTTP(context.Background(), w, r, server, earlyWriteFn)
 		if err != nil {
 			log.Print(err)
 		}
@@ -225,6 +229,7 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 		omitCertErrorLogging: true,
 		testFallbackDelay:    fallbackDelay,
 		Clock:                clock,
+		HealthTracker:        new(health.Tracker),
 	}
 
 	if param.httpInDial {
@@ -529,7 +534,7 @@ EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA==
 
 func brokenMITMHandler(clock tstime.Clock) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Upgrade", upgradeHeaderValue)
+		w.Header().Set("Upgrade", controlhttpcommon.UpgradeHeaderValue)
 		w.Header().Set("Connection", "upgrade")
 		w.WriteHeader(http.StatusSwitchingProtocols)
 		w.(http.Flusher).Flush()
@@ -574,7 +579,7 @@ func TestDialPlan(t *testing.T) {
 			close(done)
 		})
 		var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			conn, err := AcceptHTTP(context.Background(), w, r, server, nil)
+			conn, err := controlhttpserver.AcceptHTTP(context.Background(), w, r, server, nil)
 			if err != nil {
 				log.Print(err)
 			} else {
@@ -726,6 +731,7 @@ func TestDialPlan(t *testing.T) {
 				omitCertErrorLogging: true,
 				testFallbackDelay:    50 * time.Millisecond,
 				Clock:                clock,
+				HealthTracker:        new(health.Tracker),
 			}
 
 			conn, err := a.dial(ctx)
@@ -816,3 +822,14 @@ func (c *closeTrackConn) Close() error {
 	c.d.noteClose(c)
 	return c.Conn.Close()
 }
+
+func TestDeps(t *testing.T) {
+	deptest.DepChecker{
+		GOOS:   "darwin",
+		GOARCH: "arm64",
+		BadDeps: map[string]string{
+			// Only the controlhttpserver needs WebSockets...
+			"github.com/coder/websocket": "controlhttp client shouldn't need websockets",
+		},
+	}.Check(t)
+}

derp/derphttp/derphttp_client.go

@@ -313,6 +313,9 @@ func (c *Client) preferIPv6() bool {
 var dialWebsocketFunc func(ctx context.Context, urlStr string) (net.Conn, error)
 
 func useWebsockets() bool {
+	if !canWebsockets {
+		return false
+	}
 	if runtime.GOOS == "js" {
 		return true
 	}
@@ -383,7 +386,7 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	var node *tailcfg.DERPNode // nil when using c.url to dial
 	var idealNodeInRegion bool
 	switch {
-	case useWebsockets():
+	case canWebsockets && useWebsockets():
 		var urlStr string
 		if c.url != nil {
 			urlStr = c.url.String()

derp/derphttp/derphttp_test.go

@@ -17,7 +17,9 @@ import (
 
 	"tailscale.com/derp"
 	"tailscale.com/net/netmon"
+	"tailscale.com/tstest/deptest"
 	"tailscale.com/types/key"
+	"tailscale.com/util/set"
 )
 
 func TestSendRecv(t *testing.T) {
@@ -485,3 +487,23 @@ func TestProbe(t *testing.T) {
 		}
 	}
 }
+
+func TestDeps(t *testing.T) {
+	deptest.DepChecker{
+		GOOS:   "darwin",
+		GOARCH: "arm64",
+		BadDeps: map[string]string{
+			"github.com/coder/websocket": "shouldn't link websockets except on js/wasm",
+		},
+	}.Check(t)
+
+	deptest.DepChecker{
+		GOOS:   "darwin",
+		GOARCH: "arm64",
+		Tags:   "ts_debug_websockets",
+		WantDeps: set.Of(
+			"github.com/coder/websocket",
+		),
+	}.Check(t)
+
+}

derp/derphttp/websocket.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build linux || js
+//go:build js || ((linux || darwin) && ts_debug_websockets)
 
 package derphttp
 
@@ -14,6 +14,8 @@ import (
 	"tailscale.com/net/wsconn"
 )
 
+const canWebsockets = true
+
 func init() {
 	dialWebsocketFunc = dialWebsocket
 }

derp/derphttp/websocket_stub.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !(js || ((linux || darwin) && ts_debug_websockets))
+
+package derphttp
+
+const canWebsockets = false

docs/k8s/proxy.yaml

@@ -44,6 +44,14 @@ spec:
       value: "{{TS_DEST_IP}}"
     - name: TS_AUTH_ONCE
       value: "true"
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
     securityContext:
       capabilities:
         add:

docs/k8s/role.yaml

@@ -13,3 +13,6 @@ rules:
   resourceNames: ["{{TS_KUBE_SECRET}}"]
   resources: ["secrets"]
   verbs: ["get", "update", "patch"]
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["events"]
+  verbs: ["get", "create", "patch"]

docs/k8s/sidecar.yaml

@@ -26,6 +26,14 @@ spec:
           name: tailscale-auth
           key: TS_AUTHKEY
           optional: true
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
     securityContext:
       capabilities:
         add:

docs/k8s/subnet.yaml

@@ -28,6 +28,14 @@ spec:
           optional: true
     - name: TS_ROUTES
       value: "{{TS_ROUTES}}"
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
     securityContext:
       capabilities:
         add:

docs/k8s/userspace-sidecar.yaml

@@ -27,3 +27,11 @@ spec:
           name: tailscale-auth
           key: TS_AUTHKEY
           optional: true
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid

docs/windows/policy/en-US/tailscale.adml

@@ -15,16 +15,18 @@
             <string id="SINCE_V1_58">Tailscale version 1.58.0 and later</string>
             <string id="SINCE_V1_62">Tailscale version 1.62.0 and later</string>
             <string id="SINCE_V1_74">Tailscale version 1.74.0 and later</string>
+            <string id="SINCE_V1_78">Tailscale version 1.78.0 and later</string>
             <string id="Tailscale_Category">Tailscale</string>
             <string id="UI_Category">UI customization</string>
             <string id="Settings_Category">Settings</string>
             <string id="LoginURL">Require using a specific Tailscale coordination server</string>
             <string id="LoginURL_Help"><![CDATA[This policy can be used to require the use of a particular Tailscale coordination server.
-See https://tailscale.com/kb/1315/mdm-keys#set-a-custom-control-server-url for more details.
 
-If you configure this policy, set it to the URL of your coordination server, beginning with https:// and ending with no trailing slash. If blank or "https://controlplane.tailscale.com", the default coordination server will be required.
+If you enable this policy, set it to the URL of your coordination server, beginning with https:// and ending with no trailing slash. If blank or "https://controlplane.tailscale.com", the default coordination server will be required.
 
-If you disable this policy, the Tailscale SaaS coordination server will be used by default, but a non-standard Tailscale coordination server can be configured using the CLI.]]></string>
+If you disable or do not configure this policy, the Tailscale SaaS coordination server will be used by default, but a non-standard Tailscale coordination server can be configured using the CLI.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-a-custom-control-server-url for more details.]]></string>
             <string id="LogTarget">Require using a specific Tailscale log server</string>
             <string id="LogTarget_Help"><![CDATA[This policy can be used to require the use of a non-standard log server.
 Please note that using a non-standard log server will limit Tailscale Support's ability to diagnose problems.
@@ -34,15 +36,16 @@ If you configure this policy, set it to the URL of your log server, beginning wi
 If you disable this policy, the Tailscale standard log server will be used by default, but a non-standard Tailscale log server can be configured using the TS_LOG_TARGET environment variable.]]></string>
             <string id="Tailnet">Specify which Tailnet should be used for Login</string>
             <string id="Tailnet_Help"><![CDATA[This policy can be used to suggest or require a specific tailnet when opening the login page.
-See https://tailscale.com/kb/1315/mdm-keys#set-a-suggested-or-required-tailnet for more details.
 
 To suggest a tailnet at login time, set this to the name of the tailnet, as shown in the top-left of the admin panel, such as "example.com". That tailnet's SSO button will be shown prominently, along with the option to select a different tailnet.
 
 To require logging in to a particular tailnet, add the "required:" prefix, such as "required:example.com". The result is similar to the suggested tailnet but there will be no option to choose a different tailnet.
 
-If you configure this policy, set it to the name of the tailnet, possibly with the "required:" prefix, as described above.
+If you enable this policy, set it to the name of the tailnet, possibly with the "required:" prefix, as described above.
 
-If you disable this policy, the standard login page will be used.]]></string>
+If you disable or do not configure this policy, the standard login page will be used.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-a-suggested-or-required-tailnet for more details.]]></string>
             <string id="AuthKey">Specify the auth key to authenticate devices without user interaction</string>
             <string id="AuthKey_Help"><![CDATA[This policy allows specifying the default auth key to be used when registering new devices without requiring sign-in via a web browser, unless the user specifies a different auth key via the CLI.
 
@@ -52,85 +55,101 @@ While MDM solutions tend to offer better control over who can access the policy
 
 Only consider this option after carefully reviewing the organization's security posture. For example, ensure you configure the auth keys specifically for the tag of the device and that access control policies only grant necessary access between the tailnet and the tagged device. Additionally, consider using short-lived auth keys, one-time auth keys (with one GPO/MDM configuration per device), Device Approval, and/or Tailnet lock to minimize risk. If you suspect an auth key has been compromised, revoke the auth key immediately.
 
-If you configure this policy setting and specify an auth key, it will be used to authenticate the device unless the device is already logged in or an auth key is explicitly specified via the CLI.
+If you enable this policy setting and specify an auth key, it will be used to authenticate the device unless the device is already logged in or an auth key is explicitly specified via the CLI.
 
 If you disable or do not configure this policy setting, an interactive user login will be required..
 
 See https://tailscale.com/kb/1315/mdm-keys#set-an-auth-key for more details.]]></string>
             <string id="ExitNodeID">Require using a specific Exit Node</string>
             <string id="ExitNodeID_Help"><![CDATA[This policy can be used to require always using the specified Exit Node whenever the Tailscale client is connected.
-See https://tailscale.com/kb/1315/mdm-keys#force-an-exit-node-to-always-be-used and https://tailscale.com/kb/1103/exit-nodes for more details.
-
-If you enable this policy, set it to the ID of an exit node. The ID is visible on the Machines page of the admin console, or can be queried using the Tailscale API. If the specified exit node is unavailable, this device will have no Internet access unless Tailscale is disconnected.
+    
+If you enable this policy, set it to the ID of an exit node. The ID is visible on the Machines page of the admin console, or can be queried using the Tailscale API. If the specified exit node is unavailable, this device will have no Internet access unless Tailscale is disconnected. Alternatively, you can set it to "auto:any" (without quotes), which allows the Tailscale client to automatically select the most suitable exit node.
 
 If you disable this policy or supply an empty exit node ID, then usage of exit nodes will be disallowed.
 
-If you do not configure this policy, no exit node will be used by default but an exit node (if one is available and permitted by ACLs) can be chosen by the user if desired.]]></string>
+If you do not configure this policy, no exit node will be used by default but an exit node (if one is available and permitted by ACLs) can be chosen by the user if desired.
+
+See https://tailscale.com/kb/1315/mdm-keys#force-an-exit-node-to-always-be-used and https://tailscale.com/kb/1103/exit-nodes for more details.]]></string>
+            <string id="AllowedSuggestedExitNodes">Limit automated Exit Node suggestions to specific nodes</string>
+            <string id="AllowedSuggestedExitNodes_Help"><![CDATA[This policy setting allows configuring a pool of exit nodes from which the Tailscale client will automatically select the most suitable suggested exit node when required. The suggested exit node is displayed in the GUI and CLI and is automatically selected and enforced when the "Require using a specific Exit Node" policy setting is enabled and set to "auto:any".
+
+If you enable this policy setting, suggestions will be limited to exit nodes with the specified IDs. The IDs are visible on the Machines page of the admin console, or can be queried using the Tailscale API.
+
+If you disable or do not configure this policy setting, no limitations will apply, and all available exit nodes will be considered when selecting the most suitable suggested node.
+
+See https://tailscale.com/kb/1315/mdm-keys#suggest-allowed-forced-exit-nodes and https://tailscale.com/kb/1103/exit-nodes for more details.]]></string>
             <string id="AllowIncomingConnections">Allow incoming connections</string>
             <string id="AllowIncomingConnections_Help"><![CDATA[This policy can be used to require that the Allow Incoming Connections setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-whether-to-allow-incoming-connections and https://tailscale.com/kb/1072/client-preferences#allow-incoming-connections for more details.
 
 If you enable this policy, then Allow Incoming Connections is always enabled and the menu option is hidden.
 
 If you disable this policy, then Allow Incoming Connections is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Allow Incoming Connections depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Allow Incoming Connections depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-whether-to-allow-incoming-connections and https://tailscale.com/kb/1072/client-preferences#allow-incoming-connections for more details.]]></string>
             <string id="UnattendedMode">Run Tailscale in Unattended Mode</string>
             <string id="UnattendedMode_Help"><![CDATA[This policy can be used to require that the Run Unattended setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-unattended-mode and https://tailscale.com/kb/1088/run-unattended for more details.
 
 If you enable this policy, then Run Unattended is always enabled and the menu option is hidden.
 
 If you disable this policy, then Run Unattended is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Run Unattended depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Run Unattended depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-unattended-mode and https://tailscale.com/kb/1088/run-unattended for more details.]]></string>
             <string id="ExitNodeAllowLANAccess">Allow Local Network Access when an Exit Node is in use</string>
             <string id="ExitNodeAllowLANAccess_Help"><![CDATA[This policy can be used to require that the Allow Local Network Access setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#toggle-local-network-access-when-an-exit-node-is-in-use and https://tailscale.com/kb/1103/exit-nodes#step-4-use-the-exit-node for more details.
 
 If you enable this policy, then Allow Local Network Access is always enabled and the menu option is hidden.
 
 If you disable this policy, then Allow Local Network Access is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Allow Local Network Access depends on what is selected in the Exit Node submenu.]]></string>
+If you do not configure this policy, then Allow Local Network Access depends on what is selected in the Exit Node submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#toggle-local-network-access-when-an-exit-node-is-in-use and https://tailscale.com/kb/1103/exit-nodes#step-4-use-the-exit-node for more details.]]></string>
             <string id="UseTailscaleDNSSettings">Use Tailscale DNS Settings</string>
             <string id="UseTailscaleDNSSettings_Help"><![CDATA[This policy can be used to require that Use Tailscale DNS is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-uses-tailscale-dns-settings for more details.
 
 If you enable this policy, then Use Tailscale DNS is always enabled and the menu option is hidden.
 
 If you disable this policy, then Use Tailscale DNS is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Use Tailscale DNS depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Use Tailscale DNS depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-uses-tailscale-dns-settings for more details.]]></string>
             <string id="UseTailscaleSubnets">Use Tailscale Subnets</string>
             <string id="UseTailscaleSubnets_Help"><![CDATA[This policy can be used to require that Use Tailscale Subnets is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-accepts-tailscale-subnets or https://tailscale.com/kb/1019/subnets for more details.
 
 If you enable this policy, then Use Tailscale Subnets is always enabled and the menu option is hidden.
 
 If you disable this policy, then Use Tailscale Subnets is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Use Tailscale Subnets depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Use Tailscale Subnets depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#set-whether-the-device-accepts-tailscale-subnets or https://tailscale.com/kb/1019/subnets for more details.]]></string>
             <string id="InstallUpdates">Automatically install updates</string>
             <string id="InstallUpdates_Help"><![CDATA[This policy can be used to require that Automatically Install Updates is configured a certain way.
-See https://tailscale.com/kb/1067/update#auto-updates for more details.
 
 If you enable this policy, then Automatically Install Updates is always enabled and the menu option is hidden.
 
 If you disable this policy, then Automatically Install Updates is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Automatically Install Updates depends on what is selected in the Preferences submenu.]]></string>
+If you do not configure this policy, then Automatically Install Updates depends on what is selected in the Preferences submenu.
+
+See https://tailscale.com/kb/1067/update#auto-updates for more details.]]></string>
             <string id="AdvertiseExitNode">Run Tailscale as an Exit Node</string>
             <string id="AdvertiseExitNode_Help"><![CDATA[This policy can be used to require that Run Exit Node is configured a certain way.
-See https://tailscale.com/kb/1103/exit-nodes for more details.
 
 If you enable this policy, then Run Exit Node is always enabled and the menu option is hidden.
 
 If you disable this policy, then Run Exit Node is always disabled and the menu option is hidden.
 
-If you do not configure this policy, then Run Exit Node depends on what is selected in the Exit Node submenu.]]></string>
-            <string id="AdminPanel">Show the "Admin Panel" menu item</string>
-            <string id="AdminPanel_Help"><![CDATA[This policy can be used to show or hide the Admin Console item in the Tailscale Menu.
+If you do not configure this policy, then Run Exit Node depends on what is selected in the Exit Node submenu.
+
+See https://tailscale.com/kb/1103/exit-nodes for more details.]]></string>
+            <string id="AdminConsole">Show the "Admin Console" menu item</string>
+            <string id="AdminConsole_Help"><![CDATA[This policy can be used to show or hide the Admin Console item in the Tailscale Menu.
 
 If you enable or don't configure this policy, the Admin Console item will be shown in the Tailscale menu when available.
 
@@ -143,49 +162,55 @@ If you enable or don't configure this policy, the Network Devices submenu will b
 If you disable this policy, the Network Devices submenu will be hidden from the Tailscale menu. This does not affect other devices' visibility in the CLI.]]></string>
             <string id="TestMenu">Show the "Debug" submenu</string>
             <string id="TestMenu_Help"><![CDATA[This policy can be used to show or hide the Debug submenu of the Tailscale menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-debug-menu for more details.
 
 If you enable or don't configure this policy, the Debug submenu will be shown in the Tailscale menu when opened while holding Ctrl.
 
-If you disable this policy, the Debug submenu will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Debug submenu will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-debug-menu for more details.]]></string>
             <string id="UpdateMenu">Show the "Update Available" menu item</string>
             <string id="UpdateMenu_Help"><![CDATA[This policy can be used to show or hide the Update Available item in the Tailscale Menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-update-menu for more details.
 
 If you enable or don't configure this policy, the Update Available item will be shown in the Tailscale menu when there is an update.
 
-If you disable this policy, the Update Available item will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Update Available item will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-update-menu for more details.]]></string>
             <string id="RunExitNode">Show the "Run Exit Node" menu item</string>
             <string id="RunExitNode_Help"><![CDATA[This policy can be used to show or hide the Run Exit Node item in the Exit Node submenu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-run-as-exit-node-menu-item for more details.
 This does not affect using the CLI to enable or disable advertising an exit node. If you wish to enable or disable this feature, see the Run Exit Node policy in the Settings category.
 
 If you enable or don't configure this policy, the Run Exit Node item will be shown in the Exit Node submenu.
 
-If you disable this policy, the Run Exit Node item will be hidden from the Exit Node submenu.]]></string>
+If you disable this policy, the Run Exit Node item will be hidden from the Exit Node submenu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-run-as-exit-node-menu-item for more details.]]></string>
             <string id="PreferencesMenu">Show the "Preferences" submenu</string>
             <string id="PreferencesMenu_Help"><![CDATA[This policy can be used to show or hide the Preferences submenu of the Tailscale menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-preferences-menu for more details.
 This does not affect using the CLI to modify that menu's preferences. If you wish to control those, look at the policies in the Settings category.
 
 If you enable or don't configure this policy, the Preferences submenu will be shown in the Tailscale menu.
 
-If you disable this policy, the Preferences submenu will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Preferences submenu will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-preferences-menu for more details.]]></string>
             <string id="ExitNodesPicker">Show the "Exit Node" submenu</string>
             <string id="ExitNodesPicker_Help"><![CDATA[This policy can be used to show or hide the Exit Node submenu of the Tailscale menu.
-See https://tailscale.com/kb/1315/mdm-keys#hide-the-exit-node-picker for more details.
 This does not affect using the CLI to select or stop using an exit node. If you wish to control exit node usage, look at the "Require using a specific Exit Node" policy in the Settings category.
 
 If you enable or don't configure this policy, the Exit Node submenu will be shown in the Tailscale menu.
 
-If you disable this policy, the Exit Node submenu will be hidden from the Tailscale menu.]]></string>
+If you disable this policy, the Exit Node submenu will be hidden from the Tailscale menu.
+
+See https://tailscale.com/kb/1315/mdm-keys#hide-the-exit-node-picker for more details.]]></string>
             <string id="KeyExpirationNotice">Specify a custom key expiration notification time</string>
             <string id="KeyExpirationNotice_Help"><![CDATA[This policy can be used to configure how soon the notification appears before key expiry.
-See https://tailscale.com/kb/1315/mdm-keys#set-the-key-expiration-notice-period for more details.
 
 If you enable this policy and supply a valid time interval, the key expiry notification will begin to display when the current key has less than that amount of time remaining.
 
-If you disable or don't configure this policy, the default time period will be used (as of Tailscale 1.56, this is 24 hours).]]></string>
+If you disable or don't configure this policy, the default time period will be used (as of Tailscale 1.56, this is 24 hours).
+
+See https://tailscale.com/kb/1315/mdm-keys#set-the-key-expiration-notice-period for more details.]]></string>
             <string id="LogSCMInteractions">Log extra details about service events</string>
             <string id="LogSCMInteractions_Help"><![CDATA[This policy can be used to enable additional logging related to Service Control Manager for debugging purposes.
 This should only be enabled if recommended by Tailscale Support.
@@ -202,13 +227,14 @@ If you enable this policy, the DNS cache will be flushed on session unlock in ad
 If you disable or don't configure this policy, the DNS cache is managed normally.]]></string>
             <string id="PostureChecking">Collect data for posture checking</string>
             <string id="PostureChecking_Help"><![CDATA[This policy can be used to require that the Posture Checking setting is configured a certain way.
-See https://tailscale.com/kb/1315/mdm-keys#enable-gathering-device-posture-data and https://tailscale.com/kb/1326/device-identity for more details.
 
 If you enable this policy, then data collection is always enabled.
 
 If you disable this policy, then data collection is always disabled.
 
-If you do not configure this policy, then data collection depends on if it has been enabled from the CLI (as of Tailscale 1.56), it may be present in the GUI in later versions.]]></string>
+If you do not configure this policy, then data collection depends on if it has been enabled from the CLI (as of Tailscale 1.56), it may be present in the GUI in later versions.
+
+See https://tailscale.com/kb/1315/mdm-keys#enable-gathering-device-posture-data and https://tailscale.com/kb/1326/device-identity for more details.]]></string>
             <string id="ManagedBy">Show the "Managed By {Organization}" menu item</string>
             <string id="ManagedBy_Help"><![CDATA[Use this policy to configure the “Managed By {Organization}” item in the Tailscale Menu.
 
@@ -244,6 +270,9 @@ See https://tailscale.com/kb/1315/mdm-keys#set-your-organization-name for more d
                     <label>Exit Node:</label>
                 </textBox>
             </presentation>
+            <presentation id="AllowedSuggestedExitNodes">
+                <listBox refId="AllowedSuggestedExitNodesList">Target IDs:</listBox>
+            </presentation>
             <presentation id="ManagedBy">
                 <textBox refId="ManagedByOrganization">
                     <label>Organization Name:</label>

docs/windows/policy/tailscale.admx

@@ -50,6 +50,10 @@
                   displayName="$(string.SINCE_V1_74)">
         <and><reference ref="TAILSCALE_PRODUCT"/></and>
       </definition>
+      <definition name="SINCE_V1_78"
+                  displayName="$(string.SINCE_V1_78)">
+        <and><reference ref="TAILSCALE_PRODUCT"/></and>
+      </definition>
     </definitions>
   </supportedOn>
   <categories>
@@ -94,7 +98,14 @@
       <parentCategory ref="Settings_Category" />
       <supportedOn ref="SINCE_V1_56" />
       <elements>
-        <text id="ExitNodeIDPrompt" valueName="ExitNodeID" required="true" />
+        <text id="ExitNodeIDPrompt" valueName="ExitNodeID" required="true" />>
+      </elements>
+    </policy>
+    <policy name="AllowedSuggestedExitNodes" class="Machine" displayName="$(string.AllowedSuggestedExitNodes)" explainText="$(string.AllowedSuggestedExitNodes_Help)" presentation="$(presentation.AllowedSuggestedExitNodes)" key="Software\Policies\Tailscale\AllowedSuggestedExitNodes">
+      <parentCategory ref="Settings_Category" />
+      <supportedOn ref="SINCE_V1_78" />
+      <elements>
+        <list id="AllowedSuggestedExitNodesList" />
       </elements>
     </policy>
     <policy name="AllowIncomingConnections" class="Machine" displayName="$(string.AllowIncomingConnections)" explainText="$(string.AllowIncomingConnections_Help)" key="Software\Policies\Tailscale" valueName="AllowIncomingConnections">
@@ -197,7 +208,7 @@
         <decimal value="0" />
       </disabledValue>
     </policy>
-    <policy name="AdminPanel" class="Machine" displayName="$(string.AdminPanel)" explainText="$(string.AdminPanel_Help)" key="Software\Policies\Tailscale" valueName="AdminPanel">
+    <policy name="AdminConsole" class="Both" displayName="$(string.AdminConsole)" explainText="$(string.AdminConsole_Help)" key="Software\Policies\Tailscale" valueName="AdminConsole">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -207,7 +218,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="NetworkDevices" class="Machine" displayName="$(string.NetworkDevices)" explainText="$(string.NetworkDevices_Help)" key="Software\Policies\Tailscale" valueName="NetworkDevices">
+    <policy name="NetworkDevices" class="Both" displayName="$(string.NetworkDevices)" explainText="$(string.NetworkDevices_Help)" key="Software\Policies\Tailscale" valueName="NetworkDevices">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -217,7 +228,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="TestMenu" class="Machine" displayName="$(string.TestMenu)" explainText="$(string.TestMenu_Help)" key="Software\Policies\Tailscale" valueName="TestMenu">
+    <policy name="TestMenu" class="Both" displayName="$(string.TestMenu)" explainText="$(string.TestMenu_Help)" key="Software\Policies\Tailscale" valueName="TestMenu">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -227,7 +238,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="UpdateMenu" class="Machine" displayName="$(string.UpdateMenu)" explainText="$(string.UpdateMenu_Help)" key="Software\Policies\Tailscale" valueName="UpdateMenu">
+    <policy name="UpdateMenu" class="Both" displayName="$(string.UpdateMenu)" explainText="$(string.UpdateMenu_Help)" key="Software\Policies\Tailscale" valueName="UpdateMenu">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -237,7 +248,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="RunExitNode" class="Machine" displayName="$(string.RunExitNode)" explainText="$(string.RunExitNode_Help)" key="Software\Policies\Tailscale" valueName="RunExitNode">
+    <policy name="RunExitNode" class="Both" displayName="$(string.RunExitNode)" explainText="$(string.RunExitNode_Help)" key="Software\Policies\Tailscale" valueName="RunExitNode">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -247,7 +258,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="PreferencesMenu" class="Machine" displayName="$(string.PreferencesMenu)" explainText="$(string.PreferencesMenu_Help)" key="Software\Policies\Tailscale" valueName="PreferencesMenu">
+    <policy name="PreferencesMenu" class="Both" displayName="$(string.PreferencesMenu)" explainText="$(string.PreferencesMenu_Help)" key="Software\Policies\Tailscale" valueName="PreferencesMenu">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -257,7 +268,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="ExitNodesPicker" class="Machine" displayName="$(string.ExitNodesPicker)" explainText="$(string.ExitNodesPicker_Help)" key="Software\Policies\Tailscale" valueName="ExitNodesPicker">
+    <policy name="ExitNodesPicker" class="Both" displayName="$(string.ExitNodesPicker)" explainText="$(string.ExitNodesPicker_Help)" key="Software\Policies\Tailscale" valueName="ExitNodesPicker">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_22" />
       <enabledValue>
@@ -267,7 +278,7 @@
         <string>hide</string>
       </disabledValue>
     </policy>
-    <policy name="ManagedBy" class="Machine" displayName="$(string.ManagedBy)" explainText="$(string.ManagedBy_Help)" presentation="$(presentation.ManagedBy)" key="Software\Policies\Tailscale">
+    <policy name="ManagedBy" class="Both" displayName="$(string.ManagedBy)" explainText="$(string.ManagedBy_Help)" presentation="$(presentation.ManagedBy)" key="Software\Policies\Tailscale">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_62" />
       <elements>
@@ -276,7 +287,7 @@
         <text id="ManagedBySupportURL" valueName="ManagedByURL" />
       </elements>
     </policy>
-    <policy name="KeyExpirationNotice" class="Machine" displayName="$(string.KeyExpirationNotice)" explainText="$(string.KeyExpirationNotice_Help)" presentation="$(presentation.KeyExpirationNotice)" key="Software\Policies\Tailscale">
+    <policy name="KeyExpirationNotice" class="Both" displayName="$(string.KeyExpirationNotice)" explainText="$(string.KeyExpirationNotice_Help)" presentation="$(presentation.KeyExpirationNotice)" key="Software\Policies\Tailscale">
       <parentCategory ref="UI_Category" />
       <supportedOn ref="SINCE_V1_50" />
       <elements>

envknob/envknob.go

@@ -411,7 +411,7 @@ func TKASkipSignatureCheck() bool { return Bool("TS_UNSAFE_SKIP_NKS_VERIFICATION
 // Kubernetes Operator components.
 func App() string {
 	a := os.Getenv("TS_INTERNAL_APP")
-	if a == kubetypes.AppConnector || a == kubetypes.AppEgressProxy || a == kubetypes.AppIngressProxy || a == kubetypes.AppIngressResource {
+	if a == kubetypes.AppConnector || a == kubetypes.AppEgressProxy || a == kubetypes.AppIngressProxy || a == kubetypes.AppIngressResource || a == kubetypes.AppProxyGroupEgress || a == kubetypes.AppProxyGroupIngress {
 		return a
 	}
 	return ""

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/golangci/golangci-lint v1.57.1
 	github.com/google/go-cmp v0.6.0
-	github.com/google/go-containerregistry v0.18.0
+	github.com/google/go-containerregistry v0.20.2
 	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
@@ -55,7 +55,7 @@ require (
 	github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86
 	github.com/jsimonetti/rtnetlink v1.4.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
-	github.com/klauspost/compress v1.17.4
+	github.com/klauspost/compress v1.17.11
 	github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-isatty v0.0.20
@@ -80,12 +80,12 @@ require (
 	github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
-	github.com/tailscale/mkctr v0.0.0-20240628074852-17ca944da6ba
+	github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10
 	github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7
 	github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4
 	github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1
 	github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6
-	github.com/tailscale/wireguard-go v0.0.0-20240905161824-799c1978fafc
+	github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3
 	github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e
 	github.com/tc-hib/winres v0.2.1
 	github.com/tcnksm/go-httpstat v0.2.0
@@ -100,8 +100,8 @@ require (
 	golang.org/x/mod v0.19.0
 	golang.org/x/net v0.27.0
 	golang.org/x/oauth2 v0.16.0
-	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.22.0
+	golang.org/x/sync v0.9.0
+	golang.org/x/sys v0.27.0
 	golang.org/x/term v0.22.0
 	golang.org/x/time v0.5.0
 	golang.org/x/tools v0.23.0
@@ -125,7 +125,7 @@ require (
 	github.com/Antonboom/testifylint v1.2.0 // indirect
 	github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect
 	github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
@@ -138,7 +138,7 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect
 	github.com/dave/brenda v1.1.0 // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/ghostiam/protogetter v0.3.5 // indirect
@@ -160,10 +160,10 @@ require (
 	github.com/ykadowak/zerologlint v0.1.5 // indirect
 	go-simpler.org/musttag v0.9.0 // indirect
 	go-simpler.org/sloglint v0.5.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel v1.22.0 // indirect
-	go.opentelemetry.io/otel/metric v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
 	go.uber.org/automaxprocs v1.5.3 // indirect
 	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
 )
@@ -220,10 +220,10 @@ require (
 	github.com/daixiang0/gci v0.12.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/denis-tingaikin/go-header v0.5.0 // indirect
-	github.com/docker/cli v25.0.0+incompatible // indirect
+	github.com/docker/cli v27.3.1+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/docker v26.1.4+incompatible // indirect
-	github.com/docker/docker-credential-helpers v0.8.1 // indirect
+	github.com/docker/docker v27.3.1+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.8.2 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/ettle/strcase v0.2.0 // indirect
@@ -322,7 +322,7 @@ require (
 	github.com/nunnatsa/ginkgolinter v0.16.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc6 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -376,7 +376,7 @@ require (
 	github.com/ultraware/funlen v0.1.0 // indirect
 	github.com/ultraware/whitespace v0.1.0 // indirect
 	github.com/uudashr/gocognit v1.1.2 // indirect
-	github.com/vbatts/tar-split v0.11.5 // indirect
+	github.com/vbatts/tar-split v0.11.6 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect

go.sum

@@ -79,8 +79,8 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
-github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
-github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
@@ -277,16 +277,16 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
-github.com/docker/cli v25.0.0+incompatible h1:zaimaQdnX7fYWFqzN88exE9LDEvRslexpFowZBX6GoQ=
-github.com/docker/cli v25.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v27.3.1+incompatible h1:qEGdFBF3Xu6SCvCYhc7CzaQTlBmqDuzxPDpigSyeKQQ=
+github.com/docker/cli v27.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU=
-github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo=
-github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
-github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
-github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI=
+github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
+github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
@@ -490,8 +490,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.18.0 h1:ShE7erKNPqRh5ue6Z9DUOlk04WsnFWPO6YGr3OxnfoQ=
-github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
+github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo=
+github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -627,8 +627,8 @@ github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkHAIKE/contextcheck v1.1.4 h1:B6zAaLhOEEcjvUgIYEqystmnFk1Oemn8bvJhbt0GMb8=
 github.com/kkHAIKE/contextcheck v1.1.4/go.mod h1:1+i/gWqokIa+dm31mqGLZhZJ7Uh44DJGZVmr6QRBNJg=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -749,8 +749,8 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
 github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU=
-github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
@@ -931,8 +931,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw=
 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8=
-github.com/tailscale/mkctr v0.0.0-20240628074852-17ca944da6ba h1:uNo1VCm/xg4alMkIKo8RWTKNx5y1otfVOcKbp+irkL4=
-github.com/tailscale/mkctr v0.0.0-20240628074852-17ca944da6ba/go.mod h1:DxnqIXBplij66U2ZkL688xy07q97qQ83P+TVueLiHq4=
+github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 h1:ZB47BgnHcEHQJODkDubs5ZiNeJxMhcgzefV3lykRwVQ=
+github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10/go.mod h1:iDx/0Rr9VV/KanSUDpJ6I/ROf0sQ7OqljXc/esl0UIA=
 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4ZoF094vE6iYTLDl0qCiKzYXlL6UeWObU=
 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
 github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 h1:Gz0rz40FvFVLTBk/K8UNAenb36EbDSnh+q7Z9ldcC8w=
@@ -941,8 +941,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M=
 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y=
-github.com/tailscale/wireguard-go v0.0.0-20240905161824-799c1978fafc h1:cezaQN9pvKVaw56Ma5qr/G646uKIYP0yQf+OyWN/okc=
-github.com/tailscale/wireguard-go v0.0.0-20240905161824-799c1978fafc/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3 h1:dmoPb3dG27tZgMtrvqfD/LW4w7gA6BSWl8prCPNmkCQ=
+github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA=
@@ -981,8 +981,8 @@ github.com/ultraware/whitespace v0.1.0 h1:O1HKYoh0kIeqE8sFqZf1o0qbORXUCOQFrlaQyZ
 github.com/ultraware/whitespace v0.1.0/go.mod h1:/se4r3beMFNmewJ4Xmz0nMQ941GJt+qmSHGP9emHYe0=
 github.com/uudashr/gocognit v1.1.2 h1:l6BAEKJqQH2UpKAPKdMfZf5kE4W/2xk8pfU1OVLvniI=
 github.com/uudashr/gocognit v1.1.2/go.mod h1:aAVdLURqcanke8h3vg35BC++eseDm66Z7KmchI5et4k=
-github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
-github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
+github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs=
+github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
@@ -1022,20 +1022,20 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
-go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
-go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
-go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
@@ -1176,8 +1176,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1239,8 +1239,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

go.toolchain.rev

@@ -1 +1 @@
-bf15628b759344c6fc7763795a405ba65b8be5d7
+96578f73d04e1a231fa2a495ad3fa97747785bc6

hostinfo/hostinfo.go

@@ -25,7 +25,7 @@ import (
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/cloudenv"
 	"tailscale.com/util/dnsname"
-	"tailscale.com/util/lineread"
+	"tailscale.com/util/lineiter"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
@@ -231,12 +231,12 @@ func desktop() (ret opt.Bool) {
 	}
 
 	seenDesktop := false
-	lineread.File("/proc/net/unix", func(line []byte) error {
+	for lr := range lineiter.File("/proc/net/unix") {
+		line, _ := lr.Value()
 		seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S(" @/tmp/dbus-"))
 		seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S(".X11-unix"))
 		seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S("/wayland-1"))
-		return nil
-	})
+	}
 	ret.Set(seenDesktop)
 
 	// Only cache after a minute - compositors might not have started yet.
@@ -305,21 +305,21 @@ func inContainer() opt.Bool {
 		ret.Set(true)
 		return ret
 	}
-	lineread.File("/proc/1/cgroup", func(line []byte) error {
+	for lr := range lineiter.File("/proc/1/cgroup") {
+		line, _ := lr.Value()
 		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
 			mem.Contains(mem.B(line), mem.S("/lxc/")) {
 			ret.Set(true)
-			return io.EOF // arbitrary non-nil error to stop loop
+			break
 		}
-		return nil
-	})
-	lineread.File("/proc/mounts", func(line []byte) error {
+	}
+	for lr := range lineiter.File("/proc/mounts") {
+		line, _ := lr.Value()
 		if mem.Contains(mem.B(line), mem.S("lxcfs /proc/cpuinfo fuse.lxcfs")) {
 			ret.Set(true)
-			return io.EOF
+			break
 		}
-		return nil
-	})
+	}
 	return ret
 }
 

hostinfo/hostinfo_linux.go

@@ -12,7 +12,7 @@ import (
 
 	"golang.org/x/sys/unix"
 	"tailscale.com/types/ptr"
-	"tailscale.com/util/lineread"
+	"tailscale.com/util/lineiter"
 	"tailscale.com/version/distro"
 )
 
@@ -106,15 +106,18 @@ func linuxVersionMeta() (meta versionMeta) {
 	}
 
 	m := map[string]string{}
-	lineread.File(propFile, func(line []byte) error {
+	for lr := range lineiter.File(propFile) {
+		line, err := lr.Value()
+		if err != nil {
+			break
+		}
 		eq := bytes.IndexByte(line, '=')
 		if eq == -1 {
-			return nil
+			continue
 		}
 		k, v := string(line[:eq]), strings.Trim(string(line[eq+1:]), `"'`)
 		m[k] = v
-		return nil
-	})
+	}
 
 	if v := m["VERSION_CODENAME"]; v != "" {
 		meta.DistroCodeName = v

ipn/backend.go

@@ -73,6 +73,8 @@ const (
 	NotifyInitialOutgoingFiles // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles
 
 	NotifyInitialHealthState // if set, the first Notify message (sent immediately) will contain the current health.State of the client
+
+	NotifyRateLimit // if set, rate limit spammy netmap updates to every few seconds
 )
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend
@@ -100,7 +102,6 @@ type Notify struct {
 	NetMap        *netmap.NetworkMap // if non-nil, the new or current netmap
 	Engine        *EngineStatus      // if non-nil, the new or current wireguard stats
 	BrowseToURL   *string            // if non-nil, UI should open a browser right now
-	BackendLogID  *string            // if non-nil, the public logtail ID used by backend
 
 	// FilesWaiting if non-nil means that files are buffered in
 	// the Tailscale daemon and ready for local transfer to the
@@ -173,9 +174,6 @@ func (n Notify) String() string {
 	if n.BrowseToURL != nil {
 		sb.WriteString("URL=<...> ")
 	}
-	if n.BackendLogID != nil {
-		sb.WriteString("BackendLogID ")
-	}
 	if n.FilesWaiting != nil {
 		sb.WriteString("FilesWaiting ")
 	}

ipn/conf.go

@@ -32,6 +32,8 @@ type ConfigVAlpha struct {
 	AdvertiseRoutes []netip.Prefix `json:",omitempty"`
 	DisableSNAT     opt.Bool       `json:",omitempty"`
 
+	AppConnector *AppConnectorPrefs `json:",omitempty"` // advertise app connector; defaults to false (if nil or explicitly set to false)
+
 	NetfilterMode       *string  `json:",omitempty"` // "on", "off", "nodivert"
 	NoStatefulFiltering opt.Bool `json:",omitempty"`
 
@@ -137,5 +139,9 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) {
 		mp.AutoUpdate = *c.AutoUpdate
 		mp.AutoUpdateSet = AutoUpdatePrefsMask{ApplySet: true, CheckSet: true}
 	}
+	if c.AppConnector != nil {
+		mp.AppConnector = *c.AppConnector
+		mp.AppConnectorSet = true
+	}
 	return mp, nil
 }

ipn/ipnlocal/bus.go

@@ -0,0 +1,160 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package ipnlocal
+
+import (
+	"context"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/tstime"
+)
+
+type rateLimitingBusSender struct {
+	fn              func(*ipn.Notify) (keepGoing bool)
+	lastFlush       time.Time           // last call to fn, or zero value if none
+	interval        time.Duration       // 0 to flush immediately; non-zero to rate limit sends
+	clock           tstime.DefaultClock // non-nil for testing
+	didSendTestHook func()              // non-nil for testing
+
+	// pending, if non-nil, is the pending notification that we
+	// haven't sent yet. We own this memory to mutate.
+	pending *ipn.Notify
+
+	// flushTimer is non-nil if the timer is armed.
+	flushTimer  tstime.TimerController // effectively a *time.Timer
+	flushTimerC <-chan time.Time       // ... said ~Timer's C chan
+}
+
+func (s *rateLimitingBusSender) close() {
+	if s.flushTimer != nil {
+		s.flushTimer.Stop()
+	}
+}
+
+func (s *rateLimitingBusSender) flushChan() <-chan time.Time {
+	return s.flushTimerC
+}
+
+func (s *rateLimitingBusSender) flush() (keepGoing bool) {
+	if n := s.pending; n != nil {
+		s.pending = nil
+		return s.flushNotify(n)
+	}
+	return true
+}
+
+func (s *rateLimitingBusSender) flushNotify(n *ipn.Notify) (keepGoing bool) {
+	s.lastFlush = s.clock.Now()
+	return s.fn(n)
+}
+
+// send conditionally sends n to the underlying fn, possibly rate
+// limiting it, depending on whether s.interval is set, and whether
+// n is a notable notification that the client (typically a GUI) would
+// want to act on (render) immediately.
+//
+// It returns whether the caller should keep looping.
+//
+// The passed-in memory 'n' is owned by the caller and should
+// not be mutated.
+func (s *rateLimitingBusSender) send(n *ipn.Notify) (keepGoing bool) {
+	if s.interval <= 0 {
+		// No rate limiting case.
+		return s.fn(n)
+	}
+	if isNotableNotify(n) {
+		// Notable notifications are always sent immediately.
+		// But first send any boring one that was pending.
+		// TODO(bradfitz): there might be a boring one pending
+		// with a NetMap or Engine field that is redundant
+		// with the new one (n) with NetMap or Engine populated.
+		// We should clear the pending one's NetMap/Engine in
+		// that case. Or really, merge the two, but mergeBoringNotifies
+		// only handles the case of both sides being boring.
+		// So for now, flush both.
+		if !s.flush() {
+			return false
+		}
+		return s.flushNotify(n)
+	}
+	s.pending = mergeBoringNotifies(s.pending, n)
+	d := s.clock.Now().Sub(s.lastFlush)
+	if d > s.interval {
+		return s.flush()
+	}
+	nextFlushIn := s.interval - d
+	if s.flushTimer == nil {
+		s.flushTimer, s.flushTimerC = s.clock.NewTimer(nextFlushIn)
+	} else {
+		s.flushTimer.Reset(nextFlushIn)
+	}
+	return true
+}
+
+func (s *rateLimitingBusSender) Run(ctx context.Context, ch <-chan *ipn.Notify) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case n, ok := <-ch:
+			if !ok {
+				return
+			}
+			if !s.send(n) {
+				return
+			}
+			if f := s.didSendTestHook; f != nil {
+				f()
+			}
+		case <-s.flushChan():
+			if !s.flush() {
+				return
+			}
+		}
+	}
+}
+
+// mergeBoringNotify merges new notify 'src' into possibly-nil 'dst',
+// either mutating 'dst' or allocating a new one if 'dst' is nil,
+// returning the merged result.
+//
+// dst and src must both be "boring" (i.e. not notable per isNotifiableNotify).
+func mergeBoringNotifies(dst, src *ipn.Notify) *ipn.Notify {
+	if dst == nil {
+		dst = &ipn.Notify{Version: src.Version}
+	}
+	if src.NetMap != nil {
+		dst.NetMap = src.NetMap
+	}
+	if src.Engine != nil {
+		dst.Engine = src.Engine
+	}
+	return dst
+}
+
+// isNotableNotify reports whether n is a "notable" notification that
+// should be sent on the IPN bus immediately (e.g. to GUIs) without
+// rate limiting it for a few seconds.
+//
+// It effectively reports whether n contains any field set that's
+// not NetMap or Engine.
+func isNotableNotify(n *ipn.Notify) bool {
+	if n == nil {
+		return false
+	}
+	return n.State != nil ||
+		n.SessionID != "" ||
+		n.BrowseToURL != nil ||
+		n.LocalTCPPort != nil ||
+		n.ClientVersion != nil ||
+		n.Prefs != nil ||
+		n.ErrMessage != nil ||
+		n.LoginFinished != nil ||
+		!n.DriveShares.IsNil() ||
+		n.Health != nil ||
+		len(n.IncomingFiles) > 0 ||
+		len(n.OutgoingFiles) > 0 ||
+		n.FilesWaiting != nil
+}

ipn/ipnlocal/bus_test.go

@@ -0,0 +1,220 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package ipnlocal
+
+import (
+	"context"
+	"reflect"
+	"slices"
+	"testing"
+	"time"
+
+	"tailscale.com/drive"
+	"tailscale.com/ipn"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/views"
+)
+
+func TestIsNotableNotify(t *testing.T) {
+	tests := []struct {
+		name   string
+		notify *ipn.Notify
+		want   bool
+	}{
+		{"nil", nil, false},
+		{"empty", &ipn.Notify{}, false},
+		{"version", &ipn.Notify{Version: "foo"}, false},
+		{"netmap", &ipn.Notify{NetMap: new(netmap.NetworkMap)}, false},
+		{"engine", &ipn.Notify{Engine: new(ipn.EngineStatus)}, false},
+	}
+
+	// Then for all other fields, assume they're notable.
+	// We use reflect to catch fields that might be added in the future without
+	// remembering to update the [isNotableNotify] function.
+	rt := reflect.TypeFor[ipn.Notify]()
+	for i := range rt.NumField() {
+		n := &ipn.Notify{}
+		sf := rt.Field(i)
+		switch sf.Name {
+		case "_", "NetMap", "Engine", "Version":
+			// Already covered above or not applicable.
+			continue
+		case "DriveShares":
+			n.DriveShares = views.SliceOfViews[*drive.Share, drive.ShareView](make([]*drive.Share, 1))
+		default:
+			rf := reflect.ValueOf(n).Elem().Field(i)
+			switch rf.Kind() {
+			case reflect.Pointer:
+				rf.Set(reflect.New(rf.Type().Elem()))
+			case reflect.String:
+				rf.SetString("foo")
+			case reflect.Slice:
+				rf.Set(reflect.MakeSlice(rf.Type(), 1, 1))
+			default:
+				t.Errorf("unhandled field kind %v for %q", rf.Kind(), sf.Name)
+			}
+		}
+
+		tests = append(tests, struct {
+			name   string
+			notify *ipn.Notify
+			want   bool
+		}{
+			name:   "field-" + rt.Field(i).Name,
+			notify: n,
+			want:   true,
+		})
+	}
+
+	for _, tt := range tests {
+		if got := isNotableNotify(tt.notify); got != tt.want {
+			t.Errorf("%v: got %v; want %v", tt.name, got, tt.want)
+		}
+	}
+}
+
+type rateLimitingBusSenderTester struct {
+	tb    testing.TB
+	got   []*ipn.Notify
+	clock *tstest.Clock
+	s     *rateLimitingBusSender
+}
+
+func (st *rateLimitingBusSenderTester) init() {
+	if st.s != nil {
+		return
+	}
+	st.clock = tstest.NewClock(tstest.ClockOpts{
+		Start: time.Unix(1731777537, 0), // time I wrote this test :)
+	})
+	st.s = &rateLimitingBusSender{
+		clock: tstime.DefaultClock{Clock: st.clock},
+		fn: func(n *ipn.Notify) bool {
+			st.got = append(st.got, n)
+			return true
+		},
+	}
+}
+
+func (st *rateLimitingBusSenderTester) send(n *ipn.Notify) {
+	st.tb.Helper()
+	st.init()
+	if !st.s.send(n) {
+		st.tb.Fatal("unexpected send failed")
+	}
+}
+
+func (st *rateLimitingBusSenderTester) advance(d time.Duration) {
+	st.tb.Helper()
+	st.clock.Advance(d)
+	select {
+	case <-st.s.flushChan():
+		if !st.s.flush() {
+			st.tb.Fatal("unexpected flush failed")
+		}
+	default:
+	}
+}
+
+func TestRateLimitingBusSender(t *testing.T) {
+	nm1 := &ipn.Notify{NetMap: new(netmap.NetworkMap)}
+	nm2 := &ipn.Notify{NetMap: new(netmap.NetworkMap)}
+	eng1 := &ipn.Notify{Engine: new(ipn.EngineStatus)}
+	eng2 := &ipn.Notify{Engine: new(ipn.EngineStatus)}
+
+	t.Run("unbuffered", func(t *testing.T) {
+		st := &rateLimitingBusSenderTester{tb: t}
+		st.send(nm1)
+		st.send(nm2)
+		st.send(eng1)
+		st.send(eng2)
+		if !slices.Equal(st.got, []*ipn.Notify{nm1, nm2, eng1, eng2}) {
+			t.Errorf("got %d items; want 4 specific ones, unmodified", len(st.got))
+		}
+	})
+
+	t.Run("buffered", func(t *testing.T) {
+		st := &rateLimitingBusSenderTester{tb: t}
+		st.init()
+		st.s.interval = 1 * time.Second
+		st.send(&ipn.Notify{Version: "initial"})
+		if len(st.got) != 1 {
+			t.Fatalf("got %d items; expected 1 (first to flush immediately)", len(st.got))
+		}
+		st.send(nm1)
+		st.send(nm2)
+		st.send(eng1)
+		st.send(eng2)
+		if len(st.got) != 1 {
+			if len(st.got) != 1 {
+				t.Fatalf("got %d items; expected still just that first 1", len(st.got))
+			}
+		}
+
+		// But moving the clock should flush the rest, collasced into one new one.
+		st.advance(5 * time.Second)
+		if len(st.got) != 2 {
+			t.Fatalf("got %d items; want 2", len(st.got))
+		}
+		gotn := st.got[1]
+		if gotn.NetMap != nm2.NetMap {
+			t.Errorf("got wrong NetMap; got %p", gotn.NetMap)
+		}
+		if gotn.Engine != eng2.Engine {
+			t.Errorf("got wrong Engine; got %p", gotn.Engine)
+		}
+		if t.Failed() {
+			t.Logf("failed Notify was: %v", logger.AsJSON(gotn))
+		}
+	})
+
+	// Test the Run method
+	t.Run("run", func(t *testing.T) {
+		st := &rateLimitingBusSenderTester{tb: t}
+		st.init()
+		st.s.interval = 1 * time.Second
+		st.s.lastFlush = st.clock.Now() // pretend we just flushed
+
+		flushc := make(chan *ipn.Notify, 1)
+		st.s.fn = func(n *ipn.Notify) bool {
+			flushc <- n
+			return true
+		}
+		didSend := make(chan bool, 2)
+		st.s.didSendTestHook = func() { didSend <- true }
+		waitSend := func() {
+			select {
+			case <-didSend:
+			case <-time.After(5 * time.Second):
+				t.Error("timeout waiting for call to send")
+			}
+		}
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		incoming := make(chan *ipn.Notify, 2)
+		go func() {
+			incoming <- nm1
+			waitSend()
+			incoming <- nm2
+			waitSend()
+			st.advance(5 * time.Second)
+			select {
+			case n := <-flushc:
+				if n.NetMap != nm2.NetMap {
+					t.Errorf("got wrong NetMap; got %p", n.NetMap)
+				}
+			case <-time.After(10 * time.Second):
+				t.Error("timeout")
+			}
+			cancel()
+		}()
+
+		st.s.Run(ctx, incoming)
+	})
+}

ipn/ipnlocal/c2n.go

@@ -77,6 +77,9 @@ var c2nHandlers = map[methodAndPath]c2nHandler{
 
 	// Linux netfilter.
 	req("POST /netfilter-kind"): handleC2NSetNetfilterKind,
+
+	// VIP services.
+	req("GET /vip-services"): handleC2NVIPServicesGet,
 }
 
 type c2nHandler func(*LocalBackend, http.ResponseWriter, *http.Request)
@@ -269,6 +272,12 @@ func handleC2NSetNetfilterKind(b *LocalBackend, w http.ResponseWriter, r *http.R
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func handleC2NVIPServicesGet(b *LocalBackend, w http.ResponseWriter, r *http.Request) {
+	b.logf("c2n: GET /vip-services received")
+
+	json.NewEncoder(w).Encode(b.VIPServices())
+}
+
 func handleC2NUpdateGet(b *LocalBackend, w http.ResponseWriter, r *http.Request) {
 	b.logf("c2n: GET /update received")
 
@@ -332,12 +341,10 @@ func handleC2NPostureIdentityGet(b *LocalBackend, w http.ResponseWriter, r *http
 	}
 
 	if choice.ShouldEnable(b.Prefs().PostureChecking()) {
-		sns, err := posture.GetSerialNumbers(b.logf)
+		res.SerialNumbers, err = posture.GetSerialNumbers(b.logf)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
+			b.logf("c2n: GetSerialNumbers returned error: %v", err)
 		}
-		res.SerialNumbers = sns
 
 		// TODO(tailscale/corp#21371, 2024-07-10): once this has landed in a stable release
 		// and looks good in client metrics, remove this parameter and always report MAC
@@ -352,6 +359,8 @@ func handleC2NPostureIdentityGet(b *LocalBackend, w http.ResponseWriter, r *http
 		res.PostureDisabled = true
 	}
 
+	b.logf("c2n: posture identity disabled=%v reported %d serials %d hwaddrs", res.PostureDisabled, len(res.SerialNumbers), len(res.IfaceHardwareAddrs))
+
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(res)
 }

ipn/ipnlocal/drive.go

@@ -354,9 +354,8 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem
 
 				// Check that the peer is allowed to share with us.
 				addresses := peer.Addresses()
-				for i := range addresses.Len() {
-					addr := addresses.At(i)
-					capsMap := b.PeerCaps(addr.Addr())
+				for _, p := range addresses.All() {
+					capsMap := b.PeerCaps(p.Addr())
 					if capsMap.HasCapability(tailcfg.PeerCapabilityTaildriveSharer) {
 						return true
 					}

ipn/ipnlocal/local.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -86,7 +87,6 @@ import (
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/empty"
 	"tailscale.com/types/key"
-	"tailscale.com/types/lazy"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
@@ -105,6 +105,7 @@ import (
 	"tailscale.com/util/rands"
 	"tailscale.com/util/set"
 	"tailscale.com/util/syspolicy"
+	"tailscale.com/util/syspolicy/rsop"
 	"tailscale.com/util/systemd"
 	"tailscale.com/util/testenv"
 	"tailscale.com/util/uniq"
@@ -177,27 +178,28 @@ type watchSession struct {
 // state machine generates events back out to zero or more components.
 type LocalBackend struct {
 	// Elements that are thread-safe or constant after construction.
-	ctx                   context.Context    // canceled by Close
-	ctxCancel             context.CancelFunc // cancels ctx
-	logf                  logger.Logf        // general logging
-	keyLogf               logger.Logf        // for printing list of peers on change
-	statsLogf             logger.Logf        // for printing peers stats on change
-	sys                   *tsd.System
-	health                *health.Tracker // always non-nil
-	metrics               metrics
-	e                     wgengine.Engine // non-nil; TODO(bradfitz): remove; use sys
-	store                 ipn.StateStore  // non-nil; TODO(bradfitz): remove; use sys
-	dialer                *tsdial.Dialer  // non-nil; TODO(bradfitz): remove; use sys
-	pushDeviceToken       syncs.AtomicValue[string]
-	backendLogID          logid.PublicID
-	unregisterNetMon      func()
-	unregisterHealthWatch func()
-	portpoll              *portlist.Poller // may be nil
-	portpollOnce          sync.Once        // guards starting readPoller
-	varRoot               string           // or empty if SetVarRoot never called
-	logFlushFunc          func()           // or nil if SetLogFlusher wasn't called
-	em                    *expiryManager   // non-nil
-	sshAtomicBool         atomic.Bool
+	ctx                      context.Context    // canceled by Close
+	ctxCancel                context.CancelFunc // cancels ctx
+	logf                     logger.Logf        // general logging
+	keyLogf                  logger.Logf        // for printing list of peers on change
+	statsLogf                logger.Logf        // for printing peers stats on change
+	sys                      *tsd.System
+	health                   *health.Tracker // always non-nil
+	metrics                  metrics
+	e                        wgengine.Engine // non-nil; TODO(bradfitz): remove; use sys
+	store                    ipn.StateStore  // non-nil; TODO(bradfitz): remove; use sys
+	dialer                   *tsdial.Dialer  // non-nil; TODO(bradfitz): remove; use sys
+	pushDeviceToken          syncs.AtomicValue[string]
+	backendLogID             logid.PublicID
+	unregisterNetMon         func()
+	unregisterHealthWatch    func()
+	unregisterSysPolicyWatch func()
+	portpoll                 *portlist.Poller // may be nil
+	portpollOnce             sync.Once        // guards starting readPoller
+	varRoot                  string           // or empty if SetVarRoot never called
+	logFlushFunc             func()           // or nil if SetLogFlusher wasn't called
+	em                       *expiryManager   // non-nil
+	sshAtomicBool            atomic.Bool
 	// webClientAtomicBool controls whether the web client is running. This should
 	// be true unless the disable-web-client node attribute has been set.
 	webClientAtomicBool atomic.Bool
@@ -353,6 +355,12 @@ type LocalBackend struct {
 	// avoid unnecessary churn between multiple equally-good options.
 	lastSuggestedExitNode tailcfg.StableNodeID
 
+	// allowedSuggestedExitNodes is a set of exit nodes permitted by the most recent
+	// [syspolicy.AllowedSuggestedExitNodes] value. The allowedSuggestedExitNodesMu
+	// mutex guards access to this set.
+	allowedSuggestedExitNodesMu sync.Mutex
+	allowedSuggestedExitNodes   set.Set[tailcfg.StableNodeID]
+
 	// refreshAutoExitNode indicates if the exit node should be recomputed when the next netcheck report is available.
 	refreshAutoExitNode bool
 
@@ -399,11 +407,6 @@ type metrics struct {
 	// approvedRoutes is a metric that reports the number of network routes served by the local node and approved
 	// by the control server.
 	approvedRoutes *usermetric.Gauge
-
-	// primaryRoutes is a metric that reports the number of primary network routes served by the local node.
-	// A route being a primary route implies that the route is currently served by this node, and not by another
-	// subnet router in a high availability configuration.
-	primaryRoutes *usermetric.Gauge
 }
 
 // clientGen is a func that creates a control plane client.
@@ -414,7 +417,7 @@ type clientGen func(controlclient.Options) (controlclient.Client, error)
 // but is not actually running.
 //
 // If dialer is nil, a new one is made.
-func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, loginFlags controlclient.LoginFlags) (*LocalBackend, error) {
+func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, loginFlags controlclient.LoginFlags) (_ *LocalBackend, err error) {
 	e := sys.Engine.Get()
 	store := sys.StateStore.Get()
 	dialer := sys.Dialer.Get()
@@ -454,8 +457,6 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
 			"tailscaled_advertised_routes", "Number of advertised network routes (e.g. by a subnet router)"),
 		approvedRoutes: sys.UserMetricsRegistry().NewGauge(
 			"tailscaled_approved_routes", "Number of approved network routes (e.g. by a subnet router)"),
-		primaryRoutes: sys.UserMetricsRegistry().NewGauge(
-			"tailscaled_primary_routes", "Number of network routes for which this node is a primary router (in high availability configuration)"),
 	}
 
 	b := &LocalBackend{
@@ -486,11 +487,20 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
 	mConn.SetNetInfoCallback(b.setNetInfo)
 
 	if sys.InitialConfig != nil {
-		if err := b.setConfigLocked(sys.InitialConfig); err != nil {
+		if err := b.initPrefsFromConfig(sys.InitialConfig); err != nil {
 			return nil, err
 		}
 	}
 
+	if b.unregisterSysPolicyWatch, err = b.registerSysPolicyWatch(); err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			b.unregisterSysPolicyWatch()
+		}
+	}()
+
 	netMon := sys.NetMon.Get()
 	b.sockstatLogger, err = sockstatlog.NewLogger(logpolicy.LogsDir(logf), logf, logID, netMon, sys.HealthTracker())
 	if err != nil {
@@ -719,8 +729,8 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 // It returns (false, nil) if not running in declarative mode, (true, nil) on
 // success, or (false, error) on failure.
 func (b *LocalBackend) ReloadConfig() (ok bool, err error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
+	unlock := b.lockAndGetUnlock()
+	defer unlock()
 	if b.conf == nil {
 		return false, nil
 	}
@@ -728,18 +738,21 @@ func (b *LocalBackend) ReloadConfig() (ok bool, err error) {
 	if err != nil {
 		return false, err
 	}
-	if err := b.setConfigLocked(conf); err != nil {
+	if err := b.setConfigLockedOnEntry(conf, unlock); err != nil {
 		return false, fmt.Errorf("error setting config: %w", err)
 	}
 
 	return true, nil
 }
 
-func (b *LocalBackend) setConfigLocked(conf *conffile.Config) error {
-
-	// TODO(irbekrm): notify the relevant components to consume any prefs
-	// updates. Currently only initial configfile settings are applied
-	// immediately.
+// initPrefsFromConfig initializes the backend's prefs from the provided config.
+// This should only be called once, at startup. For updates at runtime, use
+// [LocalBackend.setConfigLocked].
+func (b *LocalBackend) initPrefsFromConfig(conf *conffile.Config) error {
+	// TODO(maisem,bradfitz): combine this with setConfigLocked. This is called
+	// before anything is running, so there's no need to lock and we don't
+	// update any subsystems. At runtime, we both need to lock and update
+	// subsystems with the new prefs.
 	p := b.pm.CurrentPrefs().AsStruct()
 	mp, err := conf.Parsed.ToPrefs()
 	if err != nil {
@@ -749,13 +762,14 @@ func (b *LocalBackend) setConfigLocked(conf *conffile.Config) error {
 	if err := b.pm.SetPrefs(p.View(), ipn.NetworkProfile{}); err != nil {
 		return err
 	}
+	b.setStaticEndpointsFromConfigLocked(conf)
+	b.conf = conf
+	return nil
+}
 
-	defer func() {
-		b.conf = conf
-	}()
-
+func (b *LocalBackend) setStaticEndpointsFromConfigLocked(conf *conffile.Config) {
 	if conf.Parsed.StaticEndpoints == nil && (b.conf == nil || b.conf.Parsed.StaticEndpoints == nil) {
-		return nil
+		return
 	}
 
 	// Ensure that magicsock conn has the up to date static wireguard
@@ -769,6 +783,22 @@ func (b *LocalBackend) setConfigLocked(conf *conffile.Config) error {
 			ms.SetStaticEndpoints(views.SliceOf(conf.Parsed.StaticEndpoints))
 		}
 	}
+}
+
+// setConfigLockedOnEntry uses the provided config to update the backend's prefs
+// and other state.
+func (b *LocalBackend) setConfigLockedOnEntry(conf *conffile.Config, unlock unlockOnce) error {
+	defer unlock()
+	p := b.pm.CurrentPrefs().AsStruct()
+	mp, err := conf.Parsed.ToPrefs()
+	if err != nil {
+		return fmt.Errorf("error parsing config to prefs: %w", err)
+	}
+	p.ApplyEdits(&mp)
+	b.setStaticEndpointsFromConfigLocked(conf)
+	b.setPrefsLockedOnEntry(p, unlock)
+
+	b.conf = conf
 	return nil
 }
 
@@ -787,6 +817,19 @@ func (b *LocalBackend) pauseOrResumeControlClientLocked() {
 	b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || (!networkUp && !testenv.InTest() && !assumeNetworkUpdateForTest()))
 }
 
+// DisconnectControl shuts down control client. This can be run before node shutdown to force control to consider this ndoe
+// inactive. This can be used to ensure that nodes that are HA subnet router or app connector replicas are shutting
+// down, clients switch over to other replicas whilst the existing connections are kept alive for some period of time.
+func (b *LocalBackend) DisconnectControl() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	cc := b.resetControlClientLocked()
+	if cc == nil {
+		return
+	}
+	cc.Shutdown()
+}
+
 // captivePortalDetectionInterval is the duration to wait in an unhealthy state with connectivity broken
 // before running captive portal detection.
 const captivePortalDetectionInterval = 2 * time.Second
@@ -954,6 +997,7 @@ func (b *LocalBackend) Shutdown() {
 
 	b.unregisterNetMon()
 	b.unregisterHealthWatch()
+	b.unregisterSysPolicyWatch()
 	if cc != nil {
 		cc.Shutdown()
 	}
@@ -1462,10 +1506,10 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control
 			b.logf("SetControlClientStatus failed to select auto exit node: %v", err)
 		}
 	}
-	if setExitNodeID(prefs, curNetMap, b.lastSuggestedExitNode) {
+	if applySysPolicy(prefs, b.lastSuggestedExitNode) {
 		prefsChanged = true
 	}
-	if applySysPolicy(prefs) {
+	if setExitNodeID(prefs, curNetMap) {
 		prefsChanged = true
 	}
 
@@ -1631,12 +1675,37 @@ var preferencePolicies = []preferencePolicyInfo{
 
 // applySysPolicy overwrites configured preferences with policies that may be
 // configured by the system administrator in an OS-specific way.
-func applySysPolicy(prefs *ipn.Prefs) (anyChange bool) {
+func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID) (anyChange bool) {
 	if controlURL, err := syspolicy.GetString(syspolicy.ControlURL, prefs.ControlURL); err == nil && prefs.ControlURL != controlURL {
 		prefs.ControlURL = controlURL
 		anyChange = true
 	}
 
+	if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" {
+		exitNodeID := tailcfg.StableNodeID(exitNodeIDStr)
+		if shouldAutoExitNode() && lastSuggestedExitNode != "" {
+			exitNodeID = lastSuggestedExitNode
+		}
+		// Note: when exitNodeIDStr == "auto" && lastSuggestedExitNode == "",
+		// then exitNodeID is now "auto" which will never match a peer's node ID.
+		// When there is no a peer matching the node ID, traffic will blackhole,
+		// preventing accidental non-exit-node usage when a policy is in effect that requires an exit node.
+		if prefs.ExitNodeID != exitNodeID || prefs.ExitNodeIP.IsValid() {
+			anyChange = true
+		}
+		prefs.ExitNodeID = exitNodeID
+		prefs.ExitNodeIP = netip.Addr{}
+	} else if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" {
+		exitNodeIP, err := netip.ParseAddr(exitNodeIPStr)
+		if exitNodeIP.IsValid() && err == nil {
+			if prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP {
+				anyChange = true
+			}
+			prefs.ExitNodeID = ""
+			prefs.ExitNodeIP = exitNodeIP
+		}
+	}
+
 	for _, opt := range preferencePolicies {
 		if po, err := syspolicy.GetPreferenceOption(opt.key); err == nil {
 			curVal := opt.get(prefs.View())
@@ -1651,6 +1720,54 @@ func applySysPolicy(prefs *ipn.Prefs) (anyChange bool) {
 	return anyChange
 }
 
+// registerSysPolicyWatch subscribes to syspolicy change notifications
+// and immediately applies the effective syspolicy settings to the current profile.
+func (b *LocalBackend) registerSysPolicyWatch() (unregister func(), err error) {
+	if unregister, err = syspolicy.RegisterChangeCallback(b.sysPolicyChanged); err != nil {
+		return nil, fmt.Errorf("syspolicy: LocalBacked failed to register policy change callback: %v", err)
+	}
+	if prefs, anyChange := b.applySysPolicy(); anyChange {
+		b.logf("syspolicy: changed initial profile prefs: %v", prefs.Pretty())
+	}
+	b.refreshAllowedSuggestions()
+	return unregister, nil
+}
+
+// applySysPolicy overwrites the current profile's preferences with policies
+// that may be configured by the system administrator in an OS-specific way.
+//
+// b.mu must not be held.
+func (b *LocalBackend) applySysPolicy() (_ ipn.PrefsView, anyChange bool) {
+	unlock := b.lockAndGetUnlock()
+	prefs := b.pm.CurrentPrefs().AsStruct()
+	if !applySysPolicy(prefs, b.lastSuggestedExitNode) {
+		unlock.UnlockEarly()
+		return prefs.View(), false
+	}
+	return b.setPrefsLockedOnEntry(prefs, unlock), true
+}
+
+// sysPolicyChanged is a callback triggered by syspolicy when it detects
+// a change in one or more syspolicy settings.
+func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) {
+	if policy.HasChanged(syspolicy.AllowedSuggestedExitNodes) {
+		b.refreshAllowedSuggestions()
+		// Re-evaluate exit node suggestion now that the policy setting has changed.
+		b.mu.Lock()
+		_, err := b.suggestExitNodeLocked(nil)
+		b.mu.Unlock()
+		if err != nil && !errors.Is(err, ErrNoPreferredDERP) {
+			b.logf("failed to select auto exit node: %v", err)
+		}
+		// If [syspolicy.ExitNodeID] is set to `auto:any`, the suggested exit node ID
+		// will be used when [applySysPolicy] updates the current profile's prefs.
+	}
+
+	if prefs, anyChange := b.applySysPolicy(); anyChange {
+		b.logf("syspolicy: changed profile prefs: %v", prefs.Pretty())
+	}
+}
+
 var _ controlclient.NetmapDeltaUpdater = (*LocalBackend)(nil)
 
 // UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater.
@@ -1743,30 +1860,7 @@ func (b *LocalBackend) updateNetmapDeltaLocked(muts []netmap.NodeMutation) (hand
 
 // setExitNodeID updates prefs to reference an exit node by ID, rather
 // than by IP. It returns whether prefs was mutated.
-func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap, lastSuggestedExitNode tailcfg.StableNodeID) (prefsChanged bool) {
-	if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" {
-		exitNodeID := tailcfg.StableNodeID(exitNodeIDStr)
-		if shouldAutoExitNode() && lastSuggestedExitNode != "" {
-			exitNodeID = lastSuggestedExitNode
-		}
-		// Note: when exitNodeIDStr == "auto" && lastSuggestedExitNode == "", then exitNodeID is now "auto" which will never match a peer's node ID.
-		// When there is no a peer matching the node ID, traffic will blackhole, preventing accidental non-exit-node usage when a policy is in effect that requires an exit node.
-		changed := prefs.ExitNodeID != exitNodeID || prefs.ExitNodeIP.IsValid()
-		prefs.ExitNodeID = exitNodeID
-		prefs.ExitNodeIP = netip.Addr{}
-		return changed
-	}
-
-	oldExitNodeID := prefs.ExitNodeID
-	if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" {
-		exitNodeIP, err := netip.ParseAddr(exitNodeIPStr)
-		if exitNodeIP.IsValid() && err == nil {
-			prefsChanged = prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP
-			prefs.ExitNodeID = ""
-			prefs.ExitNodeIP = exitNodeIP
-		}
-	}
-
+func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) {
 	if nm == nil {
 		// No netmap, can't resolve anything.
 		return false
@@ -1784,9 +1878,9 @@ func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap, lastSuggestedExitNod
 		prefsChanged = true
 	}
 
+	oldExitNodeID := prefs.ExitNodeID
 	for _, peer := range nm.Peers {
-		for i := range peer.Addresses().Len() {
-			addr := peer.Addresses().At(i)
+		for _, addr := range peer.Addresses().All() {
 			if !addr.IsSingleIP() || addr.Addr() != prefs.ExitNodeIP {
 				continue
 			}
@@ -1794,7 +1888,7 @@ func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap, lastSuggestedExitNod
 			// reference it directly for next time.
 			prefs.ExitNodeID = peer.StableID()
 			prefs.ExitNodeIP = netip.Addr{}
-			return oldExitNodeID != prefs.ExitNodeID
+			return prefsChanged || oldExitNodeID != prefs.ExitNodeID
 		}
 	}
 
@@ -2131,10 +2225,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 
 	blid := b.backendLogID.String()
 	b.logf("Backend: logs: be:%v fe:%v", blid, opts.FrontendLogID)
-	b.sendToLocked(ipn.Notify{
-		BackendLogID: &blid,
-		Prefs:        &prefs,
-	}, allClients)
+	b.sendToLocked(ipn.Notify{Prefs: &prefs}, allClients)
 
 	if !loggedOut && (b.hasNodeKeyLocked() || confWantRunning) {
 		// If we know that we're either logged in or meant to be
@@ -2757,20 +2848,17 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 		go b.pollRequestEngineStatus(ctx)
 	}
 
-	// TODO(marwan-at-work): check err
 	// TODO(marwan-at-work): streaming background logs?
 	defer b.DeleteForegroundSession(sessionID)
 
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case n := <-ch:
-			if !fn(n) {
-				return
-			}
-		}
+	sender := &rateLimitingBusSender{fn: fn}
+	defer sender.close()
+
+	if mask&ipn.NotifyRateLimit != 0 {
+		sender.interval = 3 * time.Second
 	}
+
+	sender.Run(ctx, ch)
 }
 
 // pollRequestEngineStatus calls b.e.RequestStatus every 2 seconds until ctx
@@ -3652,11 +3740,16 @@ func updateExitNodeUsageWarning(p ipn.PrefsView, state *netmon.State, healthTrac
 }
 
 func (b *LocalBackend) checkExitNodePrefsLocked(p *ipn.Prefs) error {
+	tryingToUseExitNode := p.ExitNodeIP.IsValid() || p.ExitNodeID != ""
+	if !tryingToUseExitNode {
+		return nil
+	}
+
 	if err := featureknob.CanUseExitNode(); err != nil {
 		return err
 	}
 
-	if (p.ExitNodeIP.IsValid() || p.ExitNodeID != "") && p.AdvertisesExitNode() {
+	if p.AdvertisesExitNode() {
 		return errors.New("Cannot advertise an exit node and use an exit node at the same time.")
 	}
 	return nil
@@ -3824,12 +3917,12 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce)
 	if oldp.Valid() {
 		newp.Persist = oldp.Persist().AsStruct() // caller isn't allowed to override this
 	}
-	// setExitNodeID returns whether it updated b.prefs, but
-	// everything in this function treats b.prefs as completely new
-	// anyway. No-op if no exit node resolution is needed.
-	setExitNodeID(newp, netMap, b.lastSuggestedExitNode)
-	// applySysPolicy does likewise so we can also ignore its return value.
-	applySysPolicy(newp)
+	// applySysPolicyToPrefsLocked returns whether it updated newp,
+	// but everything in this function treats b.prefs as completely new
+	// anyway, so its return value can be ignored here.
+	applySysPolicy(newp, b.lastSuggestedExitNode)
+	// setExitNodeID does likewise. No-op if no exit node resolution is needed.
+	setExitNodeID(newp, netMap)
 	// We do this to avoid holding the lock while doing everything else.
 
 	oldHi := b.hostinfo
@@ -3866,10 +3959,14 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce)
 	}
 
 	prefs := newp.View()
-	if err := b.pm.SetPrefs(prefs, ipn.NetworkProfile{
-		MagicDNSName: b.netMap.MagicDNSSuffix(),
-		DomainName:   b.netMap.DomainName(),
-	}); err != nil {
+	np := b.pm.CurrentProfile().NetworkProfile
+	if netMap != nil {
+		np = ipn.NetworkProfile{
+			MagicDNSName: b.netMap.MagicDNSSuffix(),
+			DomainName:   b.netMap.DomainName(),
+		}
+	}
+	if err := b.pm.SetPrefs(prefs, np); err != nil {
 		b.logf("failed to save new controlclient state: %v", err)
 	}
 
@@ -4863,6 +4960,14 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip
 	}
 	hi.SSH_HostKeys = sshHostKeys
 
+	services := vipServicesFromPrefs(prefs)
+	if len(services) > 0 {
+		buf, _ := json.Marshal(services)
+		hi.ServicesHash = fmt.Sprintf("%02x", sha256.Sum256(buf))
+	} else {
+		hi.ServicesHash = ""
+	}
+
 	// The Hostinfo.WantIngress field tells control whether this node wants to
 	// be wired up for ingress connections. If harmless if it's accidentally
 	// true; the actual policy is controlled in tailscaled by ServeConfig. But
@@ -4971,8 +5076,8 @@ func (b *LocalBackend) enterStateLockedOnEntry(newState ipn.State, unlock unlock
 	case ipn.Running:
 		var addrStrs []string
 		addrs := netMap.GetAddresses()
-		for i := range addrs.Len() {
-			addrStrs = append(addrStrs, addrs.At(i).Addr().String())
+		for _, p := range addrs.All() {
+			addrStrs = append(addrStrs, p.Addr().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrStrs, " "))
 	case ipn.NoState:
@@ -5477,7 +5582,6 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		// If there is no netmap, the client is going into a "turned off"
 		// state so reset the metrics.
 		b.metrics.approvedRoutes.Set(0)
-		b.metrics.primaryRoutes.Set(0)
 		return
 	}
 
@@ -5506,7 +5610,6 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 			}
 		}
 		b.metrics.approvedRoutes.Set(approved)
-		b.metrics.primaryRoutes.Set(float64(tsaddr.WithoutExitRoute(nm.SelfNode.PrimaryRoutes()).Len()))
 	}
 	for _, p := range nm.Peers {
 		addNode(p)
@@ -6065,8 +6168,7 @@ func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
 
 func peerAPIPorts(peer tailcfg.NodeView) (p4, p6 uint16) {
 	svcs := peer.Hostinfo().Services()
-	for i := range svcs.Len() {
-		s := svcs.At(i)
+	for _, s := range svcs.All() {
 		switch s.Proto {
 		case tailcfg.PeerAPI4:
 			p4 = s.Port
@@ -6098,8 +6200,7 @@ func peerAPIBase(nm *netmap.NetworkMap, peer tailcfg.NodeView) string {
 
 	var have4, have6 bool
 	addrs := nm.GetAddresses()
-	for i := range addrs.Len() {
-		a := addrs.At(i)
+	for _, a := range addrs.All() {
 		if !a.IsSingleIP() {
 			continue
 		}
@@ -6121,10 +6222,9 @@ func peerAPIBase(nm *netmap.NetworkMap, peer tailcfg.NodeView) string {
 }
 
 func nodeIP(n tailcfg.NodeView, pred func(netip.Addr) bool) netip.Addr {
-	for i := range n.Addresses().Len() {
-		a := n.Addresses().At(i)
-		if a.IsSingleIP() && pred(a.Addr()) {
-			return a.Addr()
+	for _, pfx := range n.Addresses().All() {
+		if pfx.IsSingleIP() && pred(pfx.Addr()) {
+			return pfx.Addr()
 		}
 	}
 	return netip.Addr{}
@@ -6354,8 +6454,8 @@ func peerCanProxyDNS(p tailcfg.NodeView) bool {
 	// If p.Cap is not populated (e.g. older control server), then do the old
 	// thing of searching through services.
 	services := p.Hostinfo().Services()
-	for i := range services.Len() {
-		if s := services.At(i); s.Proto == tailcfg.PeerAPIDNS && s.Port >= 1 {
+	for _, s := range services.All() {
+		if s.Proto == tailcfg.PeerAPIDNS && s.Port >= 1 {
 			return true
 		}
 	}
@@ -7121,7 +7221,7 @@ func (b *LocalBackend) suggestExitNodeLocked(netMap *netmap.NetworkMap) (respons
 	lastReport := b.MagicConn().GetLastNetcheckReport(b.ctx)
 	prevSuggestion := b.lastSuggestedExitNode
 
-	res, err := suggestExitNode(lastReport, netMap, prevSuggestion, randomRegion, randomNode, getAllowedSuggestions())
+	res, err := suggestExitNode(lastReport, netMap, prevSuggestion, randomRegion, randomNode, b.getAllowedSuggestions())
 	if err != nil {
 		return res, err
 	}
@@ -7135,6 +7235,22 @@ func (b *LocalBackend) SuggestExitNode() (response apitype.ExitNodeSuggestionRes
 	return b.suggestExitNodeLocked(nil)
 }
 
+// getAllowedSuggestions returns a set of exit nodes permitted by the most recent
+// [syspolicy.AllowedSuggestedExitNodes] value. Callers must not mutate the returned set.
+func (b *LocalBackend) getAllowedSuggestions() set.Set[tailcfg.StableNodeID] {
+	b.allowedSuggestedExitNodesMu.Lock()
+	defer b.allowedSuggestedExitNodesMu.Unlock()
+	return b.allowedSuggestedExitNodes
+}
+
+// refreshAllowedSuggestions rebuilds the set of permitted exit nodes
+// from the current [syspolicy.AllowedSuggestedExitNodes] value.
+func (b *LocalBackend) refreshAllowedSuggestions() {
+	b.allowedSuggestedExitNodesMu.Lock()
+	defer b.allowedSuggestedExitNodesMu.Unlock()
+	b.allowedSuggestedExitNodes = fillAllowedSuggestions()
+}
+
 // selectRegionFunc returns a DERP region from the slice of candidate regions.
 // The value is returned, not the slice index.
 type selectRegionFunc func(views.Slice[int]) int
@@ -7144,8 +7260,6 @@ type selectRegionFunc func(views.Slice[int]) int
 // choice.
 type selectNodeFunc func(nodes views.Slice[tailcfg.NodeView], last tailcfg.StableNodeID) tailcfg.NodeView
 
-var getAllowedSuggestions = lazy.SyncFunc(fillAllowedSuggestions)
-
 func fillAllowedSuggestions() set.Set[tailcfg.StableNodeID] {
 	nodes, err := syspolicy.GetStringArray(syspolicy.AllowedSuggestedExitNodes, nil)
 	if err != nil {
@@ -7465,3 +7579,42 @@ func maybeUsernameOf(actor ipnauth.Actor) string {
 	}
 	return username
 }
+
+// VIPServices returns the list of tailnet services that this node
+// is serving as a destination for.
+// The returned memory is owned by the caller.
+func (b *LocalBackend) VIPServices() []*tailcfg.VIPService {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return vipServicesFromPrefs(b.pm.CurrentPrefs())
+}
+
+func vipServicesFromPrefs(prefs ipn.PrefsView) []*tailcfg.VIPService {
+	// keyed by service name
+	var services map[string]*tailcfg.VIPService
+
+	// TODO(naman): this envknob will be replaced with service-specific port
+	// information once we start storing that.
+	var allPortsServices []string
+	if env := envknob.String("TS_DEBUG_ALLPORTS_SERVICES"); env != "" {
+		allPortsServices = strings.Split(env, ",")
+	}
+
+	for _, s := range allPortsServices {
+		mak.Set(&services, s, &tailcfg.VIPService{
+			Name:  s,
+			Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+		})
+	}
+
+	for _, s := range prefs.AdvertiseServices().AsSlice() {
+		if services == nil || services[s] == nil {
+			mak.Set(&services, s, &tailcfg.VIPService{
+				Name: s,
+			})
+		}
+		services[s].Active = true
+	}
+
+	return slices.Collect(maps.Values(services))
+}

ipn/ipnlocal/local_test.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"net/netip"
 	"os"
+	"path/filepath"
 	"reflect"
 	"slices"
 	"strings"
@@ -29,9 +30,11 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive"
 	"tailscale.com/drive/driveimpl"
+	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/conffile"
 	"tailscale.com/ipn/ipnauth"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/netcheck"
@@ -54,6 +57,8 @@ import (
 	"tailscale.com/util/must"
 	"tailscale.com/util/set"
 	"tailscale.com/util/syspolicy"
+	"tailscale.com/util/syspolicy/setting"
+	"tailscale.com/util/syspolicy/source"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/wgcfg"
@@ -430,20 +435,30 @@ func (panicOnUseTransport) RoundTrip(*http.Request) (*http.Response, error) {
 }
 
 func newTestLocalBackend(t testing.TB) *LocalBackend {
+	return newTestLocalBackendWithSys(t, new(tsd.System))
+}
+
+// newTestLocalBackendWithSys creates a new LocalBackend with the given tsd.System.
+// If the state store or engine are not set in sys, they will be set to a new
+// in-memory store and fake userspace engine, respectively.
+func newTestLocalBackendWithSys(t testing.TB, sys *tsd.System) *LocalBackend {
 	var logf logger.Logf = logger.Discard
-	sys := new(tsd.System)
-	store := new(mem.Store)
-	sys.Set(store)
-	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
-	if err != nil {
-		t.Fatalf("NewFakeUserspaceEngine: %v", err)
+	if _, ok := sys.StateStore.GetOK(); !ok {
+		sys.Set(new(mem.Store))
+	}
+	if _, ok := sys.Engine.GetOK(); !ok {
+		eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+		if err != nil {
+			t.Fatalf("NewFakeUserspaceEngine: %v", err)
+		}
+		t.Cleanup(eng.Close)
+		sys.Set(eng)
 	}
-	t.Cleanup(eng.Close)
-	sys.Set(eng)
 	lb, err := NewLocalBackend(logf, logid.PublicID{}, sys, 0)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(lb.Shutdown)
 	return lb
 }
 
@@ -1559,94 +1574,6 @@ func dnsResponse(domain, address string) []byte {
 	return must.Get(b.Finish())
 }
 
-type errorSyspolicyHandler struct {
-	t         *testing.T
-	err       error
-	key       syspolicy.Key
-	allowKeys map[syspolicy.Key]*string
-}
-
-func (h *errorSyspolicyHandler) ReadString(key string) (string, error) {
-	sk := syspolicy.Key(key)
-	if _, ok := h.allowKeys[sk]; !ok {
-		h.t.Errorf("ReadString: %q is not in list of permitted keys", h.key)
-	}
-	if sk == h.key {
-		return "", h.err
-	}
-	return "", syspolicy.ErrNoSuchKey
-}
-
-func (h *errorSyspolicyHandler) ReadUInt64(key string) (uint64, error) {
-	h.t.Errorf("ReadUInt64(%q) unexpectedly called", key)
-	return 0, syspolicy.ErrNoSuchKey
-}
-
-func (h *errorSyspolicyHandler) ReadBoolean(key string) (bool, error) {
-	h.t.Errorf("ReadBoolean(%q) unexpectedly called", key)
-	return false, syspolicy.ErrNoSuchKey
-}
-
-func (h *errorSyspolicyHandler) ReadStringArray(key string) ([]string, error) {
-	h.t.Errorf("ReadStringArray(%q) unexpectedly called", key)
-	return nil, syspolicy.ErrNoSuchKey
-}
-
-type mockSyspolicyHandler struct {
-	t *testing.T
-	// stringPolicies is the collection of policies that we expect to see
-	// queried by the current test. If the policy is expected but unset, then
-	// use nil, otherwise use a string equal to the policy's desired value.
-	stringPolicies map[syspolicy.Key]*string
-	// stringArrayPolicies is the collection of policies that we expected to see
-	// queries by the current test, that return policy string arrays.
-	stringArrayPolicies map[syspolicy.Key][]string
-	// failUnknownPolicies is set if policies other than those in stringPolicies
-	// (uint64 or bool policies are not supported by mockSyspolicyHandler yet)
-	// should be considered a test failure if they are queried.
-	failUnknownPolicies bool
-}
-
-func (h *mockSyspolicyHandler) ReadString(key string) (string, error) {
-	if s, ok := h.stringPolicies[syspolicy.Key(key)]; ok {
-		if s == nil {
-			return "", syspolicy.ErrNoSuchKey
-		}
-		return *s, nil
-	}
-	if h.failUnknownPolicies {
-		h.t.Errorf("ReadString(%q) unexpectedly called", key)
-	}
-	return "", syspolicy.ErrNoSuchKey
-}
-
-func (h *mockSyspolicyHandler) ReadUInt64(key string) (uint64, error) {
-	if h.failUnknownPolicies {
-		h.t.Errorf("ReadUInt64(%q) unexpectedly called", key)
-	}
-	return 0, syspolicy.ErrNoSuchKey
-}
-
-func (h *mockSyspolicyHandler) ReadBoolean(key string) (bool, error) {
-	if h.failUnknownPolicies {
-		h.t.Errorf("ReadBoolean(%q) unexpectedly called", key)
-	}
-	return false, syspolicy.ErrNoSuchKey
-}
-
-func (h *mockSyspolicyHandler) ReadStringArray(key string) ([]string, error) {
-	if h.failUnknownPolicies {
-		h.t.Errorf("ReadStringArray(%q) unexpectedly called", key)
-	}
-	if s, ok := h.stringArrayPolicies[syspolicy.Key(key)]; ok {
-		if s == nil {
-			return []string{}, syspolicy.ErrNoSuchKey
-		}
-		return s, nil
-	}
-	return nil, syspolicy.ErrNoSuchKey
-}
-
 func TestSetExitNodeIDPolicy(t *testing.T) {
 	pfx := netip.MustParsePrefix
 	tests := []struct {
@@ -1856,23 +1783,21 @@ func TestSetExitNodeIDPolicy(t *testing.T) {
 		},
 	}
 
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			b := newTestBackend(t)
-			msh := &mockSyspolicyHandler{
-				t: t,
-				stringPolicies: map[syspolicy.Key]*string{
-					syspolicy.ExitNodeID: nil,
-					syspolicy.ExitNodeIP: nil,
-				},
-			}
+
+			policyStore := source.NewTestStore(t)
 			if test.exitNodeIDKey {
-				msh.stringPolicies[syspolicy.ExitNodeID] = &test.exitNodeID
+				policyStore.SetStrings(source.TestSettingOf(syspolicy.ExitNodeID, test.exitNodeID))
 			}
 			if test.exitNodeIPKey {
-				msh.stringPolicies[syspolicy.ExitNodeIP] = &test.exitNodeIP
+				policyStore.SetStrings(source.TestSettingOf(syspolicy.ExitNodeIP, test.exitNodeIP))
 			}
-			syspolicy.SetHandlerForTest(t, msh)
+			syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
+
 			if test.nm == nil {
 				test.nm = new(netmap.NetworkMap)
 			}
@@ -1884,7 +1809,16 @@ func TestSetExitNodeIDPolicy(t *testing.T) {
 			b.netMap = test.nm
 			b.pm = pm
 			b.lastSuggestedExitNode = test.lastSuggestedExitNode
-			changed := setExitNodeID(b.pm.prefs.AsStruct(), test.nm, tailcfg.StableNodeID(test.lastSuggestedExitNode))
+
+			prefs := b.pm.prefs.AsStruct()
+			if changed := applySysPolicy(prefs, test.lastSuggestedExitNode) || setExitNodeID(prefs, test.nm); changed != test.prefsChanged {
+				t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed)
+			}
+
+			// Both [LocalBackend.SetPrefsForTest] and [LocalBackend.EditPrefs]
+			// apply syspolicy settings to the current profile's preferences. Therefore,
+			// we pass the current, unmodified preferences and expect the effective
+			// preferences to change.
 			b.SetPrefsForTest(pm.CurrentPrefs().AsStruct())
 
 			if got := b.pm.prefs.ExitNodeID(); got != tailcfg.StableNodeID(test.exitNodeIDWant) {
@@ -1897,10 +1831,6 @@ func TestSetExitNodeIDPolicy(t *testing.T) {
 			} else if got.String() != test.exitNodeIPWant {
 				t.Errorf("got %v want %v", got, test.exitNodeIPWant)
 			}
-
-			if changed != test.prefsChanged {
-				t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed)
-			}
 		})
 	}
 }
@@ -1994,13 +1924,13 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) {
 			report:                    report,
 		},
 	}
-	msh := &mockSyspolicyHandler{
-		t: t,
-		stringPolicies: map[syspolicy.Key]*string{
-			syspolicy.ExitNodeID: ptr.To("auto:any"),
-		},
-	}
-	syspolicy.SetHandlerForTest(t, msh)
+
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+	policyStore := source.NewTestStoreOf(t, source.TestSettingOf(
+		syspolicy.ExitNodeID, "auto:any",
+	))
+	syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			b := newTestLocalBackend(t)
@@ -2049,13 +1979,11 @@ func TestAutoExitNodeSetNetInfoCallback(t *testing.T) {
 	}
 	cc = newClient(t, opts)
 	b.cc = cc
-	msh := &mockSyspolicyHandler{
-		t: t,
-		stringPolicies: map[syspolicy.Key]*string{
-			syspolicy.ExitNodeID: ptr.To("auto:any"),
-		},
-	}
-	syspolicy.SetHandlerForTest(t, msh)
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+	policyStore := source.NewTestStoreOf(t, source.TestSettingOf(
+		syspolicy.ExitNodeID, "auto:any",
+	))
+	syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
 	peer1 := makePeer(1, withCap(26), withDERP(3), withSuggest(), withExitRoutes())
 	peer2 := makePeer(2, withCap(26), withDERP(2), withSuggest(), withExitRoutes())
 	selfNode := tailcfg.Node{
@@ -2160,13 +2088,11 @@ func TestSetControlClientStatusAutoExitNode(t *testing.T) {
 		DERPMap: derpMap,
 	}
 	b := newTestLocalBackend(t)
-	msh := &mockSyspolicyHandler{
-		t: t,
-		stringPolicies: map[syspolicy.Key]*string{
-			syspolicy.ExitNodeID: ptr.To("auto:any"),
-		},
-	}
-	syspolicy.SetHandlerForTest(t, msh)
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+	policyStore := source.NewTestStoreOf(t, source.TestSettingOf(
+		syspolicy.ExitNodeID, "auto:any",
+	))
+	syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
 	b.netMap = nm
 	b.lastSuggestedExitNode = peer1.StableID()
 	b.sys.MagicSock.Get().SetLastNetcheckReportForTest(b.ctx, report)
@@ -2400,22 +2326,21 @@ func TestApplySysPolicy(t *testing.T) {
 		},
 	}
 
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			msh := &mockSyspolicyHandler{
-				t:              t,
-				stringPolicies: make(map[syspolicy.Key]*string, len(tt.stringPolicies)),
-			}
+			settings := make([]source.TestSetting[string], 0, len(tt.stringPolicies))
 			for p, v := range tt.stringPolicies {
-				v := v // construct a unique pointer for each policy value
-				msh.stringPolicies[p] = &v
+				settings = append(settings, source.TestSettingOf(p, v))
 			}
-			syspolicy.SetHandlerForTest(t, msh)
+			policyStore := source.NewTestStoreOf(t, settings...)
+			syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
 
 			t.Run("unit", func(t *testing.T) {
 				prefs := tt.prefs.Clone()
 
-				gotAnyChange := applySysPolicy(prefs)
+				gotAnyChange := applySysPolicy(prefs, "")
 
 				if gotAnyChange && prefs.Equals(&tt.prefs) {
 					t.Errorf("anyChange but prefs is unchanged: %v", prefs.Pretty())
@@ -2546,40 +2471,24 @@ func TestPreferencePolicyInfo(t *testing.T) {
 		},
 	}
 
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			for _, pp := range preferencePolicies {
 				t.Run(string(pp.key), func(t *testing.T) {
-					var h syspolicy.Handler
-
-					allPolicies := make(map[syspolicy.Key]*string, len(preferencePolicies)+1)
-					allPolicies[syspolicy.ControlURL] = nil
-					for _, pp := range preferencePolicies {
-						allPolicies[pp.key] = nil
+					s := source.TestSetting[string]{
+						Key:   pp.key,
+						Error: tt.policyError,
+						Value: tt.policyValue,
 					}
-
-					if tt.policyError != nil {
-						h = &errorSyspolicyHandler{
-							t:         t,
-							err:       tt.policyError,
-							key:       pp.key,
-							allowKeys: allPolicies,
-						}
-					} else {
-						msh := &mockSyspolicyHandler{
-							t:                   t,
-							stringPolicies:      allPolicies,
-							failUnknownPolicies: true,
-						}
-						msh.stringPolicies[pp.key] = &tt.policyValue
-						h = msh
-					}
-					syspolicy.SetHandlerForTest(t, h)
+					policyStore := source.NewTestStoreOf(t, s)
+					syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
 
 					prefs := defaultPrefs.AsStruct()
 					pp.set(prefs, tt.initialValue)
 
-					gotAnyChange := applySysPolicy(prefs)
+					gotAnyChange := applySysPolicy(prefs, "")
 
 					if gotAnyChange != tt.wantChange {
 						t.Errorf("anyChange=%v, want %v", gotAnyChange, tt.wantChange)
@@ -3142,12 +3051,10 @@ func deterministicNodeForTest(t testing.TB, want views.Slice[tailcfg.StableNodeI
 		var ret tailcfg.NodeView
 
 		gotIDs := make([]tailcfg.StableNodeID, got.Len())
-		for i := range got.Len() {
-			nv := got.At(i)
+		for i, nv := range got.All() {
 			if !nv.Valid() {
 				t.Fatalf("invalid node at index %v", i)
 			}
-
 			gotIDs[i] = nv.StableID()
 			if nv.StableID() == use {
 				ret = nv
@@ -3825,15 +3732,16 @@ func TestShouldAutoExitNode(t *testing.T) {
 			expectedBool:          false,
 		},
 	}
+
+	syspolicy.RegisterWellKnownSettingsForTest(t)
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			msh := &mockSyspolicyHandler{
-				t: t,
-				stringPolicies: map[syspolicy.Key]*string{
-					syspolicy.ExitNodeID: ptr.To(tt.exitNodeIDPolicyValue),
-				},
-			}
-			syspolicy.SetHandlerForTest(t, msh)
+			policyStore := source.NewTestStoreOf(t, source.TestSettingOf(
+				syspolicy.ExitNodeID, tt.exitNodeIDPolicyValue,
+			))
+			syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
+
 			got := shouldAutoExitNode()
 			if got != tt.expectedBool {
 				t.Fatalf("expected %v got %v for %v policy value", tt.expectedBool, got, tt.exitNodeIDPolicyValue)
@@ -3971,17 +3879,13 @@ func TestFillAllowedSuggestions(t *testing.T) {
 			want:        []tailcfg.StableNodeID{"ABC", "def", "gHiJ"},
 		},
 	}
+	syspolicy.RegisterWellKnownSettingsForTest(t)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			mh := mockSyspolicyHandler{
-				t: t,
-			}
-			if tt.allowPolicy != nil {
-				mh.stringArrayPolicies = map[syspolicy.Key][]string{
-					syspolicy.AllowedSuggestedExitNodes: tt.allowPolicy,
-				}
-			}
-			syspolicy.SetHandlerForTest(t, &mh)
+			policyStore := source.NewTestStoreOf(t, source.TestSettingOf(
+				syspolicy.AllowedSuggestedExitNodes, tt.allowPolicy,
+			))
+			syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore)
 
 			got := fillAllowedSuggestions()
 			if got == nil {
@@ -4214,6 +4118,7 @@ func newLocalBackendWithTestControl(t *testing.T, enableLogging bool, newControl
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(b.Shutdown)
 	b.DisablePortMapperForTest()
 
 	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {
@@ -4538,3 +4443,245 @@ func TestLoginNotifications(t *testing.T) {
 		})
 	}
 }
+
+// TestConfigFileReload tests that the LocalBackend reloads its configuration
+// when the configuration file changes.
+func TestConfigFileReload(t *testing.T) {
+	cfg1 := `{"Hostname": "foo", "Version": "alpha0"}`
+	f := filepath.Join(t.TempDir(), "cfg")
+	must.Do(os.WriteFile(f, []byte(cfg1), 0600))
+	sys := new(tsd.System)
+	sys.InitialConfig = must.Get(conffile.Load(f))
+	lb := newTestLocalBackendWithSys(t, sys)
+	must.Do(lb.Start(ipn.Options{}))
+
+	lb.mu.Lock()
+	hn := lb.hostinfo.Hostname
+	lb.mu.Unlock()
+	if hn != "foo" {
+		t.Fatalf("got %q; want %q", hn, "foo")
+	}
+
+	cfg2 := `{"Hostname": "bar", "Version": "alpha0"}`
+	must.Do(os.WriteFile(f, []byte(cfg2), 0600))
+	if !must.Get(lb.ReloadConfig()) {
+		t.Fatal("reload failed")
+	}
+
+	lb.mu.Lock()
+	hn = lb.hostinfo.Hostname
+	lb.mu.Unlock()
+	if hn != "bar" {
+		t.Fatalf("got %q; want %q", hn, "bar")
+	}
+}
+
+func TestGetVIPServices(t *testing.T) {
+	tests := []struct {
+		name       string
+		advertised []string
+		mapped     []string
+		want       []*tailcfg.VIPService
+	}{
+		{
+			"advertised-only",
+			[]string{"svc:abc", "svc:def"},
+			[]string{},
+			[]*tailcfg.VIPService{
+				{
+					Name:   "svc:abc",
+					Active: true,
+				},
+				{
+					Name:   "svc:def",
+					Active: true,
+				},
+			},
+		},
+		{
+			"mapped-only",
+			[]string{},
+			[]string{"svc:abc"},
+			[]*tailcfg.VIPService{
+				{
+					Name:  "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+				},
+			},
+		},
+		{
+			"mapped-and-advertised",
+			[]string{"svc:abc"},
+			[]string{"svc:abc"},
+			[]*tailcfg.VIPService{
+				{
+					Name:   "svc:abc",
+					Active: true,
+					Ports:  []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+				},
+			},
+		},
+		{
+			"mapped-and-advertised-separately",
+			[]string{"svc:def"},
+			[]string{"svc:abc"},
+			[]*tailcfg.VIPService{
+				{
+					Name:  "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+				},
+				{
+					Name:   "svc:def",
+					Active: true,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			envknob.Setenv("TS_DEBUG_ALLPORTS_SERVICES", strings.Join(tt.mapped, ","))
+			prefs := &ipn.Prefs{
+				AdvertiseServices: tt.advertised,
+			}
+			got := vipServicesFromPrefs(prefs.View())
+			slices.SortFunc(got, func(a, b *tailcfg.VIPService) int {
+				return strings.Compare(a.Name, b.Name)
+			})
+			if !reflect.DeepEqual(tt.want, got) {
+				t.Logf("want:")
+				for _, s := range tt.want {
+					t.Logf("%+v", s)
+				}
+				t.Logf("got:")
+				for _, s := range got {
+					t.Logf("%+v", s)
+				}
+				t.Fail()
+				return
+			}
+		})
+	}
+}
+
+func TestUpdatePrefsOnSysPolicyChange(t *testing.T) {
+	const enableLogging = false
+
+	type fieldChange struct {
+		name string
+		want any
+	}
+
+	wantPrefsChanges := func(want ...fieldChange) *wantedNotification {
+		return &wantedNotification{
+			name: "Prefs",
+			cond: func(t testing.TB, actor ipnauth.Actor, n *ipn.Notify) bool {
+				if n.Prefs != nil {
+					prefs := reflect.Indirect(reflect.ValueOf(n.Prefs.AsStruct()))
+					for _, f := range want {
+						got := prefs.FieldByName(f.name).Interface()
+						if !reflect.DeepEqual(got, f.want) {
+							t.Errorf("%v: got %v; want %v", f.name, got, f.want)
+						}
+					}
+				}
+				return n.Prefs != nil
+			},
+		}
+	}
+
+	unexpectedPrefsChange := func(t testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool {
+		if n.Prefs != nil {
+			t.Errorf("Unexpected Prefs: %v", n.Prefs.Pretty())
+			return true
+		}
+		return false
+	}
+
+	tests := []struct {
+		name           string
+		initialPrefs   *ipn.Prefs
+		stringSettings []source.TestSetting[string]
+		want           *wantedNotification
+	}{
+		{
+			name:           "ShieldsUp/True",
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.EnableIncomingConnections, "never")},
+			want:           wantPrefsChanges(fieldChange{"ShieldsUp", true}),
+		},
+		{
+			name:           "ShieldsUp/False",
+			initialPrefs:   &ipn.Prefs{ShieldsUp: true},
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.EnableIncomingConnections, "always")},
+			want:           wantPrefsChanges(fieldChange{"ShieldsUp", false}),
+		},
+		{
+			name:           "ExitNodeID",
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.ExitNodeID, "foo")},
+			want:           wantPrefsChanges(fieldChange{"ExitNodeID", tailcfg.StableNodeID("foo")}),
+		},
+		{
+			name:           "EnableRunExitNode",
+			stringSettings: []source.TestSetting[string]{source.TestSettingOf(syspolicy.EnableRunExitNode, "always")},
+			want:           wantPrefsChanges(fieldChange{"AdvertiseRoutes", []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()}}),
+		},
+		{
+			name: "Multiple",
+			initialPrefs: &ipn.Prefs{
+				ExitNodeAllowLANAccess: true,
+			},
+			stringSettings: []source.TestSetting[string]{
+				source.TestSettingOf(syspolicy.EnableServerMode, "always"),
+				source.TestSettingOf(syspolicy.ExitNodeAllowLANAccess, "never"),
+				source.TestSettingOf(syspolicy.ExitNodeIP, "127.0.0.1"),
+			},
+			want: wantPrefsChanges(
+				fieldChange{"ForceDaemon", true},
+				fieldChange{"ExitNodeAllowLANAccess", false},
+				fieldChange{"ExitNodeIP", netip.MustParseAddr("127.0.0.1")},
+			),
+		},
+		{
+			name: "NoChange",
+			initialPrefs: &ipn.Prefs{
+				CorpDNS:         true,
+				ExitNodeID:      "foo",
+				AdvertiseRoutes: []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()},
+			},
+			stringSettings: []source.TestSetting[string]{
+				source.TestSettingOf(syspolicy.EnableTailscaleDNS, "always"),
+				source.TestSettingOf(syspolicy.ExitNodeID, "foo"),
+				source.TestSettingOf(syspolicy.EnableRunExitNode, "always"),
+			},
+			want: nil, // syspolicy settings match the preferences; no change notification is expected.
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			syspolicy.RegisterWellKnownSettingsForTest(t)
+			store := source.NewTestStoreOf[string](t)
+			syspolicy.MustRegisterStoreForTest(t, "TestSource", setting.DeviceScope, store)
+
+			lb := newLocalBackendWithTestControl(t, enableLogging, func(tb testing.TB, opts controlclient.Options) controlclient.Client {
+				return newClient(tb, opts)
+			})
+			if tt.initialPrefs != nil {
+				lb.SetPrefsForTest(tt.initialPrefs)
+			}
+			if err := lb.Start(ipn.Options{}); err != nil {
+				t.Fatalf("(*LocalBackend).Start(): %v", err)
+			}
+
+			nw := newNotificationWatcher(t, lb, &ipnauth.TestActor{})
+			if tt.want != nil {
+				nw.watch(0, []wantedNotification{*tt.want})
+			} else {
+				nw.watch(0, nil, unexpectedPrefsChange)
+			}
+
+			store.SetStrings(tt.stringSettings...)
+
+			nw.check()
+		})
+	}
+}

ipn/ipnlocal/network-lock.go

@@ -430,8 +430,7 @@ func (b *LocalBackend) tkaBootstrapFromGenesisLocked(g tkatype.MarshaledAUM, per
 		}
 		bootstrapStateID := fmt.Sprintf("%d:%d", genesis.State.StateID1, genesis.State.StateID2)
 
-		for i := range persist.DisallowedTKAStateIDs().Len() {
-			stateID := persist.DisallowedTKAStateIDs().At(i)
+		for _, stateID := range persist.DisallowedTKAStateIDs().All() {
 			if stateID == bootstrapStateID {
 				return fmt.Errorf("TKA with stateID of %q is disallowed on this node", stateID)
 			}
@@ -572,8 +571,7 @@ func tkaStateFromPeer(p tailcfg.NodeView) ipnstate.TKAPeer {
 		TailscaleIPs: make([]netip.Addr, 0, p.Addresses().Len()),
 		NodeKey:      p.Key(),
 	}
-	for i := range p.Addresses().Len() {
-		addr := p.Addresses().At(i)
+	for _, addr := range p.Addresses().All() {
 		if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.Addr()) {
 			fp.TailscaleIPs = append(fp.TailscaleIPs, addr.Addr())
 		}

ipn/ipnlocal/serve.go

@@ -242,8 +242,7 @@ func (b *LocalBackend) updateServeTCPPortNetMapAddrListenersLocked(ports []uint1
 	}
 
 	addrs := nm.GetAddresses()
-	for i := range addrs.Len() {
-		a := addrs.At(i)
+	for _, a := range addrs.All() {
 		for _, p := range ports {
 			addrPort := netip.AddrPortFrom(a.Addr(), p)
 			if _, ok := b.serveListeners[addrPort]; ok {

ipn/ipnlocal/ssh.go

@@ -27,7 +27,7 @@ import (
 	"github.com/tailscale/golang-x-crypto/ssh"
 	"go4.org/mem"
 	"tailscale.com/tailcfg"
-	"tailscale.com/util/lineread"
+	"tailscale.com/util/lineiter"
 	"tailscale.com/util/mak"
 )
 
@@ -80,30 +80,32 @@ func (b *LocalBackend) getSSHUsernames(req *tailcfg.C2NSSHUsernamesRequest) (*ta
 		if err != nil {
 			return nil, err
 		}
-		lineread.Reader(bytes.NewReader(out), func(line []byte) error {
+		for line := range lineiter.Bytes(out) {
 			line = bytes.TrimSpace(line)
 			if len(line) == 0 || line[0] == '_' {
-				return nil
+				continue
 			}
 			add(string(line))
-			return nil
-		})
+		}
 	default:
-		lineread.File("/etc/passwd", func(line []byte) error {
+		for lr := range lineiter.File("/etc/passwd") {
+			line, err := lr.Value()
+			if err != nil {
+				break
+			}
 			line = bytes.TrimSpace(line)
 			if len(line) == 0 || line[0] == '#' || line[0] == '_' {
-				return nil
+				continue
 			}
 			if mem.HasSuffix(mem.B(line), mem.S("/nologin")) ||
 				mem.HasSuffix(mem.B(line), mem.S("/false")) {
-				return nil
+				continue
 			}
 			colon := bytes.IndexByte(line, ':')
 			if colon != -1 {
 				add(string(line[:colon]))
 			}
-			return nil
-		})
+		}
 	}
 	return res, nil
 }

ipn/ipnlocal/state_test.go

@@ -309,6 +309,7 @@ func TestStateMachine(t *testing.T) {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(b.Shutdown)
 	b.DisablePortMapperForTest()
 
 	var cc, previousCC *mockControl
@@ -942,6 +943,7 @@ func TestEditPrefsHasNoKeys(t *testing.T) {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(b.Shutdown)
 	b.hostinfo = &tailcfg.Hostinfo{OS: "testos"}
 	b.pm.SetPrefs((&ipn.Prefs{
 		Persist: &persist.Persist{
@@ -1023,6 +1025,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	sys.Set(eng)
 	b, err := NewLocalBackend(logf, logid.PublicID{}, sys, 0)
 	c.Assert(err, qt.IsNil)
+	t.Cleanup(b.Shutdown)
 
 	var cc *mockControl
 	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {

ipn/ipnlocal/web_client.go

@@ -121,8 +121,8 @@ func (b *LocalBackend) updateWebClientListenersLocked() {
 	}
 
 	addrs := b.netMap.GetAddresses()
-	for i := range addrs.Len() {
-		addrPort := netip.AddrPortFrom(addrs.At(i).Addr(), webClientPort)
+	for _, pfx := range addrs.All() {
+		addrPort := netip.AddrPortFrom(pfx.Addr(), webClientPort)
 		if _, ok := b.webClientListeners[addrPort]; ok {
 			continue // already listening
 		}

ipn/localapi/localapi.go

@@ -62,7 +62,8 @@ import (
 	"tailscale.com/util/osdiag"
 	"tailscale.com/util/progresstracking"
 	"tailscale.com/util/rands"
-	"tailscale.com/util/testenv"
+	"tailscale.com/util/syspolicy/rsop"
+	"tailscale.com/util/syspolicy/setting"
 	"tailscale.com/version"
 	"tailscale.com/wgengine/magicsock"
 )
@@ -77,6 +78,7 @@ var handler = map[string]localAPIHandler{
 	"cert/":     (*Handler).serveCert,
 	"file-put/": (*Handler).serveFilePut,
 	"files/":    (*Handler).serveFiles,
+	"policy/":   (*Handler).servePolicy,
 	"profiles/": (*Handler).serveProfiles,
 
 	// The other /localapi/v0/NAME handlers are exact matches and contain only NAME
@@ -98,6 +100,7 @@ var handler = map[string]localAPIHandler{
 	"derpmap":                     (*Handler).serveDERPMap,
 	"dev-set-state-store":         (*Handler).serveDevSetStateStore,
 	"dial":                        (*Handler).serveDial,
+	"disconnect-control":          (*Handler).disconnectControl,
 	"dns-osconfig":                (*Handler).serveDNSOSConfig,
 	"dns-query":                   (*Handler).serveDNSQuery,
 	"drive/fileserver-address":    (*Handler).serveDriveServerAddr,
@@ -570,15 +573,9 @@ func (h *Handler) serveMetrics(w http.ResponseWriter, r *http.Request) {
 	clientmetric.WritePrometheusExpositionFormat(w)
 }
 
-// TODO(kradalby): Remove this once we have landed on a final set of
-// metrics to export to clients and consider the metrics stable.
-var debugUsermetricsEndpoint = envknob.RegisterBool("TS_DEBUG_USER_METRICS")
-
+// serveUserMetrics returns user-facing metrics in Prometheus text
+// exposition format.
 func (h *Handler) serveUserMetrics(w http.ResponseWriter, r *http.Request) {
-	if !testenv.InTest() && !debugUsermetricsEndpoint() {
-		http.Error(w, "usermetrics debug flag not enabled", http.StatusForbidden)
-		return
-	}
 	h.b.UserMetricsRegistry().Handler(w, r)
 }
 
@@ -956,6 +953,22 @@ func (h *Handler) servePprof(w http.ResponseWriter, r *http.Request) {
 	servePprofFunc(w, r)
 }
 
+// disconnectControl is the handler for local API /disconnect-control endpoint that shuts down control client, so that
+// node no longer communicates with control. Doing this makes control consider this node inactive. This can be used
+// before shutting down a replica of HA subnet  router or app connector deployments to ensure that control tells the
+// peers to switch over to another replica whilst still maintaining th existing peer connections.
+func (h *Handler) disconnectControl(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitWrite {
+		http.Error(w, "access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != httpm.POST {
+		http.Error(w, "use POST", http.StatusMethodNotAllowed)
+		return
+	}
+	h.b.DisconnectControl()
+}
+
 func (h *Handler) reloadConfig(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitWrite {
 		http.Error(w, "access denied", http.StatusForbidden)
@@ -1339,6 +1352,53 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) {
 	e.Encode(prefs)
 }
 
+func (h *Handler) servePolicy(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "policy access denied", http.StatusForbidden)
+		return
+	}
+
+	suffix, ok := strings.CutPrefix(r.URL.EscapedPath(), "/localapi/v0/policy/")
+	if !ok {
+		http.Error(w, "misconfigured", http.StatusInternalServerError)
+		return
+	}
+
+	var scope setting.PolicyScope
+	if suffix == "" {
+		scope = setting.DefaultScope()
+	} else if err := scope.UnmarshalText([]byte(suffix)); err != nil {
+		http.Error(w, fmt.Sprintf("%q is not a valid scope", suffix), http.StatusBadRequest)
+		return
+	}
+
+	policy, err := rsop.PolicyFor(scope)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	var effectivePolicy *setting.Snapshot
+	switch r.Method {
+	case "GET":
+		effectivePolicy = policy.Get()
+	case "POST":
+		effectivePolicy, err = policy.Reload()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	default:
+		http.Error(w, "unsupported method", http.StatusMethodNotAllowed)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "\t")
+	e.Encode(effectivePolicy)
+}
+
 type resJSON struct {
 	Error string `json:",omitempty"`
 }

ipn/localapi/localapi_test.go

@@ -349,6 +349,7 @@ func newTestLocalBackend(t testing.TB) *ipnlocal.LocalBackend {
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
+	t.Cleanup(lb.Shutdown)
 	return lb
 }