wgengine: return explicit lo0 for loopback addrs on sandboxed macOS

fixes tailscale/tailscale#TODO

The source address link selection on sandboxed macOS doesn't deal
with loopback addresses correctly.  This adds an explicit check to ensure
we return the loopback interface for loopback addresses instead of the
default empty interface.

Specifcially, this allows the dns resolver to route queries to a loopback
IP which is a common tactic for local DNS proxies.

.github/workflows/govulncheck.yml

@@ -30,7 +30,7 @@ jobs:
           token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
           payload: |
             {
-              "channel": "C05PXRM304B",
+              "channel": "C08FGKZCQTW",
               "blocks": [
                 {
                   "type": "section",

client/tailscale/devices.go

@@ -79,6 +79,13 @@ type Device struct {
 	// Tailscale have attempted to collect this from the device but it has not
 	// opted in, PostureIdentity will have Disabled=true.
 	PostureIdentity *DevicePostureIdentity `json:"postureIdentity"`
+
+	// TailnetLockKey is the tailnet lock public key of the node as a hex string.
+	TailnetLockKey string `json:"tailnetLockKey,omitempty"`
+
+	// TailnetLockErr indicates an issue with the tailnet lock node-key signature
+	// on this device. This field is only populated when tailnet lock is enabled.
+	TailnetLockErr string `json:"tailnetLockError,omitempty"`
 }
 
 type DevicePostureIdentity struct {

client/web/web.go

@@ -335,7 +335,8 @@ func (s *Server) requireTailscaleIP(w http.ResponseWriter, r *http.Request) (han
 		ipv6ServiceHost = "[" + tsaddr.TailscaleServiceIPv6String + "]"
 	)
 	// allow requests on quad-100 (or ipv6 equivalent)
-	if r.Host == ipv4ServiceHost || r.Host == ipv6ServiceHost {
+	host := strings.TrimSuffix(r.Host, ":80")
+	if host == ipv4ServiceHost || host == ipv6ServiceHost {
 		return false
 	}
 

client/web/web_test.go

@@ -1177,6 +1177,16 @@ func TestRequireTailscaleIP(t *testing.T) {
 			target:      "http://[fd7a:115c:a1e0::53]/",
 			wantHandled: false,
 		},
+		{
+			name:        "quad-100:80",
+			target:      "http://100.100.100.100:80/",
+			wantHandled: false,
+		},
+		{
+			name:        "ipv6-service-addr:80",
+			target:      "http://[fd7a:115c:a1e0::53]:80/",
+			wantHandled: false,
+		},
 	}
 
 	for _, tt := range tests {

clientupdate/clientupdate.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 
 	"tailscale.com/hostinfo"
+	"tailscale.com/types/lazy"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 	"tailscale.com/version"
@@ -249,9 +250,13 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 	return nil, false
 }
 
+var canAutoUpdateCache lazy.SyncValue[bool]
+
 // CanAutoUpdate reports whether auto-updating via the clientupdate package
 // is supported for the current os/distro.
-func CanAutoUpdate() bool {
+func CanAutoUpdate() bool { return canAutoUpdateCache.Get(canAutoUpdateUncached) }
+
+func canAutoUpdateUncached() bool {
 	if version.IsMacSysExt() {
 		// Macsys uses Sparkle for auto-updates, which doesn't have an update
 		// function in this package.

cmd/containerboot/certs.go

@@ -0,0 +1,147 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/util/goroutines"
+	"tailscale.com/util/mak"
+)
+
+// certManager is responsible for issuing certificates for known domains and for
+// maintaining a loop that re-attempts issuance daily.
+// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for
+// HA Ingress HTTPS endpoints ('write' replicas).
+type certManager struct {
+	lc      localClient
+	tracker goroutines.Tracker // tracks running goroutines
+	mu      sync.Mutex         // guards the following
+	// certLoops contains a map of DNS names, for which we currently need to
+	// manage certs to cancel functions that allow stopping a goroutine when
+	// we no longer need to manage certs for the DNS name.
+	certLoops map[string]context.CancelFunc
+}
+
+// ensureCertLoops ensures that, for all currently managed Service HTTPS
+// endpoints, there is a cert loop responsible for issuing and ensuring the
+// renewal of the TLS certs.
+// ServeConfig must not be nil.
+func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error {
+	if sc == nil {
+		return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
+	}
+	currentDomains := make(map[string]bool)
+	const httpsPort = "443"
+	for _, service := range sc.Services {
+		for hostPort := range service.Web {
+			domain, port, err := net.SplitHostPort(string(hostPort))
+			if err != nil {
+				return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
+			}
+			if port != httpsPort { // HA Ingress' HTTP endpoint
+				continue
+			}
+			currentDomains[domain] = true
+		}
+	}
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	for domain := range currentDomains {
+		if _, exists := cm.certLoops[domain]; !exists {
+			cancelCtx, cancel := context.WithCancel(ctx)
+			mak.Set(&cm.certLoops, domain, cancel)
+			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
+		}
+	}
+
+	// Stop goroutines for domain names that are no longer in the config.
+	for domain, cancel := range cm.certLoops {
+		if !currentDomains[domain] {
+			cancel()
+			delete(cm.certLoops, domain)
+		}
+	}
+	return nil
+}
+
+// runCertLoop:
+// - calls localAPI certificate endpoint to ensure that certs are issued for the
+// given domain name
+// - calls localAPI certificate endpoint daily to ensure that certs are renewed
+// - if certificate issuance failed retries after an exponential backoff period
+// starting at 1 minute and capped at 24 hours. Reset the backoff once issuance succeeds.
+// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get
+// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write'
+// replica does not get any HTTPS requests.
+// https://letsencrypt.org/docs/integration-guide/#retrying-failures
+func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
+	const (
+		normalInterval   = 24 * time.Hour  // regular renewal check
+		initialRetry     = 1 * time.Minute // initial backoff after a failure
+		maxRetryInterval = 24 * time.Hour  // max backoff period
+	)
+	timer := time.NewTimer(0) // fire off timer immediately
+	defer timer.Stop()
+	retryCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			// We call the certificate endpoint, but don't do anything
+			// with the returned certs here.
+			// The call to the certificate endpoint will ensure that
+			// certs are issued/renewed as needed and stored in the
+			// relevant state store. For example, for HA Ingress
+			// 'write' replica, the cert and key will be stored in a
+			// Kubernetes Secret named after the domain for which we
+			// are issuing.
+			// Note that renewals triggered by the call to the
+			// certificates endpoint here and by renewal check
+			// triggered during a call to node's HTTPS endpoint
+			// share the same state/renewal lock mechanism, so we
+			// should not run into redundant issuances during
+			// concurrent renewal checks.
+			// TODO(irbekrm): maybe it is worth adding a new
+			// issuance endpoint that explicitly only triggers
+			// issuance and stores certs in the relevant store, but
+			// does not return certs to the caller?
+			_, _, err := cm.lc.CertPair(ctx, domain)
+			if err != nil {
+				log.Printf("error refreshing certificate for %s: %v", domain, err)
+			}
+			var nextInterval time.Duration
+			// TODO(irbekrm): distinguish between LE rate limit
+			// errors and other error types like transient network
+			// errors.
+			if err == nil {
+				retryCount = 0
+				nextInterval = normalInterval
+			} else {
+				retryCount++
+				// Calculate backoff: initialRetry * 2^(retryCount-1)
+				// For retryCount=1: 1min * 2^0 = 1min
+				// For retryCount=2: 1min * 2^1 = 2min
+				// For retryCount=3: 1min * 2^2 = 4min
+				backoff := initialRetry * time.Duration(1<<(retryCount-1))
+				if backoff > maxRetryInterval {
+					backoff = maxRetryInterval
+				}
+				nextInterval = backoff
+				log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n",
+					domain, retryCount, err, nextInterval)
+			}
+			timer.Reset(nextInterval)
+		}
+	}
+}

cmd/containerboot/certs_test.go

@@ -0,0 +1,229 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+)
+
+// TestEnsureCertLoops tests that the certManager correctly starts and stops
+// update loops for certs when the serve config changes. It tracks goroutine
+// count and uses that as a validator that the expected number of cert loops are
+// running.
+func TestEnsureCertLoops(t *testing.T) {
+	tests := []struct {
+		name              string
+		initialConfig     *ipn.ServeConfig
+		updatedConfig     *ipn.ServeConfig
+		initialGoroutines int64 // after initial serve config is applied
+		updatedGoroutines int64 // after updated serve config is applied
+		wantErr           bool
+	}{
+		{
+			name:              "empty_serve_config",
+			initialConfig:     &ipn.ServeConfig{},
+			initialGoroutines: 0,
+		},
+		{
+			name:              "nil_serve_config",
+			initialConfig:     nil,
+			initialGoroutines: 0,
+			wantErr:           true,
+		},
+		{
+			name:          "empty_to_one_service",
+			initialConfig: &ipn.ServeConfig{},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 0,
+			updatedGoroutines: 1,
+		},
+		{
+			name: "single_service",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+		},
+		{
+			name: "multiple_services",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // one loop per domain across all services
+		},
+		{
+			name: "ignore_non_https_ports",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+							"my-app.tailnetxyz.ts.net:80":  {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1, // only one loop for the 443 endpoint
+		},
+		{
+			name: "remove_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // initially two loops (one per service)
+			updatedGoroutines: 1, // one loop after removing service2
+		},
+		{
+			name: "add_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+			updatedGoroutines: 2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			cm := &certManager{
+				lc:        &fakeLocalClient{},
+				certLoops: make(map[string]context.CancelFunc),
+			}
+
+			allDone := make(chan bool, 1)
+			defer cm.tracker.AddDoneCallback(func() {
+				cm.mu.Lock()
+				defer cm.mu.Unlock()
+				if cm.tracker.RunningGoroutines() > 0 {
+					return
+				}
+				select {
+				case allDone <- true:
+				default:
+				}
+			})()
+
+			err := cm.ensureCertLoops(ctx, tt.initialConfig)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ensureCertLoops() error = %v", err)
+			}
+
+			if got := cm.tracker.RunningGoroutines(); got != tt.initialGoroutines {
+				t.Errorf("after initial config: got %d running goroutines, want %d", got, tt.initialGoroutines)
+			}
+
+			if tt.updatedConfig != nil {
+				if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil {
+					t.Fatalf("ensureCertLoops() error on update = %v", err)
+				}
+
+				// Although starting goroutines and cancelling
+				// the context happens in the main goroutine, it
+				// the actual goroutine exit when a context is
+				// cancelled does not- so wait for a bit for the
+				// running goroutine count to reach the expected
+				// number.
+				deadline := time.After(5 * time.Second)
+				for {
+					if got := cm.tracker.RunningGoroutines(); got == tt.updatedGoroutines {
+						break
+					}
+					select {
+					case <-deadline:
+						t.Fatalf("timed out waiting for goroutine count to reach %d, currently at %d",
+							tt.updatedGoroutines, cm.tracker.RunningGoroutines())
+					case <-time.After(10 * time.Millisecond):
+						continue
+					}
+				}
+			}
+
+			if tt.updatedGoroutines == 0 {
+				return // no goroutines to wait for
+			}
+			// cancel context to make goroutines exit
+			cancel()
+			select {
+			case <-time.After(5 * time.Second):
+				t.Fatal("timed out waiting for goroutine to finish")
+			case <-allDone:
+			}
+		})
+	}
+}

cmd/containerboot/main.go

@@ -646,7 +646,7 @@ runLoop:
 
 				if cfg.ServeConfigPath != "" {
 					triggerWatchServeConfigChanges.Do(func() {
-						go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc)
+						go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg)
 					})
 				}
 

cmd/containerboot/serve.go

@@ -28,10 +28,11 @@ import (
 // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
 // is written to when the certDomain changes, causing the serve config to be
 // re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient) {
+func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings) {
 	if certDomainAtomic == nil {
 		panic("certDomainAtomic must not be nil")
 	}
+
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	if w, err := fsnotify.NewWatcher(); err != nil {
@@ -43,7 +44,7 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
+		if err := w.Add(filepath.Dir(cfg.ServeConfigPath)); err != nil {
 			log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
 		}
 		eventChan = w.Events
@@ -51,6 +52,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 
 	var certDomain string
 	var prevServeConfig *ipn.ServeConfig
+	var cm certManager
+	if cfg.CertShareMode == "rw" {
+		cm = certManager{
+			lc: lc,
+		}
+	}
 	for {
 		select {
 		case <-ctx.Done():
@@ -63,12 +70,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			// k8s handles these mounts. So just re-read the file and apply it
 			// if it's changed.
 		}
-		sc, err := readServeConfig(path, certDomain)
+		sc, err := readServeConfig(cfg.ServeConfigPath, certDomain)
 		if err != nil {
 			log.Fatalf("serve proxy: failed to read serve config: %v", err)
 		}
 		if sc == nil {
-			log.Printf("serve proxy: no serve config at %q, skipping", path)
+			log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath)
 			continue
 		}
 		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
@@ -83,6 +90,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			}
 		}
 		prevServeConfig = sc
+		if cfg.CertShareMode != "rw" {
+			continue
+		}
+		if err := cm.ensureCertLoops(ctx, sc); err != nil {
+			log.Fatalf("serve proxy: error ensuring cert loops: %v", err)
+		}
 	}
 }
 
@@ -96,6 +109,7 @@ func certDomainFromNetmap(nm *netmap.NetworkMap) string {
 // localClient is a subset of [local.Client] that can be mocked for testing.
 type localClient interface {
 	SetServeConfig(context.Context, *ipn.ServeConfig) error
+	CertPair(context.Context, string) ([]byte, []byte, error)
 }
 
 func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error {

cmd/containerboot/serve_test.go

@@ -206,6 +206,10 @@ func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConf
 	return nil
 }
 
+func (m *fakeLocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+	return nil, nil, nil
+}
+
 func TestHasHTTPSEndpoint(t *testing.T) {
 	tests := []struct {
 		name string

cmd/containerboot/settings.go

@@ -74,6 +74,12 @@ type settings struct {
 	HealthCheckEnabled   bool
 	DebugAddrPort        string
 	EgressProxiesCfgPath string
+	// CertShareMode is set for Kubernetes Pods running cert share mode.
+	// Possible values are empty (containerboot doesn't run any certs
+	// logic),  'ro' (for Pods that shold never attempt to issue/renew
+	// certs) and 'rw' for Pods that should manage the TLS certs shared
+	// amongst the replicas.
+	CertShareMode string
 }
 
 func configFromEnv() (*settings, error) {
@@ -128,6 +134,17 @@ func configFromEnv() (*settings, error) {
 			cfg.PodIPv6 = parsed.String()
 		}
 	}
+	// If cert share is enabled, set the replica as read or write. Only 0th
+	// replica should be able to write.
+	isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false)
+	if isInCertShareMode {
+		cfg.CertShareMode = "ro"
+		podName := os.Getenv("POD_NAME")
+		if strings.HasSuffix(podName, "-0") {
+			cfg.CertShareMode = "rw"
+		}
+	}
+
 	if err := cfg.validate(); err != nil {
 		return nil, fmt.Errorf("invalid configuration: %v", err)
 	}

cmd/containerboot/tailscaled.go

@@ -33,6 +33,9 @@ func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Pro
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
+	if cfg.CertShareMode != "" {
+		cmd.Env = append(os.Environ(), "TS_CERT_SHARE_MODE="+cfg.CertShareMode)
+	}
 	log.Printf("Starting tailscaled")
 	if err := cmd.Start(); err != nil {
 		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)

cmd/derper/depaware.txt

@@ -96,6 +96,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/drive                                          from tailscale.com/client/local+
         tailscale.com/envknob                                        from tailscale.com/client/local+
+        tailscale.com/feature                                        from tailscale.com/tsweb
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/local
@@ -128,8 +129,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/derp
-        tailscale.com/tsweb                                          from tailscale.com/cmd/derper
-        tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb                                          from tailscale.com/cmd/derper+
+        tailscale.com/tsweb/promvarz                                 from tailscale.com/cmd/derper
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
@@ -309,7 +310,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         html                                                         from net/http/pprof+
         html/template                                                from tailscale.com/cmd/derper
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -319,12 +320,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         internal/filepathlite                                        from os+
         internal/fmtsort                                             from fmt+
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
-        internal/godebug                                             from crypto/tls+
+        internal/godebug                                             from crypto/internal/fips140deps/godebug+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/derper/derper.go

@@ -49,6 +49,9 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/derpprobe/derpprobe.go

@@ -15,6 +15,9 @@ import (
 	"tailscale.com/prober"
 	"tailscale.com/tsweb"
 	"tailscale.com/version"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/k8s-operator/depaware.txt

@@ -1151,7 +1151,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -1163,11 +1163,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
         internal/godebug                                             from archive/tar+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
         internal/lazyregexp                                          from go/doc
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -75,7 +75,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
-  verbs: ["get", "create", "patch", "update", "list", "watch"]
+  verbs: ["get", "create", "patch", "update", "list", "watch", "deletecollection"]
 - apiGroups: ["monitoring.coreos.com"]
   resources: ["servicemonitors"]
   verbs: ["get", "list", "update", "create", "delete"]

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2215,6 +2215,22 @@ spec:
                         https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
+                useLetsEncryptStagingEnvironment:
+                  description: |-
+                    Set UseLetsEncryptStagingEnvironment to true to issue TLS
+                    certificates for any HTTPS endpoints exposed to the tailnet from
+                    LetsEncrypt's staging environment.
+                    https://letsencrypt.org/docs/staging-environment/
+                    This setting only affects Tailscale Ingress resources.
+                    By default Ingress TLS certificates are issued from LetsEncrypt's
+                    production environment.
+                    Changing this setting true -> false, will result in any
+                    existing certs being re-issued from the production environment.
+                    Changing this setting false (default) -> true, when certs have already
+                    been provisioned from production environment will NOT result in certs
+                    being re-issued from the staging environment before they need to be
+                    renewed.
+                  type: boolean
             status:
               description: |-
                 Status of the ProxyClass. This is set and managed automatically.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2685,6 +2685,22 @@ spec:
                                             Defaults to false.
                                         type: boolean
                                 type: object
+                            useLetsEncryptStagingEnvironment:
+                                description: |-
+                                    Set UseLetsEncryptStagingEnvironment to true to issue TLS
+                                    certificates for any HTTPS endpoints exposed to the tailnet from
+                                    LetsEncrypt's staging environment.
+                                    https://letsencrypt.org/docs/staging-environment/
+                                    This setting only affects Tailscale Ingress resources.
+                                    By default Ingress TLS certificates are issued from LetsEncrypt's
+                                    production environment.
+                                    Changing this setting true -> false, will result in any
+                                    existing certs being re-issued from the production environment.
+                                    Changing this setting false (default) -> true, when certs have already
+                                    been provisioned from production environment will NOT result in certs
+                                    being re-issued from the staging environment before they need to be
+                                    renewed.
+                                type: boolean
                         type: object
                     status:
                         description: |-
@@ -4898,6 +4914,7 @@ rules:
         - update
         - list
         - watch
+        - deletecollection
     - apiGroups:
         - monitoring.coreos.com
       resources:

cmd/k8s-operator/dnsrecords_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 )
@@ -163,10 +164,10 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 			Name:      o.GetName(),
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      o.GetName(),
-				LabelParentNamespace: o.GetNamespace(),
-				LabelParentType:      typ,
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        o.GetName(),
+				LabelParentNamespace:   o.GetNamespace(),
+				LabelParentType:        typ,
 			},
 		},
 		Spec: corev1.ServiceSpec{

cmd/k8s-operator/egress-pod-readiness.go

@@ -112,9 +112,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
 	}
 	// Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup.
 	lbls := map[string]string{
-		LabelManaged:    "true",
-		labelProxyGroup: proxyGroupName,
-		labelSvcType:    typeEgress,
+		kubetypes.LabelManaged: "true",
+		labelProxyGroup:        proxyGroupName,
+		labelSvcType:           typeEgress,
 	}
 	svcs := &corev1.ServiceList{}
 	if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil {

cmd/k8s-operator/egress-pod-readiness_test.go

@@ -450,9 +450,9 @@ func newSvc(name string, port int32) (*corev1.Service, string) {
 			Namespace: "operator-ns",
 			Name:      name,
 			Labels: map[string]string{
-				LabelManaged:    "true",
-				labelProxyGroup: "dev",
-				labelSvcType:    typeEgress,
+				kubetypes.LabelManaged: "true",
+				labelProxyGroup:        "dev",
+				labelSvcType:           typeEgress,
 			},
 		},
 		Spec: corev1.ServiceSpec{},

cmd/k8s-operator/egress-services.go

@@ -680,12 +680,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 // should probably validate and truncate (?) the names is they are too long.
 func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentType:      "svc",
-		LabelParentName:      svc.Name,
-		LabelParentNamespace: svc.Namespace,
-		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
-		labelSvcType:         typeEgress,
+		kubetypes.LabelManaged: "true",
+		LabelParentType:        "svc",
+		LabelParentName:        svc.Name,
+		LabelParentNamespace:   svc.Namespace,
+		labelProxyGroup:        svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:           typeEgress,
 	}
 }
 

cmd/k8s-operator/ingress-for-pg.go

@@ -22,6 +22,7 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -154,13 +155,13 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 	pg := &tsapi.ProxyGroup{}
 	if err := r.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil {
 		if apierrors.IsNotFound(err) {
-			logger.Infof("ProxyGroup %q does not exist", pgName)
+			logger.Infof("ProxyGroup does not exist")
 			return false, nil
 		}
 		return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
 	}
 	if !tsoperator.ProxyGroupIsReady(pg) {
-		logger.Infof("ProxyGroup %q is not (yet) ready", pgName)
+		logger.Infof("ProxyGroup is not (yet) ready")
 		return false, nil
 	}
 
@@ -175,8 +176,6 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		r.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
 	}
 
-	logger = logger.With("proxy-group", pg.Name)
-
 	if !slices.Contains(ing.Finalizers, FinalizerNamePG) {
 		// This log line is printed exactly once during initial provisioning,
 		// because once the finalizer is in place this block gets skipped. So,
@@ -229,12 +228,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 			return false, fmt.Errorf("error getting VIPService %q: %w", hostname, err)
 		}
 	}
-	// Generate the VIPService comment for new or existing VIPService. This
-	// checks and ensures that VIPService's owner references are updated for
-	// this Ingress and errors if that is not possible (i.e. because it
-	// appears that the VIPService has been created by a non-operator
-	// actor).
-	svcComment, err := r.ownerRefsComment(existingVIPSvc)
+	// Generate the VIPService owner annotation for new or existing VIPService.
+	// This checks and ensures that VIPService's owner references are updated
+	// for this Ingress and errors if that is not possible (i.e. because it
+	// appears that the VIPService has been created by a non-operator actor).
+	updatedAnnotations, err := r.ownerAnnotations(existingVIPSvc)
 	if err != nil {
 		const instr = "To proceed, you can either manually delete the existing VIPService or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition"
 		msg := fmt.Sprintf("error ensuring ownership of VIPService %s: %v. %s", hostname, err, instr)
@@ -242,8 +240,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidVIPService", msg)
 		return false, nil
 	}
+	// 3. Ensure that TLS Secret and RBAC exists
+	if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil {
+		return false, fmt.Errorf("error ensuring cert resources: %w", err)
+	}
 
-	// 3. Ensure that the serve config for the ProxyGroup contains the VIPService.
+	// 4. Ensure that the serve config for the ProxyGroup contains the VIPService.
 	cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
 	if err != nil {
 		return false, fmt.Errorf("error getting Ingress serve config: %w", err)
@@ -310,11 +312,13 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		vipPorts = append(vipPorts, "80")
 	}
 
+	const managedVIPServiceComment = "This VIPService is managed by the Tailscale Kubernetes Operator, do not modify"
 	vipSvc := &tailscale.VIPService{
-		Name:    serviceName,
-		Tags:    tags,
-		Ports:   vipPorts,
-		Comment: svcComment,
+		Name:        serviceName,
+		Tags:        tags,
+		Ports:       vipPorts,
+		Comment:     managedVIPServiceComment,
+		Annotations: updatedAnnotations,
 	}
 	if existingVIPSvc != nil {
 		vipSvc.Addrs = existingVIPSvc.Addrs
@@ -325,8 +329,8 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 	if existingVIPSvc == nil ||
 		!reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) ||
 		!reflect.DeepEqual(vipSvc.Ports, existingVIPSvc.Ports) ||
-		!strings.EqualFold(vipSvc.Comment, existingVIPSvc.Comment) {
-		logger.Infof("Ensuring VIPService %q exists and is up to date", hostname)
+		!ownersAreSetAndEqual(vipSvc, existingVIPSvc) {
+		logger.Infof("Ensuring VIPService exists and is up to date")
 		if err := r.tsClient.CreateOrUpdateVIPService(ctx, vipSvc); err != nil {
 			return false, fmt.Errorf("error creating VIPService: %w", err)
 		}
@@ -338,31 +342,48 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		return false, fmt.Errorf("failed to update tailscaled config: %w", err)
 	}
 
-	// TODO(irbekrm): check that the replicas are ready to route traffic for the VIPService before updating Ingress
-	// status.
-	// 6. Update Ingress status
+	// 6. Update Ingress status if ProxyGroup Pods are ready.
+	count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName)
+	if err != nil {
+		return false, fmt.Errorf("failed to check if any Pods are configured: %w", err)
+	}
+
 	oldStatus := ing.Status.DeepCopy()
-	ports := []networkingv1.IngressPortStatus{
-		{
-			Protocol: "TCP",
-			Port:     443,
-		},
+
+	switch count {
+	case 0:
+		ing.Status.LoadBalancer.Ingress = nil
+	default:
+		ports := []networkingv1.IngressPortStatus{
+			{
+				Protocol: "TCP",
+				Port:     443,
+			},
+		}
+		if isHTTPEndpointEnabled(ing) {
+			ports = append(ports, networkingv1.IngressPortStatus{
+				Protocol: "TCP",
+				Port:     80,
+			})
+		}
+		ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
+			{
+				Hostname: dnsName,
+				Ports:    ports,
+			},
+		}
 	}
-	if isHTTPEndpointEnabled(ing) {
-		ports = append(ports, networkingv1.IngressPortStatus{
-			Protocol: "TCP",
-			Port:     80,
-		})
-	}
-	ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
-		{
-			Hostname: dnsName,
-			Ports:    ports,
-		},
-	}
-	if apiequality.Semantic.DeepEqual(oldStatus, ing.Status) {
+	if apiequality.Semantic.DeepEqual(oldStatus, &ing.Status) {
 		return svcsChanged, nil
 	}
+
+	const prefix = "Updating Ingress status"
+	if count == 0 {
+		logger.Infof("%s. No Pods are advertising VIPService yet", prefix)
+	} else {
+		logger.Infof("%s. %d Pod(s) advertising VIPService", prefix, count)
+	}
+
 	if err := r.Status().Update(ctx, ing); err != nil {
 		return false, fmt.Errorf("failed to update Ingress status: %w", err)
 	}
@@ -402,24 +423,24 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
 			logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipServiceName)
 
 			// Delete the VIPService from control if necessary.
-			svc, _ := r.tsClient.GetVIPService(ctx, vipServiceName)
-			if svc != nil && isVIPServiceForAnyIngress(svc) {
-				logger.Infof("cleaning up orphaned VIPService %q", vipServiceName)
-				svcsChanged, err = r.cleanupVIPService(ctx, vipServiceName, logger)
-				if err != nil {
-					errResp := &tailscale.ErrResponse{}
-					if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
-						return false, fmt.Errorf("deleting VIPService %q: %w", vipServiceName, err)
-					}
-				}
+			svcsChanged, err = r.cleanupVIPService(ctx, vipServiceName, logger)
+			if err != nil {
+				return false, fmt.Errorf("deleting VIPService %q: %w", vipServiceName, err)
 			}
 
 			// Make sure the VIPService is not advertised in tailscaled or serve config.
 			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
 				return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 			}
-			delete(cfg.Services, vipServiceName)
-			serveConfigChanged = true
+			_, ok := cfg.Services[vipServiceName]
+			if ok {
+				logger.Infof("Removing VIPService %q from serve config", vipServiceName)
+				delete(cfg.Services, vipServiceName)
+				serveConfigChanged = true
+			}
+			if err := r.cleanupCertResources(ctx, proxyGroupName, vipServiceName); err != nil {
+				return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+			}
 		}
 	}
 
@@ -480,16 +501,22 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
 	if err != nil {
 		return false, fmt.Errorf("error deleting VIPService: %w", err)
 	}
+
+	// 3. Clean up any cluster resources
+	if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil {
+		return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+	}
+
 	if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup
 		return svcChanged, nil
 	}
 
-	// 3. Unadvertise the VIPService in tailscaled config.
+	// 4. Unadvertise the VIPService in tailscaled config.
 	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 	}
 
-	// 4. Remove the VIPService from the serve config for the ProxyGroup.
+	// 5. Remove the VIPService from the serve config for the ProxyGroup.
 	logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
 	delete(cfg.Services, serviceName)
 	cfgBytes, err := json.Marshal(cfg)
@@ -570,13 +597,6 @@ func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
 	return isTSIngress && pgAnnot != ""
 }
 
-func isVIPServiceForAnyIngress(svc *tailscale.VIPService) bool {
-	if svc == nil {
-		return false
-	}
-	return strings.HasPrefix(svc.Comment, "tailscale.com/k8s-operator:owned-by:")
-}
-
 // validateIngress validates that the Ingress is properly configured.
 // Currently validates:
 // - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags
@@ -650,34 +670,34 @@ func (r *HAIngressReconciler) cleanupVIPService(ctx context.Context, name tailcf
 	if svc == nil {
 		return false, nil
 	}
-	c, err := parseComment(svc)
+	o, err := parseOwnerAnnotation(svc)
 	if err != nil {
-		return false, fmt.Errorf("error parsing VIPService comment")
+		return false, fmt.Errorf("error parsing VIPService owner annotation")
 	}
-	if c == nil || len(c.OwnerRefs) == 0 {
+	if o == nil || len(o.OwnerRefs) == 0 {
 		return false, nil
 	}
 	// Comparing with the operatorID only means that we will not be able to
 	// clean up VIPServices in cases where the operator was deleted from the
 	// cluster before deleting the Ingress. Perhaps the comparison could be
 	// 'if or.OperatorID === r.operatorID || or.ingressUID == r.ingressUID'.
-	ix := slices.IndexFunc(c.OwnerRefs, func(or OwnerRef) bool {
+	ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool {
 		return or.OperatorID == r.operatorID
 	})
 	if ix == -1 {
 		return false, nil
 	}
-	if len(c.OwnerRefs) == 1 {
+	if len(o.OwnerRefs) == 1 {
 		logger.Infof("Deleting VIPService %q", name)
 		return false, r.tsClient.DeleteVIPService(ctx, name)
 	}
-	c.OwnerRefs = slices.Delete(c.OwnerRefs, ix, ix+1)
+	o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1)
 	logger.Infof("Deleting VIPService %q", name)
-	json, err := json.Marshal(c)
+	json, err := json.Marshal(o)
 	if err != nil {
 		return false, fmt.Errorf("error marshalling updated VIPService owner reference: %w", err)
 	}
-	svc.Comment = string(json)
+	svc.Annotations[ownerAnnotation] = string(json)
 	return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc)
 }
 
@@ -740,6 +760,39 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
 	return nil
 }
 
+func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) {
+	// Get all state Secrets for this ProxyGroup.
+	secrets := &corev1.SecretList{}
+	if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil {
+		return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err)
+	}
+
+	var count int
+	for _, secret := range secrets.Items {
+		prefs, ok, err := getDevicePrefs(&secret)
+		if err != nil {
+			return 0, fmt.Errorf("error getting node metadata: %w", err)
+		}
+		if !ok {
+			continue
+		}
+		if slices.Contains(prefs.AdvertiseServices, serviceName.String()) {
+			count++
+		}
+	}
+
+	return count, nil
+}
+
+const ownerAnnotation = "tailscale.com/owner-references"
+
+// ownerAnnotationValue is the content of the VIPService.Annotation[ownerAnnotation] field.
+type ownerAnnotationValue struct {
+	// OwnerRefs is a list of owner references that identify all operator
+	// instances that manage this VIPService.
+	OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
+}
+
 // OwnerRef is an owner reference that uniquely identifies a Tailscale
 // Kubernetes operator instance.
 type OwnerRef struct {
@@ -747,60 +800,110 @@ type OwnerRef struct {
 	OperatorID string `json:"operatorID,omitempty"`
 }
 
-// comment is the content of the VIPService.Comment field.
-type comment struct {
-	// OwnerRefs is a list of owner references that identify all operator
-	// instances that manage this VIPService.
-	OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
-}
-
-// ownerRefsComment return VIPService Comment that includes owner reference for this
-// operator instance for the provided VIPService. If the VIPService is nil, a
-// new comment with owner ref is returned. If the VIPService is not nil, the
-// existing comment is returned with the owner reference added, if not already
-// present. If the VIPService is not nil, but does not contain a comment we
-// return an error as this likely means that the VIPService was created by
-// somthing other than a Tailscale Kubernetes operator.
-func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (string, error) {
+// ownerAnnotations returns the updated annotations required to ensure this
+// instance of the operator is included as an owner. If the VIPService is not
+// nil, but does not contain an owner we return an error as this likely means
+// that the VIPService was created by somthing other than a Tailscale
+// Kubernetes operator.
+func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) {
 	ref := OwnerRef{
 		OperatorID: r.operatorID,
 	}
 	if svc == nil {
-		c := &comment{OwnerRefs: []OwnerRef{ref}}
+		c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
 		json, err := json.Marshal(c)
 		if err != nil {
-			return "", fmt.Errorf("[unexpected] unable to marshal VIPService comment contents: %w, please report this", err)
+			return nil, fmt.Errorf("[unexpected] unable to marshal VIPService owner annotation contents: %w, please report this", err)
 		}
-		return string(json), nil
+		return map[string]string{
+			ownerAnnotation: string(json),
+		}, nil
 	}
-	c, err := parseComment(svc)
+	o, err := parseOwnerAnnotation(svc)
 	if err != nil {
-		return "", fmt.Errorf("error parsing existing VIPService comment: %w", err)
+		return nil, err
 	}
-	if c == nil || len(c.OwnerRefs) == 0 {
-		return "", fmt.Errorf("VIPService %s exists, but does not contain Comment field with owner references- not proceeding as this is likely a resource created by something other than a Tailscale Kubernetes Operator", svc.Name)
+	if o == nil || len(o.OwnerRefs) == 0 {
+		return nil, fmt.Errorf("VIPService %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
 	}
-	if slices.Contains(c.OwnerRefs, ref) { // up to date
-		return svc.Comment, nil
+	if slices.Contains(o.OwnerRefs, ref) { // up to date
+		return svc.Annotations, nil
 	}
-	c.OwnerRefs = append(c.OwnerRefs, ref)
-	json, err := json.Marshal(c)
+	o.OwnerRefs = append(o.OwnerRefs, ref)
+	json, err := json.Marshal(o)
 	if err != nil {
-		return "", fmt.Errorf("error marshalling updated owner references: %w", err)
+		return nil, fmt.Errorf("error marshalling updated owner references: %w", err)
 	}
-	return string(json), nil
+
+	newAnnots := make(map[string]string, len(svc.Annotations)+1)
+	for k, v := range svc.Annotations {
+		newAnnots[k] = v
+	}
+	newAnnots[ownerAnnotation] = string(json)
+	return newAnnots, nil
 }
 
-// parseComment returns VIPService comment or nil if none found or not matching the expected format.
-func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
-	if vipSvc.Comment == "" {
+// parseOwnerAnnotation returns nil if no valid owner found.
+func parseOwnerAnnotation(vipSvc *tailscale.VIPService) (*ownerAnnotationValue, error) {
+	if vipSvc.Annotations == nil || vipSvc.Annotations[ownerAnnotation] == "" {
 		return nil, nil
 	}
-	c := &comment{}
-	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
-		return nil, fmt.Errorf("error parsing VIPService Comment field %q: %w", vipSvc.Comment, err)
+	o := &ownerAnnotationValue{}
+	if err := json.Unmarshal([]byte(vipSvc.Annotations[ownerAnnotation]), o); err != nil {
+		return nil, fmt.Errorf("error parsing VIPService %s annotation %q: %w", ownerAnnotation, vipSvc.Annotations[ownerAnnotation], err)
 	}
-	return c, nil
+	return o, nil
+}
+
+func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool {
+	return a != nil && b != nil &&
+		a.Annotations != nil && b.Annotations != nil &&
+		a.Annotations[ownerAnnotation] != "" &&
+		b.Annotations[ownerAnnotation] != "" &&
+		strings.EqualFold(a.Annotations[ownerAnnotation], b.Annotations[ownerAnnotation])
+}
+
+// ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC
+// resources that allow proxies to manage the Secret are created.
+// Note that Tailscale VIPService name validation matches Kubernetes
+// resource name validation, so we can be certain that the VIPService name
+// (domain) is a valid Kubernetes resource name.
+// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
+// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error {
+	secret := certSecret(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
+		return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
+	}
+	role := certSecretRole(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil {
+		return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
+	}
+	rb := certSecretRoleBinding(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil {
+		return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err)
+	}
+	return nil
+}
+
+// cleanupCertResources ensures that the TLS Secret and associated RBAC
+// resources that allow proxies to read/write to the Secret are deleted.
+func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
+	domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
+	if err != nil {
+		return fmt.Errorf("error getting DNS name for VIPService %s: %w", name, err)
+	}
+	labels := certResourceLabels(pgName, domainName)
+	if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err)
+	}
+	if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err)
+	}
+	if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err)
+	}
+	return nil
 }
 
 // requeueInterval returns a time duration between 5 and 10 minutes, which is
@@ -811,3 +914,93 @@ func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
 func requeueInterval() time.Duration {
 	return time.Duration(rand.N(5)+5) * time.Minute
 }
+
+// certSecretRole creates a Role that will allow proxies to manage the TLS
+// Secret for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecretRole(pgName, namespace, domain string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    certResourceLabels(pgName, domain),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{""},
+				Resources:     []string{"secrets"},
+				ResourceNames: []string{domain},
+				Verbs: []string{
+					"get",
+					"list",
+					"patch",
+					"update",
+				},
+			},
+		},
+	}
+}
+
+// certSecretRoleBinding creates a RoleBinding for Role that will allow proxies
+// to manage the TLS Secret for the given domain. Domain must be a valid
+// Kubernetes resource name.
+func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    certResourceLabels(pgName, domain),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      pgName,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: domain,
+		},
+	}
+}
+
+// certSecret creates a Secret that will store the TLS certificate and private
+// key for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecret(pgName, namespace, domain string) *corev1.Secret {
+	labels := certResourceLabels(pgName, domain)
+	labels[kubetypes.LabelSecretType] = "certs"
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       nil,
+			corev1.TLSPrivateKeyKey: nil,
+		},
+		Type: corev1.SecretTypeTLS,
+	}
+}
+
+func certResourceLabels(pgName, domain string) map[string]string {
+	return map[string]string{
+		kubetypes.LabelManaged:      "true",
+		"tailscale.com/proxy-group": pgName,
+		"tailscale.com/domain":      domain,
+	}
+}
+
+// dnsNameForService returns the DNS name for the given VIPService name.
+func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
+	s := svc.WithoutPrefix()
+	tcd, err := r.tailnetCertDomain(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error determining DNS name base: %w", err)
+	}
+	return s + "." + tcd, nil
+}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -8,8 +8,10 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"maps"
+	"net/http"
 	"reflect"
 	"testing"
 
@@ -18,6 +20,7 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -68,6 +71,11 @@ func TestIngressPGReconciler(t *testing.T) {
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
+	// Verify cert resources were created for the first Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
+
 	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
 		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
 	})
@@ -122,6 +130,11 @@ func TestIngressPGReconciler(t *testing.T) {
 	verifyServeConfig(t, fc, "svc:my-other-svc", false)
 	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
 
+	// Verify cert resources were created for the second Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
+
 	// Verify first Ingress is still working
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
@@ -158,6 +171,9 @@ func TestIngressPGReconciler(t *testing.T) {
 	}
 
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
 
 	// Delete the first Ingress and verify cleanup
 	if err := fc.Delete(context.Background(), ing); err != nil {
@@ -184,6 +200,66 @@ func TestIngressPGReconciler(t *testing.T) {
 		t.Error("serve config not cleaned up")
 	}
 	verifyTailscaledConfig(t, fc, nil)
+
+	// Add verification that cert resources were cleaned up
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-svc.ts.net")
+}
+
+func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+
+	// Verify initial reconciliation
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+
+	// Update the Ingress hostname and make sure the original VIPService is deleted.
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		ing.Spec.TLS[0].Hosts[0] = "updated-svc.tailnetxyz.ts.net"
+	})
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:updated-svc", false)
+	verifyVIPService(t, ft, "svc:updated-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})
+
+	_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc"))
+	if err == nil {
+		t.Fatalf("svc:my-svc not cleaned up")
+	}
+	var errResp *tailscale.ErrResponse
+	if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
+		t.Fatalf("unexpected error: %v", err)
+	}
 }
 
 func TestValidateIngress(t *testing.T) {
@@ -404,6 +480,31 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Status will be empty until the VIPService shows up in prefs.
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress, []networkingv1.IngressLoadBalancerIngress(nil)) {
+		t.Errorf("incorrect Ingress status: got %v, want empty",
+			ing.Status.LoadBalancer.Ingress)
+	}
+
+	// Add the VIPService to prefs to have the Ingress recognised as ready.
+	mustCreate(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg-0",
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "state"),
+		},
+		Data: map[string][]byte{
+			"_current-profile": []byte("profile-foo"),
+			"profile-foo":      []byte(`{"AdvertiseServices":["svc:my-svc"],"Config":{"NodeID":"node-foo"}}`),
+		},
+	})
+
+	// Reconcile and re-fetch Ingress.
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	if err := fc.Get(context.Background(), client.ObjectKeyFromObject(ing), ing); err != nil {
+		t.Fatal(err)
+	}
+
 	wantStatus := []networkingv1.IngressPortStatus{
 		{Port: 443, Protocol: "TCP"},
 		{Port: 80, Protocol: "TCP"},
@@ -644,8 +745,10 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 
 	// Simulate existing VIPService from another cluster
 	existingVIPSvc := &tailscale.VIPService{
-		Name:    "svc:my-svc",
-		Comment: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
+		Name: "svc:my-svc",
+		Annotations: map[string]string{
+			ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
+		},
 	}
 	ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{
 		"svc:my-svc": existingVIPSvc,
@@ -662,17 +765,17 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 		t.Fatal("VIPService not found")
 	}
 
-	c := &comment{}
-	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
-		t.Fatalf("parsing comment: %v", err)
+	o, err := parseOwnerAnnotation(vipSvc)
+	if err != nil {
+		t.Fatalf("parsing owner annotation: %v", err)
 	}
 
 	wantOwnerRefs := []OwnerRef{
 		{OperatorID: "operator-2"},
 		{OperatorID: "operator-1"},
 	}
-	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
-		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
+	if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
 	}
 
 	// Delete the Ingress and verify VIPService still exists with one owner ref
@@ -689,15 +792,15 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 		t.Fatal("VIPService was incorrectly deleted")
 	}
 
-	c = &comment{}
-	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
-		t.Fatalf("parsing comment after deletion: %v", err)
+	o, err = parseOwnerAnnotation(vipSvc)
+	if err != nil {
+		t.Fatalf("parsing owner annotation: %v", err)
 	}
 
 	wantOwnerRefs = []OwnerRef{
 		{OperatorID: "operator-2"},
 	}
-	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
-		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
+	if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
 	}
 }

cmd/k8s-operator/ingress_test.go

@@ -6,6 +6,7 @@
 package main
 
 import (
+	"context"
 	"testing"
 
 	"go.uber.org/zap"
@@ -15,17 +16,18 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
 
 func TestTailscaleIngress(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewFakeClient(tsIngressClass)
+	fc := fake.NewFakeClient(ingressClass())
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -46,45 +48,8 @@ func TestTailscaleIngress(t *testing.T) {
 	}
 
 	// 1. Resources get created for regular Ingress
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -114,6 +79,9 @@ func TestTailscaleIngress(t *testing.T) {
 		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
 	})
 	expectReconciled(t, ingR, "default", "test")
+
+	// Get the ingress and update it with expected changes
+	ing := ingress()
 	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 	ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
 		Ingress: []networkingv1.IngressLoadBalancerIngress{
@@ -143,8 +111,7 @@ func TestTailscaleIngress(t *testing.T) {
 }
 
 func TestTailscaleIngressHostname(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewFakeClient(tsIngressClass)
+	fc := fake.NewFakeClient(ingressClass())
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -165,45 +132,8 @@ func TestTailscaleIngressHostname(t *testing.T) {
 	}
 
 	// 1. Resources get created for regular Ingress
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -241,8 +171,10 @@ func TestTailscaleIngressHostname(t *testing.T) {
 		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
 	})
 	expectReconciled(t, ingR, "default", "test")
-	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 
+	// Get the ingress and update it with expected changes
+	ing := ingress()
+	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 	expectEqual(t, fc, ing)
 
 	// 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint
@@ -299,10 +231,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass).
+		WithObjects(pc, ingressClass()).
 		WithStatusSubresource(pc).
 		Build()
 	ft := &fakeTSClient{}
@@ -326,45 +257,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 
 	// 1. Ingress is created with no ProxyClass specified, default proxy
 	// resources get configured.
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -432,54 +326,19 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				ObservedGeneration: 1,
 			}}},
 	}
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-			Labels: map[string]string{
-				"tailscale.com/proxy-class": "metrics",
-			},
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	}
 	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+
+	// Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service
+	ing := ingress()
+	ing.Labels = map[string]string{
+		LabelProxyClass: "metrics",
+	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass, ing, svc).
+		WithObjects(pc, ingressClass(), ing, service()).
 		WithStatusSubresource(pc).
 		Build()
+
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -560,3 +419,118 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
 	// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
 }
+
+func TestIngressLetsEncryptStaging(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest()
+
+	testCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther)
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther)
+
+			fc := builder.Build()
+
+			if tt.proxyClassPerResource != "" || tt.defaultProxyClass != "" {
+				name := tt.proxyClassPerResource
+				if name == "" {
+					name = tt.defaultProxyClass
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			mustCreate(t, fc, ingressClass())
+			mustCreate(t, fc, service())
+			ing := ingress()
+			if tt.proxyClassPerResource != "" {
+				ing.Labels = map[string]string{
+					LabelProxyClass: tt.proxyClassPerResource,
+				}
+			}
+			mustCreate(t, fc, ing)
+
+			ingR := &IngressReconciler{
+				Client: fc,
+				ssr: &tailscaleSTSReconciler{
+					Client:            fc,
+					tsClient:          &fakeTSClient{},
+					tsnetServer:       &fakeTSNetServer{certDomains: []string{"test-host"}},
+					defaultTags:       []string{"tag:test"},
+					operatorNamespace: "operator-ns",
+					proxyImage:        "tailscale/tailscale:test",
+				},
+				logger:            zl.Sugar(),
+				defaultProxyClass: tt.defaultProxyClass,
+			}
+
+			expectReconciled(t, ingR, "default", "test")
+
+			_, shortName := findGenName(t, fc, "default", "test", "ingress")
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			if tt.useLEStagingEndpoint {
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			} else {
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			}
+		})
+	}
+}
+
+func ingressClass() *networkingv1.IngressClass {
+	return &networkingv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
+		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
+	}
+}
+
+func service() *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "1.2.3.4",
+			Ports: []corev1.ServicePort{{
+				Port: 8080,
+				Name: "http"},
+			},
+		},
+	}
+}
+
+func ingress() *networkingv1.Ingress {
+	return &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"default-test"}},
+			},
+		},
+	}
+}

cmd/k8s-operator/metrics_resources.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 )
 
 const (
@@ -222,7 +223,7 @@ func metricsResourceName(stsName string) string {
 // proxy.
 func metricsResourceLabels(opts *metricsOpts) map[string]string {
 	lbls := map[string]string{
-		LabelManaged:             "true",
+		kubetypes.LabelManaged:   "true",
 		labelMetricsTarget:       opts.proxyStsName,
 		labelPromProxyType:       opts.proxyType,
 		labelPromProxyParentName: opts.proxyLabels[LabelParentName],

cmd/k8s-operator/operator.go

@@ -347,6 +347,7 @@ func runReconcilers(opts reconcilerOpts) {
 		For(&networkingv1.Ingress{}).
 		Named("ingress-pg-reconciler").
 		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(ingressesFromPGStateSecret(mgr.GetClient(), startlog))).
 		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
 		Complete(&HAIngressReconciler{
 			recorder:    eventRecorder,
@@ -636,8 +637,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Service.
 		svcProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "svc",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "svc",
 		}
 		svcHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
@@ -650,8 +651,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Ingress.
 		ingProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "ingress",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "ingress",
 		}
 		ingHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
@@ -718,7 +719,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 func isManagedResource(o client.Object) bool {
 	ls := o.GetLabels()
-	return ls[LabelManaged] == "true"
+	return ls[kubetypes.LabelManaged] == "true"
 }
 
 func isManagedByType(o client.Object, typ string) bool {
@@ -955,7 +956,7 @@ func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
 		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
@@ -975,15 +976,13 @@ func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
 		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
 			return nil
 		}
-		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
+		if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
 			return nil
 		}
 		pg, ok := o.GetLabels()[LabelParentName]
@@ -1000,7 +999,7 @@ func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
 	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
 		return nil
 	}
-	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+	if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 		return nil
 	}
 	svcName, ok := o.GetLabels()[LabelParentName]
@@ -1040,6 +1039,45 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
 	return reqs
 }
 
+func ingressesFromPGStateSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		secret, ok := o.(*corev1.Secret)
+		if !ok {
+			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
+			return nil
+		}
+		if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" {
+			return nil
+		}
+		if secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
+			return nil
+		}
+		if secret.ObjectMeta.Labels[kubetypes.LabelSecretType] != "state" {
+			return nil
+		}
+		pgName, ok := secret.ObjectMeta.Labels[LabelParentName]
+		if !ok {
+			return nil
+		}
+
+		ingList := &networkingv1.IngressList{}
+		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pgName}); err != nil {
+			logger.Infof("error listing Ingresses, skipping a reconcile for event on Secret %s: %v", secret.Name, err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, ing := range ingList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: ing.Namespace,
+					Name:      ing.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
 // user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1145,9 +1183,9 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h
 			return nil
 		}
 		podLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "proxygroup",
-			LabelParentName: eps.Labels[labelProxyGroup],
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "proxygroup",
+			LabelParentName:        eps.Labels[labelProxyGroup],
 		}
 		podList := &corev1.PodList{}
 		if err := cl.List(ctx, podList, client.InNamespace(ns),

cmd/k8s-operator/operator_test.go

@@ -1387,10 +1387,10 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 			Name:      "headless-1",
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      "ing-1",
-				LabelParentNamespace: "ns-1",
-				LabelParentType:      "ingress",
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        "ing-1",
+				LabelParentNamespace:   "ns-1",
+				LabelParentType:        "ingress",
 			},
 		},
 	}

cmd/k8s-operator/proxygroup.go

@@ -302,7 +302,10 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	if err != nil {
 		return fmt.Errorf("error generating StatefulSet spec: %w", err)
 	}
-	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
+	cfg := &tailscaleSTSConfig{
+		proxyType: string(pg.Spec.Type),
+	}
+	ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger)
 	capver, err := r.capVerForPG(ctx, pg, logger)
 	if err != nil {
 		return fmt.Errorf("error getting device info: %w", err)
@@ -461,7 +464,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 
 		var existingCfgSecret *corev1.Secret // unmodified copy of secret
 		if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
-			logger.Debugf("secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
+			logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
 			existingCfgSecret = cfgSecret.DeepCopy()
 		} else if !apierrors.IsNotFound(err) {
 			return "", err
@@ -469,7 +472,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 
 		var authKey string
 		if existingCfgSecret == nil {
-			logger.Debugf("creating authkey for new ProxyGroup proxy")
+			logger.Debugf("Creating authkey for new ProxyGroup proxy")
 			tags := pg.Spec.Tags.Stringify()
 			if len(tags) == 0 {
 				tags = r.defaultTags
@@ -490,7 +493,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			if err != nil {
 				return "", fmt.Errorf("error marshalling tailscaled config: %w", err)
 			}
-			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
+			mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
 		}
 
 		// The config sha256 sum is a value for a hash annotation used to trigger
@@ -520,12 +523,14 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		}
 
 		if existingCfgSecret != nil {
-			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
-			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
-				return "", err
+			if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) {
+				logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name)
+				if err := r.Update(ctx, cfgSecret); err != nil {
+					return "", err
+				}
 			}
 		} else {
-			logger.Debugf("creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
+			logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
 			if err := r.Create(ctx, cfgSecret); err != nil {
 				return "", err
 			}
@@ -645,7 +650,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 			return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err)
 		}
 
-		id, dnsName, ok, err := getNodeMetadata(ctx, &secret)
+		prefs, ok, err := getDevicePrefs(&secret)
 		if err != nil {
 			return nil, err
 		}
@@ -656,8 +661,8 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 		nm := nodeMetadata{
 			ordinal:     ordinal,
 			stateSecret: &secret,
-			tsID:        id,
-			dnsName:     dnsName,
+			tsID:        prefs.Config.NodeID,
+			dnsName:     prefs.Config.UserProfile.LoginName,
 		}
 		pod := &corev1.Pod{}
 		if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) {

cmd/k8s-operator/proxygroup_specs.go

@@ -178,7 +178,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				corev1.EnvVar{
 					Name:  "TS_SERVE_CONFIG",
 					Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),
-				})
+				},
+				corev1.EnvVar{
+					// Run proxies in cert share mode to
+					// ensure that only one TLS cert is
+					// issued for an HA Ingress.
+					Name:  "TS_EXPERIMENTAL_CERT_SHARE",
+					Value: "true",
+				},
+			)
 		}
 		return append(c.Env, envs...)
 	}()
@@ -225,6 +233,13 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 			OwnerReferences: pgOwnerReference(pg),
 		},
 		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"list",
+				},
+			},
 			{
 				APIGroups: []string{""},
 				Resources: []string{"secrets"},
@@ -318,9 +333,9 @@ func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
 	}
 }
 
-func pgSecretLabels(pgName, typ string) map[string]string {
+func pgSecretLabels(pgName, secretType string) map[string]string {
 	return pgLabels(pgName, map[string]string{
-		labelSecretType: typ, // "config" or "state".
+		kubetypes.LabelSecretType: secretType, // "config" or "state".
 	})
 }
 
@@ -330,7 +345,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
 		l[k] = v
 	}
 
-	l[LabelManaged] = "true"
+	l[kubetypes.LabelManaged] = "true"
 	l[LabelParentType] = "proxygroup"
 	l[LabelParentName] = pgName
 

cmd/k8s-operator/proxygroup_test.go

@@ -247,7 +247,6 @@ func TestProxyGroup(t *testing.T) {
 		// The fake client does not clean up objects whose owner has been
 		// deleted, so we can't test for the owned resources getting deleted.
 	})
-
 }
 
 func TestProxyGroupTypes(t *testing.T) {
@@ -417,6 +416,7 @@ func TestProxyGroupTypes(t *testing.T) {
 		}
 		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
 		verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
+		verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
 
 		// Verify ConfigMap volume mount
 		cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
@@ -475,8 +475,6 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
 			Name:      pgConfigSecretName(pgName, 0),
 			Namespace: tsNamespace,
 		},
-		// Write directly to Data because the fake client doesn't copy the write-only
-		// StringData field across to Data for us.
 		Data: map[string][]byte{
 			tsoperator.TailscaledConfigFileName(106): existingConfigBytes,
 		},
@@ -514,10 +512,64 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
 			Namespace:       tsNamespace,
 			ResourceVersion: "2",
 		},
-		StringData: map[string]string{
-			tsoperator.TailscaledConfigFileName(106): string(expectedConfigBytes),
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): expectedConfigBytes,
 		},
-	}, omitSecretData)
+	})
+}
+
+func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) {
+	pcLEStaging := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "le-staging",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: true,
+		},
+	}
+
+	pcLEStagingFalse := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "le-staging-false",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: false,
+		},
+	}
+
+	pcOther := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "other",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{},
+	}
+
+	return pcLEStaging, pcLEStagingFalse, pcOther
+}
+
+func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name string) *tsapi.ProxyClass {
+	t.Helper()
+	pc := &tsapi.ProxyClass{}
+	if err := fc.Get(context.Background(), client.ObjectKey{Name: name}, pc); err != nil {
+		t.Fatal(err)
+	}
+	pc.Status = tsapi.ProxyClassStatus{
+		Conditions: []metav1.Condition{{
+			Type:               string(tsapi.ProxyClassReady),
+			Status:             metav1.ConditionTrue,
+			Reason:             reasonProxyClassValid,
+			Message:            reasonProxyClassValid,
+			LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+			ObservedGeneration: pc.Generation,
+		}},
+	}
+	if err := fc.Status().Update(context.Background(), pc); err != nil {
+		t.Fatal(err)
+	}
+	return pc
 }
 
 func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
@@ -543,6 +595,16 @@ func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue str
 	t.Errorf("%s environment variable not found", name)
 }
 
+func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) {
+	t.Helper()
+	for _, env := range sts.Spec.Template.Spec.Containers[0].Env {
+		if env.Name == name {
+			t.Errorf("environment variable %s should not be present", name)
+			return
+		}
+	}
+}
+
 func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) {
 	t.Helper()
 
@@ -621,10 +683,145 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
 	}
 }
 
-// The operator mostly writes to StringData and reads from Data, but the fake
-// client doesn't copy StringData across to Data on write. When comparing actual
-// vs expected Secrets, use this function to only check what the operator writes
-// to StringData.
-func omitSecretData(secret *corev1.Secret) {
-	secret.Data = nil
+func TestProxyGroupLetsEncryptStaging(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	// Set up test cases- most are shared with non-HA Ingress.
+	type proxyGroupLETestCase struct {
+		leStagingTestCase
+		pgType tsapi.ProxyGroupType
+	}
+	pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest()
+	sharedTestCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther)
+	var tests []proxyGroupLETestCase
+	for _, tt := range sharedTestCases {
+		tests = append(tests, proxyGroupLETestCase{
+			leStagingTestCase: tt,
+			pgType:            tsapi.ProxyGroupTypeIngress,
+		})
+	}
+	tests = append(tests, proxyGroupLETestCase{
+		leStagingTestCase: leStagingTestCase{
+			name:                  "egress_pg_with_staging_proxyclass",
+			proxyClassPerResource: "le-staging",
+			useLEStagingEndpoint:  false,
+		},
+		pgType: tsapi.ProxyGroupTypeEgress,
+	})
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			// Pre-populate the fake client with ProxyClasses.
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther)
+
+			fc := builder.Build()
+
+			// If the test case needs a ProxyClass to exist, ensure it is set to Ready.
+			if tt.proxyClassPerResource != "" || tt.defaultProxyClass != "" {
+				name := tt.proxyClassPerResource
+				if name == "" {
+					name = tt.defaultProxyClass
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			// Create ProxyGroup
+			pg := &tsapi.ProxyGroup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+				},
+				Spec: tsapi.ProxyGroupSpec{
+					Type:       tt.pgType,
+					Replicas:   ptr.To[int32](1),
+					ProxyClass: tt.proxyClassPerResource,
+				},
+			}
+			mustCreate(t, fc, pg)
+
+			reconciler := &ProxyGroupReconciler{
+				tsNamespace:       tsNamespace,
+				proxyImage:        testProxyImage,
+				defaultTags:       []string{"tag:test"},
+				defaultProxyClass: tt.defaultProxyClass,
+				Client:            fc,
+				tsClient:          &fakeTSClient{},
+				l:                 zl.Sugar(),
+				clock:             cl,
+			}
+
+			expectReconciled(t, reconciler, "", pg.Name)
+
+			// Verify that the StatefulSet created for ProxyGrup has
+			// the expected setting for the staging endpoint.
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			if tt.useLEStagingEndpoint {
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			} else {
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			}
+		})
+	}
+}
+
+type leStagingTestCase struct {
+	name string
+	// ProxyClass set on ProxyGroup or Ingress resource.
+	proxyClassPerResource string
+	// Default ProxyClass.
+	defaultProxyClass    string
+	useLEStagingEndpoint bool
+}
+
+// Shared test cases for LE staging endpoint configuration for ProxyGroup and
+// non-HA Ingress.
+func testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther *tsapi.ProxyClass) []leStagingTestCase {
+	return []leStagingTestCase{
+		{
+			name:                  "with_staging_proxyclass",
+			proxyClassPerResource: "le-staging",
+			useLEStagingEndpoint:  true,
+		},
+		{
+			name:                  "with_staging_proxyclass_false",
+			proxyClassPerResource: "le-staging-false",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_other_proxyclass",
+			proxyClassPerResource: "other",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "no_proxyclass",
+			proxyClassPerResource: "",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_default_staging_proxyclass",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "le-staging",
+			useLEStagingEndpoint:  true,
+		},
+		{
+			name:                  "with_default_other_proxyclass",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "other",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_default_staging_proxyclass_false",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "le-staging-false",
+			useLEStagingEndpoint:  false,
+		},
+	}
 }

cmd/k8s-operator/sts.go

@@ -44,11 +44,9 @@ const (
 	// Labels that the operator sets on StatefulSets and Pods. If you add a
 	// new label here, do also add it to tailscaleManagedLabels var to
 	// ensure that it does not get overwritten by ProxyClass configuration.
-	LabelManaged         = "tailscale.com/managed"
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
 	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
 	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
@@ -104,11 +102,13 @@ const (
 
 	envVarTSLocalAddrPort = "TS_LOCAL_ADDR_PORT"
 	defaultLocalAddrPort  = 9002 // metrics and health check port
+
+	letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory"
 )
 
 var (
 	// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
-	tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
+	tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
 	// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
 	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
 )
@@ -785,6 +785,17 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 			enableEndpoints(ss, metricsEnabled, debugEnabled)
 		}
 	}
+	if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) {
+		for i, c := range ss.Spec.Template.Spec.Containers {
+			if c.Name == "tailscale" {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{
+					Name:  "TS_DEBUG_ACME_DIRECTORY_URL",
+					Value: letsEncryptStagingEndpoint,
+				})
+				break
+			}
+		}
+	}
 
 	if pc.Spec.StatefulSet == nil {
 		return ss

cmd/k8s-operator/sts_test.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
 
@@ -156,8 +157,8 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// Set a couple additional fields so we can test that we don't
 	// mistakenly override those.
 	labels := map[string]string{
-		LabelManaged:    "true",
-		LabelParentName: "foo",
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        "foo",
 	}
 	annots := map[string]string{
 		podAnnotationLastSetClusterIP: "1.2.3.4",
@@ -303,28 +304,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 	}{
 		{
 			name:    "no custom labels specified and none present in current labels, return current labels",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels",
-			current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels only contain tailscale managed labels, return a union of both",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels",
-			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{

cmd/k8s-operator/svc.go

@@ -84,10 +84,10 @@ func childResourceLabels(name, ns, typ string) map[string]string {
 	// proxying. Instead, we have to do our own filtering and tracking with
 	// labels.
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 }
 

cmd/k8s-operator/testutils_test.go

@@ -32,6 +32,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
@@ -563,10 +564,10 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
 	t.Helper()
 	labels := map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 	s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
 	if err != nil {

cmd/k8s-operator/tsrecorder.go

@@ -230,7 +230,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
 func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
 	logger := r.logger(tsr.Name)
 
-	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
+	prefs, ok, err := r.getDevicePrefs(ctx, tsr.Name)
 	if err != nil {
 		return false, err
 	}
@@ -243,6 +243,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
 		return true, nil
 	}
 
+	id := string(prefs.Config.NodeID)
 	logger.Debugf("deleting device %s from control", string(id))
 	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
 		errResp := &tailscale.ErrResponse{}
@@ -327,34 +328,33 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string)
 	return secret, nil
 }
 
-func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+func (r *RecorderReconciler) getDevicePrefs(ctx context.Context, tsrName string) (prefs prefs, ok bool, err error) {
 	secret, err := r.getStateSecret(ctx, tsrName)
 	if err != nil || secret == nil {
-		return "", "", false, err
+		return prefs, false, err
 	}
 
-	return getNodeMetadata(ctx, secret)
+	return getDevicePrefs(secret)
 }
 
-// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// getDevicePrefs returns 'ok == true' iff the node ID is found. The dnsName
 // is expected to always be non-empty if the node ID is, but not required.
-func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) {
 	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
 	currentProfile, ok := secret.Data[currentProfileKey]
 	if !ok {
-		return "", "", false, nil
+		return prefs, false, nil
 	}
 	profileBytes, ok := secret.Data[string(currentProfile)]
 	if !ok {
-		return "", "", false, nil
+		return prefs, false, nil
 	}
-	var profile profile
-	if err := json.Unmarshal(profileBytes, &profile); err != nil {
-		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
+	if err := json.Unmarshal(profileBytes, &prefs); err != nil {
+		return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
 	}
 
-	ok = profile.Config.NodeID != ""
-	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
+	ok = prefs.Config.NodeID != ""
+	return prefs, ok, nil
 }
 
 func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
@@ -367,14 +367,14 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string)
 }
 
 func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	nodeID, dnsName, ok, err := getNodeMetadata(ctx, secret)
+	prefs, ok, err := getDevicePrefs(secret)
 	if !ok || err != nil {
 		return tsapi.RecorderTailnetDevice{}, false, err
 	}
 
 	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
 	// need the API. Should we instead update the profile to include addresses?
-	device, err := tsClient.Device(ctx, string(nodeID), nil)
+	device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil)
 	if err != nil {
 		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
 	}
@@ -383,20 +383,25 @@ func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret
 		Hostname:   device.Hostname,
 		TailnetIPs: device.Addresses,
 	}
-	if dnsName != "" {
+	if dnsName := prefs.Config.UserProfile.LoginName; dnsName != "" {
 		d.URL = fmt.Sprintf("https://%s", dnsName)
 	}
 
 	return d, true, nil
 }
 
-type profile struct {
+// [prefs] is a subset of the ipn.Prefs struct used for extracting information
+// from the state Secret of Tailscale devices.
+type prefs struct {
 	Config struct {
-		NodeID      string `json:"NodeID"`
+		NodeID      tailcfg.StableNodeID `json:"NodeID"`
 		UserProfile struct {
+			// LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net.
 			LoginName string `json:"LoginName"`
 		} `json:"UserProfile"`
 	} `json:"Config"`
+
+	AdvertiseServices []string `json:"AdvertiseServices"`
 }
 
 func markedForDeletion(obj metav1.Object) bool {

cmd/proxy-to-grafana/proxy-to-grafana.go

@@ -19,8 +19,25 @@
 //	header_property = username
 //	auto_sign_up = true
 //	whitelist = 127.0.0.1
-//	headers = Name:X-WEBAUTH-NAME
+//	headers = Email:X-Webauth-User, Name:X-Webauth-Name, Role:X-Webauth-Role
 //	enable_login_token = true
+//
+// You can use grants in Tailscale ACL to give users different roles in Grafana.
+// For example, to give group:eng the Editor role, add the following to your ACLs:
+//
+//	 "grants": [
+//			{
+//				"src": ["group:eng"],
+//				"dst": ["tag:grafana"],
+//				"app": {
+//					"tailscale.com/cap/proxy-to-grafana": [{
+//						"role": "editor",
+//					}],
+//				},
+//			},
+//	 ],
+//
+// If multiple roles are specified, the most permissive role is used.
 package main
 
 import (
@@ -49,6 +66,57 @@ var (
 	loginServer  = flag.String("login-server", "", "URL to alternative control server. If empty, the default Tailscale control is used.")
 )
 
+// aclCap is the Tailscale ACL capability used to configure proxy-to-grafana.
+const aclCap tailcfg.PeerCapability = "tailscale.com/cap/proxy-to-grafana"
+
+// aclGrant is an access control rule that assigns Grafana permissions
+// while provisioning a user.
+type aclGrant struct {
+	// Role is one of: "viewer", "editor", "admin".
+	Role string `json:"role"`
+}
+
+// grafanaRole defines possible Grafana roles.
+type grafanaRole int
+
+const (
+	// Roles are ordered by their permissions, with the least permissive role first.
+	// If a user has multiple roles, the most permissive role is used.
+	ViewerRole grafanaRole = iota
+	EditorRole
+	AdminRole
+)
+
+// String returns the string representation of a grafanaRole.
+// It is used as a header value in the HTTP request to Grafana.
+func (r grafanaRole) String() string {
+	switch r {
+	case ViewerRole:
+		return "Viewer"
+	case EditorRole:
+		return "Editor"
+	case AdminRole:
+		return "Admin"
+	default:
+		// A safe default.
+		return "Viewer"
+	}
+}
+
+// roleFromString converts a string to a grafanaRole.
+// It is used to parse the role from the ACL grant.
+func roleFromString(s string) (grafanaRole, error) {
+	switch strings.ToLower(s) {
+	case "viewer":
+		return ViewerRole, nil
+	case "editor":
+		return EditorRole, nil
+	case "admin":
+		return AdminRole, nil
+	}
+	return ViewerRole, fmt.Errorf("unknown role: %q", s)
+}
+
 func main() {
 	flag.Parse()
 	if *hostname == "" || strings.Contains(*hostname, ".") {
@@ -134,7 +202,15 @@ func modifyRequest(req *http.Request, localClient *local.Client) {
 		return
 	}
 
-	user, err := getTailscaleUser(req.Context(), localClient, req.RemoteAddr)
+	// Delete any existing X-Webauth-* headers to prevent possible spoofing
+	// if getting Tailnet identity fails.
+	for h := range req.Header {
+		if strings.HasPrefix(h, "X-Webauth-") {
+			req.Header.Del(h)
+		}
+	}
+
+	user, role, err := getTailscaleIdentity(req.Context(), localClient, req.RemoteAddr)
 	if err != nil {
 		log.Printf("error getting Tailscale user: %v", err)
 		return
@@ -142,19 +218,33 @@ func modifyRequest(req *http.Request, localClient *local.Client) {
 
 	req.Header.Set("X-Webauth-User", user.LoginName)
 	req.Header.Set("X-Webauth-Name", user.DisplayName)
+	req.Header.Set("X-Webauth-Role", role.String())
 }
 
-func getTailscaleUser(ctx context.Context, localClient *local.Client, ipPort string) (*tailcfg.UserProfile, error) {
+func getTailscaleIdentity(ctx context.Context, localClient *local.Client, ipPort string) (*tailcfg.UserProfile, grafanaRole, error) {
 	whois, err := localClient.WhoIs(ctx, ipPort)
 	if err != nil {
-		return nil, fmt.Errorf("failed to identify remote host: %w", err)
+		return nil, ViewerRole, fmt.Errorf("failed to identify remote host: %w", err)
 	}
 	if whois.Node.IsTagged() {
-		return nil, fmt.Errorf("tagged nodes are not users")
+		return nil, ViewerRole, fmt.Errorf("tagged nodes are not users")
 	}
 	if whois.UserProfile == nil || whois.UserProfile.LoginName == "" {
-		return nil, fmt.Errorf("failed to identify remote user")
+		return nil, ViewerRole, fmt.Errorf("failed to identify remote user")
 	}
 
-	return whois.UserProfile, nil
+	role := ViewerRole
+	grants, err := tailcfg.UnmarshalCapJSON[aclGrant](whois.CapMap, aclCap)
+	if err != nil {
+		return nil, ViewerRole, fmt.Errorf("failed to unmarshal ACL grants: %w", err)
+	}
+	for _, g := range grants {
+		r, err := roleFromString(g.Role)
+		if err != nil {
+			return nil, ViewerRole, fmt.Errorf("failed to parse role: %w", err)
+		}
+		role = max(role, r)
+	}
+
+	return whois.UserProfile, role, nil
 }

cmd/stund/depaware.txt

@@ -49,6 +49,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/envknob                                        from tailscale.com/tsweb+
+        tailscale.com/feature                                        from tailscale.com/tsweb
         tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/net/stunserver+
         tailscale.com/net/netaddr                                    from tailscale.com/net/tsaddr
@@ -57,8 +58,8 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/net/tsaddr                                     from tailscale.com/tsweb
         tailscale.com/syncs                                          from tailscale.com/metrics
         tailscale.com/tailcfg                                        from tailscale.com/version
-        tailscale.com/tsweb                                          from tailscale.com/cmd/stund
-        tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb                                          from tailscale.com/cmd/stund+
+        tailscale.com/tsweb/promvarz                                 from tailscale.com/cmd/stund
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg
@@ -194,7 +195,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -204,12 +205,12 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         internal/filepathlite                                        from os+
         internal/fmtsort                                             from fmt
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
-        internal/godebug                                             from crypto/tls+
+        internal/godebug                                             from crypto/internal/fips140deps/godebug+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/stund/stund.go

@@ -15,6 +15,9 @@ import (
 
 	"tailscale.com/net/stunserver"
 	"tailscale.com/tsweb"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/tailscale/cli/debug.go

@@ -136,6 +136,17 @@ func debugCmd() *ffcli.Command {
 				Exec:       runLocalCreds,
 				ShortHelp:  "Print how to access Tailscale LocalAPI",
 			},
+			{
+				Name:       "localapi",
+				ShortUsage: "tailscale debug localapi [<method>] <path> [<body| \"-\">]",
+				Exec:       runLocalAPI,
+				ShortHelp:  "Call a LocalAPI method directly",
+				FlagSet: (func() *flag.FlagSet {
+					fs := newFlagSet("localapi")
+					fs.BoolVar(&localAPIFlags.verbose, "v", false, "verbose; dump HTTP headers")
+					return fs
+				})(),
+			},
 			{
 				Name:       "restun",
 				ShortUsage: "tailscale debug restun",
@@ -451,6 +462,81 @@ func runLocalCreds(ctx context.Context, args []string) error {
 	return nil
 }
 
+func looksLikeHTTPMethod(s string) bool {
+	if len(s) > len("OPTIONS") {
+		return false
+	}
+	for _, r := range s {
+		if r < 'A' || r > 'Z' {
+			return false
+		}
+	}
+	return true
+}
+
+var localAPIFlags struct {
+	verbose bool
+}
+
+func runLocalAPI(ctx context.Context, args []string) error {
+	if len(args) == 0 {
+		return errors.New("expected at least one argument")
+	}
+	method := "GET"
+	if looksLikeHTTPMethod(args[0]) {
+		method = args[0]
+		args = args[1:]
+		if len(args) == 0 {
+			return errors.New("expected at least one argument after method")
+		}
+	}
+	path := args[0]
+	if !strings.HasPrefix(path, "/localapi/") {
+		if !strings.Contains(path, "/") {
+			path = "/localapi/v0/" + path
+		} else {
+			path = "/localapi/" + path
+		}
+	}
+
+	var body io.Reader
+	if len(args) > 1 {
+		if args[1] == "-" {
+			fmt.Fprintf(Stderr, "# reading request body from stdin...\n")
+			all, err := io.ReadAll(os.Stdin)
+			if err != nil {
+				return fmt.Errorf("reading Stdin: %q", err)
+			}
+			body = bytes.NewReader(all)
+		} else {
+			body = strings.NewReader(args[1])
+		}
+	}
+	req, err := http.NewRequest(method, "http://local-tailscaled.sock"+path, body)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(Stderr, "# doing request %s %s\n", method, path)
+
+	res, err := localClient.DoLocalRequest(req)
+	if err != nil {
+		return err
+	}
+	is2xx := res.StatusCode >= 200 && res.StatusCode <= 299
+	if localAPIFlags.verbose {
+		res.Write(Stdout)
+	} else {
+		if !is2xx {
+			fmt.Fprintf(Stderr, "# Response status %s\n", res.Status)
+		}
+		io.Copy(Stdout, res.Body)
+	}
+	if is2xx {
+		return nil
+	}
+	return errors.New(res.Status)
+}
+
 type localClientRoundTripper struct{}
 
 func (localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {

cmd/tailscale/depaware.txt

@@ -333,7 +333,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         image/color                                                  from github.com/skip2/go-qrcode+
         image/png                                                    from github.com/skip2/go-qrcode
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -345,10 +345,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
         internal/godebug                                             from archive/tar+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/tailscaled/depaware.txt

@@ -286,7 +286,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/ipnlocal+
    L    tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
    L    tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
-        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -589,7 +589,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -601,10 +601,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
         internal/godebug                                             from archive/tar+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/testwrapper/testwrapper.go

@@ -259,6 +259,7 @@ func main() {
 			fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j)
 		}
 
+		fatalFailures := make(map[string]struct{}) // pkg.Test key
 		toRetry := make(map[string][]*testAttempt) // pkg -> tests to retry
 		for _, pt := range thisRun.tests {
 			ch := make(chan *testAttempt)
@@ -301,11 +302,24 @@ func main() {
 				if tr.isMarkedFlaky {
 					toRetry[tr.pkg] = append(toRetry[tr.pkg], tr)
 				} else {
+					fatalFailures[tr.pkg+"."+tr.testName] = struct{}{}
 					failed = true
 				}
 			}
 			if failed {
 				fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
+
+				// Print the list of non-flakytest failures.
+				// We will later analyze the retried GitHub Action runs to see
+				// if non-flakytest failures succeeded upon retry. This will
+				// highlight tests which are flaky but not yet flagged as such.
+				if len(fatalFailures) > 0 {
+					tests := slicesx.MapKeys(fatalFailures)
+					sort.Strings(tests)
+					j, _ := json.Marshal(tests)
+					fmt.Printf("non-flakytest failures: %s\n", j)
+				}
+				fmt.Println()
 				os.Exit(1)
 			}
 

cmd/xdpderper/xdpderper.go

@@ -18,6 +18,9 @@ import (
 	"tailscale.com/derp/xdp"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tsweb"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

envknob/envknob.go

@@ -417,6 +417,29 @@ func App() string {
 	return ""
 }
 
+// IsCertShareReadOnlyMode returns true if this replica should never attempt to
+// issue or renew TLS credentials for any of the HTTPS endpoints that it is
+// serving. It should only return certs found in its cert store.  Currently,
+// this is used by the Kubernetes Operator's HA Ingress via VIPServices, where
+// multiple Ingress proxy instances serve the same HTTPS endpoint with a shared
+// TLS credentials. The TLS credentials should only be issued by one of the
+// replicas.
+// For HTTPS Ingress the operator and containerboot ensure
+// that read-only replicas will not be serving the HTTPS endpoints before there
+// is a shared cert available.
+func IsCertShareReadOnlyMode() bool {
+	m := String("TS_CERT_SHARE_MODE")
+	return m == "ro"
+}
+
+// IsCertShareReadWriteMode returns true if this instance is the replica
+// responsible for issuing and renewing TLS certs in an HA setup with certs
+// shared between multiple replicas.
+func IsCertShareReadWriteMode() bool {
+	m := String("TS_CERT_SHARE_MODE")
+	return m == "rw"
+}
+
 // CrashOnUnexpected reports whether the Tailscale client should panic
 // on unexpected conditions. If TS_DEBUG_CRASH_ON_UNEXPECTED is set, that's
 // used. Otherwise the default value is true for unstable builds.

go.mod

@@ -97,7 +97,7 @@ require (
 	golang.org/x/crypto v0.35.0
 	golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
 	golang.org/x/mod v0.23.0
-	golang.org/x/net v0.35.0
+	golang.org/x/net v0.36.0
 	golang.org/x/oauth2 v0.26.0
 	golang.org/x/sync v0.11.0
 	golang.org/x/sys v0.30.0

go.sum

@@ -1135,8 +1135,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
+golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

internal/client/tailscale/vip_service.go

@@ -27,6 +27,8 @@ type VIPService struct {
 	Addrs []string `json:"addrs,omitempty"`
 	// Comment is an optional text string for display in the admin panel.
 	Comment string `json:"comment,omitempty"`
+	// Annotations are optional key-value pairs that can be used to store arbitrary metadata.
+	Annotations map[string]string `json:"annotations,omitempty"`
 	// Ports are the ports of a VIPService that will be configured via Tailscale serve config.
 	// If set, any node wishing to advertise this VIPService must have this port configured via Tailscale serve.
 	Ports []string `json:"ports,omitempty"`

ipn/conf.go

@@ -145,9 +145,15 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) {
 		mp.AppConnector = *c.AppConnector
 		mp.AppConnectorSet = true
 	}
+	// Configfile should be the source of truth for whether this node
+	// advertises any services.  We need to ensure that each reload updates
+	// currently advertised services as else the transition from 'some
+	// services are advertised' to 'advertised services are empty/unset in
+	// conffile' would have no effect (especially given that an empty
+	// service slice would be omitted from the JSON config).
+	mp.AdvertiseServicesSet = true
 	if c.AdvertiseServices != nil {
 		mp.AdvertiseServices = c.AdvertiseServices
-		mp.AdvertiseServicesSet = true
 	}
 	return mp, nil
 }

ipn/ipnlocal/cert.go

@@ -119,6 +119,9 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string
 	}
 
 	if pair, err := getCertPEMCached(cs, domain, now); err == nil {
+		if envknob.IsCertShareReadOnlyMode() {
+			return pair, nil
+		}
 		// If we got here, we have a valid unexpired cert.
 		// Check whether we should start an async renewal.
 		shouldRenew, err := b.shouldStartDomainRenewal(cs, domain, now, pair, minValidity)
@@ -134,7 +137,7 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string
 		if minValidity == 0 {
 			logf("starting async renewal")
 			// Start renewal in the background, return current valid cert.
-			go b.getCertPEM(context.Background(), cs, logf, traceACME, domain, now, minValidity)
+			b.goTracker.Go(func() { getCertPEM(context.Background(), b, cs, logf, traceACME, domain, now, minValidity) })
 			return pair, nil
 		}
 		// If the caller requested a specific validity duration, fall through
@@ -142,7 +145,11 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string
 		logf("starting sync renewal")
 	}
 
-	pair, err := b.getCertPEM(ctx, cs, logf, traceACME, domain, now, minValidity)
+	if envknob.IsCertShareReadOnlyMode() {
+		return nil, fmt.Errorf("retrieving cached TLS certificate failed and cert store is configured in read-only mode, not attempting to issue a new certificate: %w", err)
+	}
+
+	pair, err := getCertPEM(ctx, b, cs, logf, traceACME, domain, now, minValidity)
 	if err != nil {
 		logf("getCertPEM: %v", err)
 		return nil, err
@@ -358,7 +365,29 @@ type certStateStore struct {
 	testRoots *x509.CertPool
 }
 
+// TLSCertKeyReader is an interface implemented by state stores where it makes
+// sense to read the TLS cert and key in a single operation that can be
+// distinguished from generic state value reads. Currently this is only implemented
+// by the kubestore.Store, which, in some cases, need to read cert and key from a
+// non-cached TLS Secret.
+type TLSCertKeyReader interface {
+	ReadTLSCertAndKey(domain string) ([]byte, []byte, error)
+}
+
 func (s certStateStore) Read(domain string, now time.Time) (*TLSCertKeyPair, error) {
+	// If we're using a store that supports atomic reads, use that
+	if kr, ok := s.StateStore.(TLSCertKeyReader); ok {
+		cert, key, err := kr.ReadTLSCertAndKey(domain)
+		if err != nil {
+			return nil, err
+		}
+		if !validCertPEM(domain, key, cert, s.testRoots, now) {
+			return nil, errCertExpired
+		}
+		return &TLSCertKeyPair{CertPEM: cert, KeyPEM: key, Cached: true}, nil
+	}
+
+	// Otherwise fall back to separate reads
 	certPEM, err := s.ReadState(ipn.StateKey(domain + ".crt"))
 	if err != nil {
 		return nil, err
@@ -446,7 +475,9 @@ func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKey
 	return cs.Read(domain, now)
 }
 
-func (b *LocalBackend) getCertPEM(ctx context.Context, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*TLSCertKeyPair, error) {
+// getCertPem checks if a cert needs to be renewed and if so, renews it.
+// It can be overridden in tests.
+var getCertPEM = func(ctx context.Context, b *LocalBackend, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*TLSCertKeyPair, error) {
 	acmeMu.Lock()
 	defer acmeMu.Unlock()
 

ipn/ipnlocal/cert_test.go

@@ -6,6 +6,7 @@
 package ipnlocal
 
 import (
+	"context"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
@@ -14,11 +15,17 @@ import (
 	"embed"
 	"encoding/pem"
 	"math/big"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tstest"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
 )
 
 func TestValidLookingCertDomain(t *testing.T) {
@@ -221,3 +228,151 @@ func TestDebugACMEDirectoryURL(t *testing.T) {
 		})
 	}
 }
+
+func TestGetCertPEMWithValidity(t *testing.T) {
+	const testDomain = "example.com"
+	b := &LocalBackend{
+		store:   &mem.Store{},
+		varRoot: t.TempDir(),
+		ctx:     context.Background(),
+		logf:    t.Logf,
+	}
+	certDir, err := b.certDir()
+	if err != nil {
+		t.Fatalf("certDir error: %v", err)
+	}
+	if _, err := b.getCertStore(); err != nil {
+		t.Fatalf("getCertStore error: %v", err)
+	}
+	testRoot, err := certTestFS.ReadFile("testdata/rootCA.pem")
+	if err != nil {
+		t.Fatal(err)
+	}
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(testRoot) {
+		t.Fatal("Unable to add test CA to the cert pool")
+	}
+	testX509Roots = roots
+	defer func() { testX509Roots = nil }()
+	tests := []struct {
+		name string
+		now  time.Time
+		// storeCerts is true if the test cert and key should be written to store.
+		storeCerts       bool
+		readOnlyMode     bool // TS_READ_ONLY_CERTS env var
+		wantAsyncRenewal bool // async issuance should be started
+		wantIssuance     bool // sync issuance should be started
+		wantErr          bool
+	}{
+		{
+			name:             "valid_no_renewal",
+			now:              time.Date(2023, time.February, 20, 0, 0, 0, 0, time.UTC),
+			storeCerts:       true,
+			wantAsyncRenewal: false,
+			wantIssuance:     false,
+			wantErr:          false,
+		},
+		{
+			name:             "issuance_needed",
+			now:              time.Date(2023, time.February, 20, 0, 0, 0, 0, time.UTC),
+			storeCerts:       false,
+			wantAsyncRenewal: false,
+			wantIssuance:     true,
+			wantErr:          false,
+		},
+		{
+			name:             "renewal_needed",
+			now:              time.Date(2025, time.May, 1, 0, 0, 0, 0, time.UTC),
+			storeCerts:       true,
+			wantAsyncRenewal: true,
+			wantIssuance:     false,
+			wantErr:          false,
+		},
+		{
+			name:             "renewal_needed_read_only_mode",
+			now:              time.Date(2025, time.May, 1, 0, 0, 0, 0, time.UTC),
+			storeCerts:       true,
+			readOnlyMode:     true,
+			wantAsyncRenewal: false,
+			wantIssuance:     false,
+			wantErr:          false,
+		},
+		{
+			name:             "no_certs_read_only_mode",
+			now:              time.Date(2025, time.May, 1, 0, 0, 0, 0, time.UTC),
+			storeCerts:       false,
+			readOnlyMode:     true,
+			wantAsyncRenewal: false,
+			wantIssuance:     false,
+			wantErr:          true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			if tt.readOnlyMode {
+				envknob.Setenv("TS_CERT_SHARE_MODE", "ro")
+			}
+
+			os.RemoveAll(certDir)
+			if tt.storeCerts {
+				os.MkdirAll(certDir, 0755)
+				if err := os.WriteFile(filepath.Join(certDir, "example.com.crt"),
+					must.Get(os.ReadFile("testdata/example.com.pem")), 0644); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(filepath.Join(certDir, "example.com.key"),
+					must.Get(os.ReadFile("testdata/example.com-key.pem")), 0644); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			b.clock = tstest.NewClock(tstest.ClockOpts{Start: tt.now})
+
+			allDone := make(chan bool, 1)
+			defer b.goTracker.AddDoneCallback(func() {
+				b.mu.Lock()
+				defer b.mu.Unlock()
+				if b.goTracker.RunningGoroutines() > 0 {
+					return
+				}
+				select {
+				case allDone <- true:
+				default:
+				}
+			})()
+
+			// Set to true if get getCertPEM is called. GetCertPEM can be called in a goroutine for async
+			// renewal or in the main goroutine if issuance is required to obtain valid TLS credentials.
+			getCertPemWasCalled := false
+			getCertPEM = func(ctx context.Context, b *LocalBackend, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*TLSCertKeyPair, error) {
+				getCertPemWasCalled = true
+				return nil, nil
+			}
+			prevGoRoutines := b.goTracker.StartedGoroutines()
+			_, err = b.GetCertPEMWithValidity(context.Background(), testDomain, 0)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("b.GetCertPemWithValidity got err %v, wants error: '%v'", err, tt.wantErr)
+			}
+			// GetCertPEMWithValidity calls getCertPEM in a goroutine if async renewal is needed. That's the
+			// only goroutine it starts, so this can be used to test if async renewal was started.
+			gotAsyncRenewal := b.goTracker.StartedGoroutines()-prevGoRoutines != 0
+			if gotAsyncRenewal {
+				select {
+				case <-time.After(5 * time.Second):
+					t.Fatal("timed out waiting for goroutines to finish")
+				case <-allDone:
+				}
+			}
+			// Verify that async renewal was triggered if expected.
+			if tt.wantAsyncRenewal != gotAsyncRenewal {
+				t.Fatalf("wants getCertPem to be called async: %v, got called %v", tt.wantAsyncRenewal, gotAsyncRenewal)
+			}
+			// Verify that (non-async) issuance was started if expected.
+			gotIssuance := getCertPemWasCalled && !gotAsyncRenewal
+			if tt.wantIssuance != gotIssuance {
+				t.Errorf("wants getCertPem to be called: %v, got called %v", tt.wantIssuance, gotIssuance)
+			}
+		})
+	}
+}

ipn/ipnlocal/local.go

@@ -2380,12 +2380,10 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 	b.applyPrefsToHostinfoLocked(hostinfo, prefs)
 
-	b.setNetMapLocked(nil)
 	persistv := prefs.Persist().AsStruct()
 	if persistv == nil {
 		persistv = new(persist.Persist)
 	}
-	b.updateFilterLocked(nil, ipn.PrefsView{})
 
 	if b.portpoll != nil {
 		b.portpollOnce.Do(func() {
@@ -3481,18 +3479,20 @@ func (b *LocalBackend) onTailnetDefaultAutoUpdate(au bool) {
 		// can still manually enable auto-updates on this node.
 		return
 	}
-	b.logf("using tailnet default auto-update setting: %v", au)
-	prefsClone := prefs.AsStruct()
-	prefsClone.AutoUpdate.Apply = opt.NewBool(au)
-	_, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{
-		Prefs: *prefsClone,
-		AutoUpdateSet: ipn.AutoUpdatePrefsMask{
-			ApplySet: true,
-		},
-	}, unlock)
-	if err != nil {
-		b.logf("failed to apply tailnet-wide default for auto-updates (%v): %v", au, err)
-		return
+	if clientupdate.CanAutoUpdate() {
+		b.logf("using tailnet default auto-update setting: %v", au)
+		prefsClone := prefs.AsStruct()
+		prefsClone.AutoUpdate.Apply = opt.NewBool(au)
+		_, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{
+			Prefs: *prefsClone,
+			AutoUpdateSet: ipn.AutoUpdatePrefsMask{
+				ApplySet: true,
+			},
+		}, unlock)
+		if err != nil {
+			b.logf("failed to apply tailnet-wide default for auto-updates (%v): %v", au, err)
+			return
+		}
 	}
 }
 
@@ -5920,6 +5920,9 @@ func (b *LocalBackend) requestEngineStatusAndWait() {
 	b.logf("requestEngineStatusAndWait: got status update.")
 }
 
+// [controlclient.Auto] implements [auditlog.Transport].
+var _ auditlog.Transport = (*controlclient.Auto)(nil)
+
 // setControlClientLocked sets the control client to cc,
 // which may be nil.
 //
@@ -5927,12 +5930,12 @@ func (b *LocalBackend) requestEngineStatusAndWait() {
 func (b *LocalBackend) setControlClientLocked(cc controlclient.Client) {
 	b.cc = cc
 	b.ccAuto, _ = cc.(*controlclient.Auto)
-	if b.auditLogger != nil {
+	if t, ok := b.cc.(auditlog.Transport); ok && b.auditLogger != nil {
 		if err := b.auditLogger.SetProfileID(b.pm.CurrentProfile().ID()); err != nil {
 			b.logf("audit logger set profile ID failure: %v", err)
 		}
 
-		if err := b.auditLogger.Start(b.ccAuto); err != nil {
+		if err := b.auditLogger.Start(t); err != nil {
 			b.logf("audit logger start failure: %v", err)
 		}
 	}
@@ -7531,6 +7534,7 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err
 		return nil
 	}
 	b.setNetMapLocked(nil) // Reset netmap.
+	b.updateFilterLocked(nil, ipn.PrefsView{})
 	// Reset the NetworkMap in the engine
 	b.e.SetNetworkMap(new(netmap.NetworkMap))
 	if prevCC := b.resetControlClientLocked(); prevCC != nil {

ipn/ipnlocal/local_test.go

@@ -44,6 +44,7 @@ import (
 	"tailscale.com/tsd"
 	"tailscale.com/tstest"
 	"tailscale.com/types/dnstype"
+	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/logid"
@@ -60,6 +61,7 @@ import (
 	"tailscale.com/util/syspolicy/source"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
+	"tailscale.com/wgengine/filter/filtertype"
 	"tailscale.com/wgengine/wgcfg"
 )
 
@@ -1508,6 +1510,15 @@ func TestReconfigureAppConnector(t *testing.T) {
 func TestBackfillAppConnectorRoutes(t *testing.T) {
 	// Create backend with an empty app connector.
 	b := newTestBackend(t)
+	// newTestBackend creates a backend with a non-nil netmap,
+	// but this test requires a nil netmap.
+	// Otherwise, instead of backfilling, [LocalBackend.reconfigAppConnectorLocked]
+	// uses the domains and routes from netmap's [appctype.AppConnectorAttr].
+	// Additionally, a non-nil netmap makes reconfigAppConnectorLocked
+	// asynchronous, resulting in a flaky test.
+	// Therefore, we set the netmap to nil to simulate a fresh backend start
+	// or a profile switch where the netmap is not yet available.
+	b.setNetMapLocked(nil)
 	if err := b.Start(ipn.Options{}); err != nil {
 		t.Fatal(err)
 	}
@@ -4743,32 +4754,133 @@ func TestLoginNotifications(t *testing.T) {
 // TestConfigFileReload tests that the LocalBackend reloads its configuration
 // when the configuration file changes.
 func TestConfigFileReload(t *testing.T) {
-	cfg1 := `{"Hostname": "foo", "Version": "alpha0"}`
-	f := filepath.Join(t.TempDir(), "cfg")
-	must.Do(os.WriteFile(f, []byte(cfg1), 0600))
-	sys := new(tsd.System)
-	sys.InitialConfig = must.Get(conffile.Load(f))
-	lb := newTestLocalBackendWithSys(t, sys)
-	must.Do(lb.Start(ipn.Options{}))
-
-	lb.mu.Lock()
-	hn := lb.hostinfo.Hostname
-	lb.mu.Unlock()
-	if hn != "foo" {
-		t.Fatalf("got %q; want %q", hn, "foo")
+	type testCase struct {
+		name    string
+		initial *conffile.Config
+		updated *conffile.Config
+		checkFn func(*testing.T, *LocalBackend)
 	}
 
-	cfg2 := `{"Hostname": "bar", "Version": "alpha0"}`
-	must.Do(os.WriteFile(f, []byte(cfg2), 0600))
-	if !must.Get(lb.ReloadConfig()) {
-		t.Fatal("reload failed")
+	tests := []testCase{
+		{
+			name: "hostname_change",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:  "alpha0",
+					Hostname: ptr.To("initial-host"),
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:  "alpha0",
+					Hostname: ptr.To("updated-host"),
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if got := b.Prefs().Hostname(); got != "updated-host" {
+					t.Errorf("hostname = %q; want updated-host", got)
+				}
+			},
+		},
+		{
+			name: "start_advertising_services",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version: "alpha0",
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc", "svc:def"},
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if got := b.Prefs().AdvertiseServices().AsSlice(); !reflect.DeepEqual(got, []string{"svc:abc", "svc:def"}) {
+					t.Errorf("AdvertiseServices = %v; want [svc:abc, svc:def]", got)
+				}
+			},
+		},
+		{
+			name: "change_advertised_services",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc", "svc:def"},
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc", "svc:ghi"},
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if got := b.Prefs().AdvertiseServices().AsSlice(); !reflect.DeepEqual(got, []string{"svc:abc", "svc:ghi"}) {
+					t.Errorf("AdvertiseServices = %v; want [svc:abc, svc:ghi]", got)
+				}
+			},
+		},
+		{
+			name: "unset_advertised_services",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc"},
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version: "alpha0",
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if b.Prefs().AdvertiseServices().Len() != 0 {
+					t.Errorf("got %d AdvertiseServices wants none", b.Prefs().AdvertiseServices().Len())
+				}
+			},
+		},
 	}
 
-	lb.mu.Lock()
-	hn = lb.hostinfo.Hostname
-	lb.mu.Unlock()
-	if hn != "bar" {
-		t.Fatalf("got %q; want %q", hn, "bar")
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "tailscale.conf")
+
+			// Write initial config
+			initialJSON, err := json.Marshal(tc.initial.Parsed)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(path, initialJSON, 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create backend with initial config
+			tc.initial.Path = path
+			tc.initial.Raw = initialJSON
+			sys := &tsd.System{
+				InitialConfig: tc.initial,
+			}
+			b := newTestLocalBackendWithSys(t, sys)
+
+			// Update config file
+			updatedJSON, err := json.Marshal(tc.updated.Parsed)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(path, updatedJSON, 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			// Trigger reload
+			if ok, err := b.ReloadConfig(); !ok || err != nil {
+				t.Fatalf("ReloadConfig() = %v, %v; want true, nil", ok, err)
+			}
+
+			// Check outcome
+			tc.checkFn(t, b)
+		})
 	}
 }
 
@@ -5206,3 +5318,60 @@ func TestUpdateIngressLocked(t *testing.T) {
 		})
 	}
 }
+
+// TestSrcCapPacketFilter tests that LocalBackend handles packet filters with
+// SrcCaps instead of Srcs (IPs)
+func TestSrcCapPacketFilter(t *testing.T) {
+	lb := newLocalBackendWithTestControl(t, false, func(tb testing.TB, opts controlclient.Options) controlclient.Client {
+		return newClient(tb, opts)
+	})
+	if err := lb.Start(ipn.Options{}); err != nil {
+		t.Fatalf("(*LocalBackend).Start(): %v", err)
+	}
+
+	var k key.NodePublic
+	must.Do(k.UnmarshalText([]byte("nodekey:5c8f86d5fc70d924e55f02446165a5dae8f822994ad26bcf4b08fd841f9bf261")))
+
+	controlClient := lb.cc.(*mockControl)
+	controlClient.send(nil, "", false, &netmap.NetworkMap{
+		SelfNode: (&tailcfg.Node{
+			Addresses: []netip.Prefix{netip.MustParsePrefix("1.1.1.1/32")},
+		}).View(),
+		Peers: []tailcfg.NodeView{
+			(&tailcfg.Node{
+				Addresses: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")},
+				ID:        2,
+				Key:       k,
+				CapMap:    tailcfg.NodeCapMap{"cap-X": nil}, // node 2 has cap
+			}).View(),
+			(&tailcfg.Node{
+				Addresses: []netip.Prefix{netip.MustParsePrefix("3.3.3.3/32")},
+				ID:        3,
+				Key:       k,
+				CapMap:    tailcfg.NodeCapMap{}, // node 3 does not have the cap
+			}).View(),
+		},
+		PacketFilter: []filtertype.Match{{
+			IPProto: views.SliceOf([]ipproto.Proto{ipproto.TCP}),
+			SrcCaps: []tailcfg.NodeCapability{"cap-X"}, // cap in packet filter rule
+			Dsts: []filtertype.NetPortRange{{
+				Net: netip.MustParsePrefix("1.1.1.1/32"),
+				Ports: filtertype.PortRange{
+					First: 22,
+					Last:  22,
+				},
+			}},
+		}},
+	})
+
+	f := lb.GetFilterForTest()
+	res := f.Check(netip.MustParseAddr("2.2.2.2"), netip.MustParseAddr("1.1.1.1"), 22, ipproto.TCP)
+	if res != filter.Accept {
+		t.Errorf("Check(2.2.2.2, ...) = %s, want %s", res, filter.Accept)
+	}
+
+	res = f.Check(netip.MustParseAddr("3.3.3.3"), netip.MustParseAddr("1.1.1.1"), 22, ipproto.TCP)
+	if !res.IsDrop() {
+		t.Error("IsDrop() for node without cap = false, want true")
+	}
+}

ipn/ipnlocal/state_test.go

@@ -735,12 +735,10 @@ func TestStateMachine(t *testing.T) {
 		// b.Shutdown() explicitly ourselves.
 		previousCC.assertShutdown(false)
 
-		// Note: unpause happens because ipn needs to get at least one netmap
-		//  on startup, otherwise UIs can't show the node list, login
-		//  name, etc when in state ipn.Stopped.
-		//  Arguably they shouldn't try. But they currently do.
 		nn := notifies.drain(2)
-		cc.assertCalls("New", "Login")
+		// We already have a netmap for this node,
+		// and WantRunning is false, so cc should be paused.
+		cc.assertCalls("New", "Login", "pause")
 		c.Assert(nn[0].Prefs, qt.IsNotNil)
 		c.Assert(nn[1].State, qt.IsNotNil)
 		c.Assert(nn[0].Prefs.WantRunning(), qt.IsFalse)
@@ -751,7 +749,11 @@ func TestStateMachine(t *testing.T) {
 	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
 	// the first netmap. Simulate that netmap being received, after which
 	// it should pause us, to avoid wasting CPU retrieving unnecessarily
-	// additional netmap updates.
+	// additional netmap updates. Since our LocalBackend instance already
+	// has a netmap, we will reset it to nil to simulate the first netmap
+	// retrieval.
+	b.setNetMapLocked(nil)
+	cc.assertCalls("unpause")
 	//
 	// TODO: really the various GUIs and prefs should be refactored to
 	//  not require the netmap structure at all when starting while
@@ -853,7 +855,7 @@ func TestStateMachine(t *testing.T) {
 	// The last test case is the most common one: restarting when both
 	// logged in and WantRunning.
 	t.Logf("\n\nStart5")
-	notifies.expect(1)
+	notifies.expect(2)
 	c.Assert(b.Start(ipn.Options{}), qt.IsNil)
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
@@ -861,30 +863,32 @@ func TestStateMachine(t *testing.T) {
 		previousCC.assertShutdown(false)
 		cc.assertCalls("New", "Login")
 
-		nn := notifies.drain(1)
+		nn := notifies.drain(2)
 		cc.assertCalls()
 		c.Assert(nn[0].Prefs, qt.IsNotNil)
 		c.Assert(nn[0].Prefs.LoggedOut(), qt.IsFalse)
 		c.Assert(nn[0].Prefs.WantRunning(), qt.IsTrue)
-		c.Assert(b.State(), qt.Equals, ipn.NoState)
+		// We're logged in and have a valid netmap, so we should
+		// be in the Starting state.
+		c.Assert(nn[1].State, qt.IsNotNil)
+		c.Assert(*nn[1].State, qt.Equals, ipn.Starting)
+		c.Assert(b.State(), qt.Equals, ipn.Starting)
 	}
 
 	// Control server accepts our valid key from before.
 	t.Logf("\n\nLoginFinished5")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		SelfNode: (&tailcfg.Node{MachineAuthorized: true}).View(),
 	})
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		cc.assertCalls()
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
-		c.Assert(nn[0].State, qt.IsNotNil)
-		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 		// NOTE: No prefs change this time. WantRunning stays true.
 		// We were in Starting in the first place, so that doesn't
-		// change either.
+		// change either, so we don't expect any notifications.
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
 	t.Logf("\n\nExpireKey")

ipn/store/kubestore/store_kube.go

@@ -13,11 +13,14 @@ import (
 	"strings"
 	"time"
 
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
 )
 
@@ -32,21 +35,37 @@ const (
 	reasonTailscaleStateLoadFailed   = "TailscaleStateLoadFailed"
 	eventTypeWarning                 = "Warning"
 	eventTypeNormal                  = "Normal"
+
+	keyTLSCert = "tls.crt"
+	keyTLSKey  = "tls.key"
 )
 
 // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
 type Store struct {
-	client     kubeclient.Client
-	canPatch   bool
-	secretName string
+	client        kubeclient.Client
+	canPatch      bool
+	secretName    string // state Secret
+	certShareMode string // 'ro', 'rw', or empty
+	podName       string
 
-	// memory holds the latest tailscale state. Writes write state to a kube Secret and memory, Reads read from
-	// memory.
+	// memory holds the latest tailscale state. Writes write state to a kube
+	// Secret and memory, Reads read from memory.
 	memory mem.Store
 }
 
-// New returns a new Store that persists to the named Secret.
-func New(_ logger.Logf, secretName string) (*Store, error) {
+// New returns a new Store that persists state to Kubernets Secret(s).
+// Tailscale state is stored in a Secret named by the secretName parameter.
+// TLS certs are stored and retrieved from state Secret or separate Secrets
+// named after TLS endpoints if running in cert share mode.
+func New(logf logger.Logf, secretName string) (*Store, error) {
+	c, err := newClient()
+	if err != nil {
+		return nil, err
+	}
+	return newWithClient(logf, c, secretName)
+}
+
+func newClient() (kubeclient.Client, error) {
 	c, err := kubeclient.New("tailscale-state-store")
 	if err != nil {
 		return nil, err
@@ -55,6 +74,10 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 		// Derive the API server address from the environment variables
 		c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
 	}
+	return c, nil
+}
+
+func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*Store, error) {
 	canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
 	if err != nil {
 		return nil, err
@@ -63,11 +86,30 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 		client:     c,
 		canPatch:   canPatch,
 		secretName: secretName,
+		podName:    os.Getenv("POD_NAME"),
 	}
+	if envknob.IsCertShareReadWriteMode() {
+		s.certShareMode = "rw"
+	} else if envknob.IsCertShareReadOnlyMode() {
+		s.certShareMode = "ro"
+	}
+
 	// Load latest state from kube Secret if it already exists.
 	if err := s.loadState(); err != nil && err != ipn.ErrStateNotExist {
 		return nil, fmt.Errorf("error loading state from kube Secret: %w", err)
 	}
+	// If we are in cert share mode, pre-load existing shared certs.
+	if s.certShareMode == "rw" || s.certShareMode == "ro" {
+		sel := s.certSecretSelector()
+		if err := s.loadCerts(context.Background(), sel); err != nil {
+			// We will attempt to again retrieve the certs from Secrets when a request for an HTTPS endpoint
+			// is received.
+			log.Printf("[unexpected] error loading TLS certs: %v", err)
+		}
+	}
+	if s.certShareMode == "ro" {
+		go s.runCertReload(context.Background(), logf)
+	}
 	return s, nil
 }
 
@@ -84,27 +126,101 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
 
 // WriteState implements the StateStore interface.
 func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
-	return s.updateStateSecret(map[string][]byte{string(id): bs})
-}
-
-// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields of a Tailscale Kubernetes node's state
-// Secret.
-func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) error {
-	return s.updateStateSecret(map[string][]byte{domain + ".crt": cert, domain + ".key": key})
-}
-
-func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer func() {
 		if err == nil {
-			for id, bs := range data {
-				// The in-memory store does not distinguish between values read from state Secret on
-				// init and values written to afterwards. Values read from the state
-				// Secret will always be sanitized, so we also need to sanitize values written to store
-				// later, so that the Read logic can just lookup keys in sanitized form.
-				s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
-			}
+			s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
 		}
+	}()
+	return s.updateSecret(map[string][]byte{string(id): bs}, s.secretName)
+}
+
+// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields
+// of a Tailscale Kubernetes node's state Secret.
+func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) {
+	if s.certShareMode == "ro" {
+		log.Printf("[unexpected] TLS cert and key write in read-only mode")
+	}
+	if err := dnsname.ValidHostname(domain); err != nil {
+		return fmt.Errorf("invalid domain name %q: %w", domain, err)
+	}
+	defer func() {
+		// TODO(irbekrm): a read between these two separate writes would
+		// get a mismatched cert and key.  Allow writing both cert and
+		// key to the memory store in a single, lock-protected operation.
+		if err == nil {
+			s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
+			s.memory.WriteState(ipn.StateKey(domain+".key"), key)
+		}
+	}()
+	secretName := s.secretName
+	data := map[string][]byte{
+		domain + ".crt": cert,
+		domain + ".key": key,
+	}
+	// If we run in cert share mode, cert and key for a DNS name are written
+	// to a separate Secret.
+	if s.certShareMode == "rw" {
+		secretName = domain
+		data = map[string][]byte{
+			keyTLSCert: cert,
+			keyTLSKey:  key,
+		}
+	}
+	return s.updateSecret(data, secretName)
+}
+
+// ReadTLSCertAndKey reads a TLS cert and key from memory or from a
+// domain-specific Secret. It first checks the in-memory store, if not found in
+// memory and running cert store in read-only mode, looks up a Secret.
+func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
+	if err := dnsname.ValidHostname(domain); err != nil {
+		return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
+	}
+	certKey := domain + ".crt"
+	keyKey := domain + ".key"
+
+	cert, err = s.memory.ReadState(ipn.StateKey(certKey))
+	if err == nil {
+		key, err = s.memory.ReadState(ipn.StateKey(keyKey))
+		if err == nil {
+			return cert, key, nil
+		}
+	}
+	if s.certShareMode != "ro" {
+		return nil, nil, ipn.ErrStateNotExist
+	}
+	// If we are in cert share read only mode, it is possible that a write
+	// replica just issued the TLS cert for this DNS name and it has not
+	// been loaded to store yet, so check the Secret.
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, domain)
+	if err != nil {
+		if kubeclient.IsNotFoundErr(err) {
+			// TODO(irbekrm): we should return a more specific error
+			// that wraps ipn.ErrStateNotExist here.
+			return nil, nil, ipn.ErrStateNotExist
+		}
+		return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
+	}
+	cert = secret.Data[keyTLSCert]
+	key = secret.Data[keyTLSKey]
+	if len(cert) == 0 || len(key) == 0 {
+		return nil, nil, ipn.ErrStateNotExist
+	}
+	// TODO(irbekrm): a read between these two separate writes would
+	// get a mismatched cert and key.  Allow writing both cert and
+	// key to the memory store in a single lock-protected operation.
+	s.memory.WriteState(ipn.StateKey(certKey), cert)
+	s.memory.WriteState(ipn.StateKey(keyKey), key)
+	return cert, key, nil
+}
+
+func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer func() {
 		if err != nil {
 			if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil {
 				log.Printf("kubestore: error creating tailscaled state update Event: %v", err)
@@ -116,17 +232,17 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 		}
 		cancel()
 	}()
-	secret, err := s.client.GetSecret(ctx, s.secretName)
+	secret, err := s.client.GetSecret(ctx, secretName)
 	if err != nil {
 		// If the Secret does not exist, create it with the required data.
-		if kubeclient.IsNotFoundErr(err) {
+		if kubeclient.IsNotFoundErr(err) && s.canCreateSecret(secretName) {
 			return s.client.CreateSecret(ctx, &kubeapi.Secret{
 				TypeMeta: kubeapi.TypeMeta{
 					APIVersion: "v1",
 					Kind:       "Secret",
 				},
 				ObjectMeta: kubeapi.ObjectMeta{
-					Name: s.secretName,
+					Name: secretName,
 				},
 				Data: func(m map[string][]byte) map[string][]byte {
 					d := make(map[string][]byte, len(m))
@@ -137,9 +253,9 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 				}(data),
 			})
 		}
-		return err
+		return fmt.Errorf("error getting Secret %s: %w", secretName, err)
 	}
-	if s.canPatch {
+	if s.canPatchSecret(secretName) {
 		var m []kubeclient.JSONPatch
 		// If the user has pre-created a Secret with no data, we need to ensure the top level /data field.
 		if len(secret.Data) == 0 {
@@ -166,8 +282,8 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 				})
 			}
 		}
-		if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
-			return fmt.Errorf("error patching Secret %s: %w", s.secretName, err)
+		if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
+			return fmt.Errorf("error patching Secret %s: %w", secretName, err)
 		}
 		return nil
 	}
@@ -176,9 +292,9 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 		mak.Set(&secret.Data, sanitizeKey(key), val)
 	}
 	if err := s.client.UpdateSecret(ctx, secret); err != nil {
-		return err
+		return fmt.Errorf("error updating Secret %s: %w", s.secretName, err)
 	}
-	return err
+	return nil
 }
 
 func (s *Store) loadState() (err error) {
@@ -202,6 +318,96 @@ func (s *Store) loadState() (err error) {
 	return nil
 }
 
+// runCertReload relists and reloads all TLS certs for endpoints shared by this
+// node from Secrets other than the state Secret to ensure that renewed certs get eventually loaded.
+// It is not critical to reload a cert immediately after
+// renewal, so a daily check is acceptable.
+// Currently (3/2025) this is only used for the shared HA Ingress certs on 'read' replicas.
+// Note that if shared certs are not found in memory on an HTTPS request, we
+// do a Secret lookup, so this mechanism does not need to ensure that newly
+// added Ingresses' certs get loaded.
+func (s *Store) runCertReload(ctx context.Context, logf logger.Logf) {
+	ticker := time.NewTicker(time.Hour * 24)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			sel := s.certSecretSelector()
+			if err := s.loadCerts(ctx, sel); err != nil {
+				logf("[unexpected] error reloading TLS certs: %v", err)
+			}
+		}
+	}
+}
+
+// loadCerts lists all Secrets matching the provided selector and loads TLS
+// certs and keys from those.
+func (s *Store) loadCerts(ctx context.Context, sel map[string]string) error {
+	ss, err := s.client.ListSecrets(ctx, sel)
+	if err != nil {
+		return fmt.Errorf("error listing TLS Secrets: %w", err)
+	}
+	for _, secret := range ss.Items {
+		if !hasTLSData(&secret) {
+			continue
+		}
+		// Only load secrets that have valid domain names (ending in .ts.net)
+		if !strings.HasSuffix(secret.Name, ".ts.net") {
+			continue
+		}
+		s.memory.WriteState(ipn.StateKey(secret.Name)+".crt", secret.Data[keyTLSCert])
+		s.memory.WriteState(ipn.StateKey(secret.Name)+".key", secret.Data[keyTLSKey])
+	}
+	return nil
+}
+
+// canCreateSecret returns true if this node should be allowed to create the given
+// Secret in its namespace.
+func (s *Store) canCreateSecret(secret string) bool {
+	// Only allow creating the state Secret (and not TLS Secrets).
+	return secret == s.secretName
+}
+
+// canPatchSecret returns true if this node should be allowed to patch the given
+// Secret.
+func (s *Store) canPatchSecret(secret string) bool {
+	// For backwards compatibility reasons, setups where the proxies are not
+	// given PATCH permissions for state Secrets are allowed. For TLS
+	// Secrets, we should always have PATCH permissions.
+	if secret == s.secretName {
+		return s.canPatch
+	}
+	return true
+}
+
+// certSecretSelector returns a label selector that can be used to list all
+// Secrets that aren't Tailscale state Secrets and contain TLS certificates for
+// HTTPS endpoints that this node serves.
+// Currently (3/2025) this only applies to the Kubernetes Operator's ingress
+// ProxyGroup.
+func (s *Store) certSecretSelector() map[string]string {
+	if s.podName == "" {
+		return map[string]string{}
+	}
+	p := strings.LastIndex(s.podName, "-")
+	if p == -1 {
+		return map[string]string{}
+	}
+	pgName := s.podName[:p]
+	return map[string]string{
+		kubetypes.LabelSecretType:   "certs",
+		kubetypes.LabelManaged:      "true",
+		"tailscale.com/proxy-group": pgName,
+	}
+}
+
+// hasTLSData returns true if the provided Secret contains non-empty TLS cert and key.
+func hasTLSData(s *kubeapi.Secret) bool {
+	return len(s.Data[keyTLSCert]) != 0 && len(s.Data[keyTLSKey]) != 0
+}
+
 // sanitizeKey converts any value that can be converted to a string into a valid Kubernetes Secret key.
 // Valid characters are alphanumeric, -, _, and .
 // https://kubernetes.io/docs/concepts/configuration/secret/#restriction-names-data.

ipn/store/kubestore/store_kube_test.go

@@ -4,33 +4,37 @@
 package kubestore
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
 )
 
-func TestUpdateStateSecret(t *testing.T) {
+func TestWriteState(t *testing.T) {
 	tests := []struct {
 		name       string
 		initial    map[string][]byte
-		updates    map[string][]byte
+		key        ipn.StateKey
+		value      []byte
 		wantData   map[string][]byte
 		allowPatch bool
 	}{
 		{
-			name: "basic_update",
+			name: "basic_write",
 			initial: map[string][]byte{
 				"existing": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"foo": []byte("bar"),
-			},
+			key:   "foo",
+			value: []byte("bar"),
 			wantData: map[string][]byte{
 				"existing": []byte("old"),
 				"foo":      []byte("bar"),
@@ -42,35 +46,17 @@ func TestUpdateStateSecret(t *testing.T) {
 			initial: map[string][]byte{
 				"foo": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"foo": []byte("new"),
-			},
+			key:   "foo",
+			value: []byte("new"),
 			wantData: map[string][]byte{
 				"foo": []byte("new"),
 			},
 			allowPatch: true,
 		},
 		{
-			name: "multiple_updates",
-			initial: map[string][]byte{
-				"keep": []byte("keep"),
-			},
-			updates: map[string][]byte{
-				"foo": []byte("bar"),
-				"baz": []byte("qux"),
-			},
-			wantData: map[string][]byte{
-				"keep": []byte("keep"),
-				"foo":  []byte("bar"),
-				"baz":  []byte("qux"),
-			},
-			allowPatch: true,
-		},
-		{
-			name: "create_new_secret",
-			updates: map[string][]byte{
-				"foo": []byte("bar"),
-			},
+			name:  "create_new_secret",
+			key:   "foo",
+			value: []byte("bar"),
 			wantData: map[string][]byte{
 				"foo": []byte("bar"),
 			},
@@ -81,29 +67,23 @@ func TestUpdateStateSecret(t *testing.T) {
 			initial: map[string][]byte{
 				"foo": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"foo": []byte("new"),
-			},
+			key:   "foo",
+			value: []byte("new"),
 			wantData: map[string][]byte{
 				"foo": []byte("new"),
 			},
 			allowPatch: false,
 		},
 		{
-			name: "sanitize_keys",
+			name: "sanitize_key",
 			initial: map[string][]byte{
 				"clean-key": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"dirty@key": []byte("new"),
-				"also/bad":  []byte("value"),
-				"good.key":  []byte("keep"),
-			},
+			key:   "dirty@key",
+			value: []byte("new"),
 			wantData: map[string][]byte{
 				"clean-key": []byte("old"),
 				"dirty_key": []byte("new"),
-				"also_bad":  []byte("value"),
-				"good.key":  []byte("keep"),
 			},
 			allowPatch: true,
 		},
@@ -152,13 +132,13 @@ func TestUpdateStateSecret(t *testing.T) {
 			s := &Store{
 				client:     client,
 				canPatch:   tt.allowPatch,
-				secretName: "test-secret",
+				secretName: "ts-state",
 				memory:     mem.Store{},
 			}
 
-			err := s.updateStateSecret(tt.updates)
+			err := s.WriteState(tt.key, tt.value)
 			if err != nil {
-				t.Errorf("updateStateSecret() error = %v", err)
+				t.Errorf("WriteState() error = %v", err)
 				return
 			}
 
@@ -168,16 +148,576 @@ func TestUpdateStateSecret(t *testing.T) {
 			}
 
 			// Verify memory store was updated
-			for k, v := range tt.updates {
-				got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(k)))
+			got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(string(tt.key))))
+			if err != nil {
+				t.Errorf("reading from memory store: %v", err)
+			}
+			if !cmp.Equal(got, tt.value) {
+				t.Errorf("memory store key %q = %v, want %v", tt.key, got, tt.value)
+			}
+		})
+	}
+}
+
+func TestWriteTLSCertAndKey(t *testing.T) {
+	const (
+		testDomain = "my-app.tailnetxyz.ts.net"
+		testCert   = "fake-cert"
+		testKey    = "fake-key"
+	)
+
+	tests := []struct {
+		name            string
+		initial         map[string][]byte // pre-existing cert and key
+		certShareMode   string
+		allowPatch      bool   // whether client can patch the Secret
+		wantSecretName  string // name of the Secret where cert and key should be written
+		wantSecretData  map[string][]byte
+		wantMemoryStore map[ipn.StateKey][]byte
+	}{
+		{
+			name: "basic_write",
+			initial: map[string][]byte{
+				"existing": []byte("old"),
+			},
+			allowPatch:     true,
+			wantSecretName: "ts-state",
+			wantSecretData: map[string][]byte{
+				"existing":                     []byte("old"),
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:           "cert_share_mode_write",
+			certShareMode:  "rw",
+			allowPatch:     true,
+			wantSecretName: "my-app.tailnetxyz.ts.net",
+			wantSecretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name: "cert_share_mode_write_update_existing",
+			initial: map[string][]byte{
+				"tls.crt": []byte("old-cert"),
+				"tls.key": []byte("old-key"),
+			},
+			certShareMode:  "rw",
+			allowPatch:     true,
+			wantSecretName: "my-app.tailnetxyz.ts.net",
+			wantSecretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name: "update_existing",
+			initial: map[string][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte("old-cert"),
+				"my-app.tailnetxyz.ts.net.key": []byte("old-key"),
+			},
+			certShareMode:  "",
+			allowPatch:     true,
+			wantSecretName: "ts-state",
+			wantSecretData: map[string][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:           "patch_denied",
+			certShareMode:  "",
+			allowPatch:     false,
+			wantSecretName: "ts-state",
+			wantSecretData: map[string][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			// Set POD_NAME for testing selectors
+			envknob.Setenv("POD_NAME", "ingress-proxies-1")
+			defer envknob.Setenv("POD_NAME", "")
+
+			secret := tt.initial // track current state
+			client := &kubeclient.FakeClient{
+				GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
+					if secret == nil {
+						return nil, &kubeapi.Status{Code: 404}
+					}
+					return &kubeapi.Secret{Data: secret}, nil
+				},
+				CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
+					return tt.allowPatch, true, nil
+				},
+				CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
+					if s.Name != tt.wantSecretName {
+						t.Errorf("CreateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
+					}
+					secret = s.Data
+					return nil
+				},
+				UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
+					if s.Name != tt.wantSecretName {
+						t.Errorf("UpdateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
+					}
+					secret = s.Data
+					return nil
+				},
+				JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
+					if !tt.allowPatch {
+						return &kubeapi.Status{Reason: "Forbidden"}
+					}
+					if name != tt.wantSecretName {
+						t.Errorf("JSONPatchResource called with wrong name, got %q, want %q", name, tt.wantSecretName)
+					}
+					if secret == nil {
+						secret = make(map[string][]byte)
+					}
+					for _, p := range patches {
+						if p.Op == "add" && p.Path == "/data" {
+							secret = p.Value.(map[string][]byte)
+						} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
+							key := strings.TrimPrefix(p.Path, "/data/")
+							secret[key] = p.Value.([]byte)
+						}
+					}
+					return nil
+				},
+			}
+
+			s := &Store{
+				client:        client,
+				canPatch:      tt.allowPatch,
+				secretName:    tt.wantSecretName,
+				certShareMode: tt.certShareMode,
+				memory:        mem.Store{},
+			}
+
+			err := s.WriteTLSCertAndKey(testDomain, []byte(testCert), []byte(testKey))
+			if err != nil {
+				t.Errorf("WriteTLSCertAndKey() error = '%v'", err)
+				return
+			}
+
+			// Verify secret data
+			if diff := cmp.Diff(secret, tt.wantSecretData); diff != "" {
+				t.Errorf("secret data mismatch (-got +want):\n%s", diff)
+			}
+
+			// Verify memory store was updated
+			for key, want := range tt.wantMemoryStore {
+				got, err := s.memory.ReadState(key)
 				if err != nil {
 					t.Errorf("reading from memory store: %v", err)
 					continue
 				}
-				if !cmp.Equal(got, v) {
-					t.Errorf("memory store key %q = %v, want %v", k, got, v)
+				if !cmp.Equal(got, want) {
+					t.Errorf("memory store key %q = %v, want %v", key, got, want)
 				}
 			}
 		})
 	}
 }
+
+func TestReadTLSCertAndKey(t *testing.T) {
+	const (
+		testDomain = "my-app.tailnetxyz.ts.net"
+		testCert   = "fake-cert"
+		testKey    = "fake-key"
+	)
+
+	tests := []struct {
+		name          string
+		memoryStore   map[ipn.StateKey][]byte // pre-existing memory store state
+		certShareMode string
+		domain        string
+		secretData    map[string][]byte // data to return from mock GetSecret
+		secretGetErr  error             // error to return from mock GetSecret
+		wantCert      []byte
+		wantKey       []byte
+		wantErr       error
+		// what should end up in memory store after the store is created
+		wantMemoryStore map[ipn.StateKey][]byte
+	}{
+		{
+			name: "found",
+			memoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			domain:   testDomain,
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:    "not_found",
+			domain:  testDomain,
+			wantErr: ipn.ErrStateNotExist,
+		},
+		{
+			name:          "cert_share_ro_mode_found_in_secret",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:          "cert_share_ro_mode_found_in_memory",
+			certShareMode: "ro",
+			memoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			domain:   testDomain,
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:          "cert_share_ro_mode_not_found",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretGetErr:  &kubeapi.Status{Code: 404},
+			wantErr:       ipn.ErrStateNotExist,
+		},
+		{
+			name:          "cert_share_ro_mode_empty_cert_in_secret",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretData: map[string][]byte{
+				"tls.crt": {},
+				"tls.key": []byte(testKey),
+			},
+			wantErr: ipn.ErrStateNotExist,
+		},
+		{
+			name:          "cert_share_ro_mode_kube_api_error",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretGetErr:  fmt.Errorf("api error"),
+			wantErr:       fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			client := &kubeclient.FakeClient{
+				GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
+					if tt.secretGetErr != nil {
+						return nil, tt.secretGetErr
+					}
+					return &kubeapi.Secret{Data: tt.secretData}, nil
+				},
+			}
+
+			s := &Store{
+				client:        client,
+				secretName:    "ts-state",
+				certShareMode: tt.certShareMode,
+				memory:        mem.Store{},
+			}
+
+			// Initialize memory store
+			for k, v := range tt.memoryStore {
+				s.memory.WriteState(k, v)
+			}
+
+			gotCert, gotKey, err := s.ReadTLSCertAndKey(tt.domain)
+			if tt.wantErr != nil {
+				if err == nil {
+					t.Errorf("ReadTLSCertAndKey() error = nil, want error containing %v", tt.wantErr)
+					return
+				}
+				if !strings.Contains(err.Error(), tt.wantErr.Error()) {
+					t.Errorf("ReadTLSCertAndKey() error = %v, want error containing %v", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Errorf("ReadTLSCertAndKey() unexpected error: %v", err)
+				return
+			}
+
+			if !bytes.Equal(gotCert, tt.wantCert) {
+				t.Errorf("ReadTLSCertAndKey() gotCert = %v, want %v", gotCert, tt.wantCert)
+			}
+			if !bytes.Equal(gotKey, tt.wantKey) {
+				t.Errorf("ReadTLSCertAndKey() gotKey = %v, want %v", gotKey, tt.wantKey)
+			}
+
+			// Verify memory store contents after operation
+			if tt.wantMemoryStore != nil {
+				for key, want := range tt.wantMemoryStore {
+					got, err := s.memory.ReadState(key)
+					if err != nil {
+						t.Errorf("reading from memory store: %v", err)
+						continue
+					}
+					if !bytes.Equal(got, want) {
+						t.Errorf("memory store key %q = %v, want %v", key, got, want)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestNewWithClient(t *testing.T) {
+	const (
+		secretName = "ts-state"
+		testCert   = "fake-cert"
+		testKey    = "fake-key"
+	)
+
+	certSecretsLabels := map[string]string{
+		"tailscale.com/secret-type": "certs",
+		"tailscale.com/managed":     "true",
+		"tailscale.com/proxy-group": "ingress-proxies",
+	}
+
+	// Helper function to create Secret objects for testing
+	makeSecret := func(name string, labels map[string]string, certSuffix string) kubeapi.Secret {
+		return kubeapi.Secret{
+			ObjectMeta: kubeapi.ObjectMeta{
+				Name:   name,
+				Labels: labels,
+			},
+			Data: map[string][]byte{
+				"tls.crt": []byte(testCert + certSuffix),
+				"tls.key": []byte(testKey + certSuffix),
+			},
+		}
+	}
+
+	tests := []struct {
+		name                    string
+		stateSecretContents     map[string][]byte // data in state Secret
+		TLSSecrets              []kubeapi.Secret  // list of TLS cert Secrets
+		certMode                string
+		secretGetErr            error // error to return from GetSecret
+		secretsListErr          error // error to return from ListSecrets
+		wantMemoryStoreContents map[ipn.StateKey][]byte
+		wantErr                 error
+	}{
+		{
+			name:                    "empty_state_secret",
+			stateSecretContents:     map[string][]byte{},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{},
+		},
+		{
+			name:                    "state_secret_not_found",
+			secretGetErr:            &kubeapi.Status{Code: 404},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{},
+		},
+		{
+			name:         "state_secret_get_error",
+			secretGetErr: fmt.Errorf("some error"),
+			wantErr:      fmt.Errorf("error loading state from kube Secret: some error"),
+		},
+		{
+			name: "load_existing_state",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+				"baz": []byte("qux"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo": []byte("bar"),
+				"baz": []byte("qux"),
+			},
+		},
+		{
+			name:     "load_select_certs_in_read_only_mode",
+			certMode: "ro",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			TLSSecrets: []kubeapi.Secret{
+				makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
+				makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
+				makeSecret("some-other-secret", nil, "3"),
+				makeSecret("app3.other-proxies.ts.net", map[string]string{
+					"tailscale.com/secret-type": "certs",
+					"tailscale.com/managed":     "true",
+					"tailscale.com/proxy-group": "some-other-proxygroup",
+				}, "4"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo":                        []byte("bar"),
+				"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
+				"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
+				"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
+				"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
+			},
+		},
+		{
+			name:     "load_select_certs_in_read_write_mode",
+			certMode: "rw",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			TLSSecrets: []kubeapi.Secret{
+				makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
+				makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
+				makeSecret("some-other-secret", nil, "3"),
+				makeSecret("app3.other-proxies.ts.net", map[string]string{
+					"tailscale.com/secret-type": "certs",
+					"tailscale.com/managed":     "true",
+					"tailscale.com/proxy-group": "some-other-proxygroup",
+				}, "4"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo":                        []byte("bar"),
+				"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
+				"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
+				"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
+				"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
+			},
+		},
+		{
+			name:     "list_cert_secrets_fails",
+			certMode: "ro",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			secretsListErr: fmt.Errorf("list error"),
+			// The error is logged but not returned, and state is still loaded
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo": []byte("bar"),
+			},
+		},
+		{
+			name:     "cert_secrets_not_loaded_when_not_in_share_mode",
+			certMode: "",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			TLSSecrets: []kubeapi.Secret{
+				makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo": []byte("bar"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			envknob.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
+
+			t.Setenv("POD_NAME", "ingress-proxies-1")
+
+			client := &kubeclient.FakeClient{
+				GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
+					if tt.secretGetErr != nil {
+						return nil, tt.secretGetErr
+					}
+					if name == secretName {
+						return &kubeapi.Secret{Data: tt.stateSecretContents}, nil
+					}
+					return nil, &kubeapi.Status{Code: 404}
+				},
+				CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
+					return true, true, nil
+				},
+				ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
+					if tt.secretsListErr != nil {
+						return nil, tt.secretsListErr
+					}
+					var matchingSecrets []kubeapi.Secret
+					for _, secret := range tt.TLSSecrets {
+						matches := true
+						for k, v := range selector {
+							if secret.Labels[k] != v {
+								matches = false
+								break
+							}
+						}
+						if matches {
+							matchingSecrets = append(matchingSecrets, secret)
+						}
+					}
+					return &kubeapi.SecretList{Items: matchingSecrets}, nil
+				},
+			}
+
+			s, err := newWithClient(t.Logf, client, secretName)
+			if tt.wantErr != nil {
+				if err == nil {
+					t.Errorf("NewWithClient() error = nil, want error containing %v", tt.wantErr)
+					return
+				}
+				if !strings.Contains(err.Error(), tt.wantErr.Error()) {
+					t.Errorf("NewWithClient() error = %v, want error containing %v", err, tt.wantErr)
+				}
+				return
+			}
+
+			if err != nil {
+				t.Errorf("NewWithClient() unexpected error: %v", err)
+				return
+			}
+
+			// Verify memory store contents
+			gotJSON, err := s.memory.ExportToJSON()
+			if err != nil {
+				t.Errorf("ExportToJSON failed: %v", err)
+				return
+			}
+			var got map[ipn.StateKey][]byte
+			if err := json.Unmarshal(gotJSON, &got); err != nil {
+				t.Errorf("failed to unmarshal memory store JSON: %v", err)
+				return
+			}
+			want := tt.wantMemoryStoreContents
+			if want == nil {
+				want = map[ipn.StateKey][]byte{}
+			}
+			if diff := cmp.Diff(got, want); diff != "" {
+				t.Errorf("memory store contents mismatch (-got +want):\n%s", diff)
+			}
+		})
+	}
+}

k8s-operator/api.md

@@ -517,6 +517,7 @@ _Appears in:_
 | `statefulSet` _[StatefulSet](#statefulset)_ | Configuration parameters for the proxy's StatefulSet. Tailscale<br />Kubernetes operator deploys a StatefulSet for each of the user<br />configured proxies (Tailscale Ingress, Tailscale Service, Connector). |  |  |
 | `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported<br />for egress proxies and for Ingress proxies that have been configured<br />with tailscale.com/experimental-forward-cluster-traffic-via-ingress<br />annotation. Note that the metrics are currently considered unstable<br />and will likely change in breaking ways in the future - we only<br />recommend that you use those for debugging purposes. |  |  |
 | `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific<br />parameters of proxies. |  |  |
+| `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS<br />certificates for any HTTPS endpoints exposed to the tailnet from<br />LetsEncrypt's staging environment.<br />https://letsencrypt.org/docs/staging-environment/<br />This setting only affects Tailscale Ingress resources.<br />By default Ingress TLS certificates are issued from LetsEncrypt's<br />production environment.<br />Changing this setting true -> false, will result in any<br />existing certs being re-issued from the production environment.<br />Changing this setting false (default) -> true, when certs have already<br />been provisioned from production environment will NOT result in certs<br />being re-issued from the staging environment before they need to be<br />renewed. |  |  |
 
 
 #### ProxyClassStatus

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -66,6 +66,21 @@ type ProxyClassSpec struct {
 	// parameters of proxies.
 	// +optional
 	TailscaleConfig *TailscaleConfig `json:"tailscale,omitempty"`
+	// Set UseLetsEncryptStagingEnvironment to true to issue TLS
+	// certificates for any HTTPS endpoints exposed to the tailnet from
+	// LetsEncrypt's staging environment.
+	// https://letsencrypt.org/docs/staging-environment/
+	// This setting only affects Tailscale Ingress resources.
+	// By default Ingress TLS certificates are issued from LetsEncrypt's
+	// production environment.
+	// Changing this setting true -> false, will result in any
+	// existing certs being re-issued from the production environment.
+	// Changing this setting false (default) -> true, when certs have already
+	// been provisioned from production environment will NOT result in certs
+	// being re-issued from the staging environment before they need to be
+	// renewed.
+	// +optional
+	UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"`
 }
 
 type TailscaleConfig struct {

kube/kubeapi/api.go

@@ -153,6 +153,14 @@ type Secret struct {
 	Data map[string][]byte `json:"data,omitempty"`
 }
 
+// SecretList is a list of Secret objects.
+type SecretList struct {
+	TypeMeta   `json:",inline"`
+	ObjectMeta `json:"metadata"`
+
+	Items []Secret `json:"items,omitempty"`
+}
+
 // Event contains a subset of fields from corev1.Event.
 // https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7034
 // It is copied here to avoid having to import kube libraries.

kube/kubeclient/client.go

@@ -60,6 +60,7 @@ func readFile(n string) ([]byte, error) {
 // It expects to be run inside a cluster.
 type Client interface {
 	GetSecret(context.Context, string) (*kubeapi.Secret, error)
+	ListSecrets(context.Context, map[string]string) (*kubeapi.SecretList, error)
 	UpdateSecret(context.Context, *kubeapi.Secret) error
 	CreateSecret(context.Context, *kubeapi.Secret) error
 	// Event attempts to ensure an event with the specified options associated with the Pod in which we are
@@ -248,21 +249,35 @@ func (c *client) newRequest(ctx context.Context, method, url string, in any) (*h
 // GetSecret fetches the secret from the Kubernetes API.
 func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) {
 	s := &kubeapi.Secret{Data: make(map[string][]byte)}
-	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets), nil, s); err != nil {
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets, ""), nil, s); err != nil {
 		return nil, err
 	}
 	return s, nil
 }
 
+// ListSecrets fetches the secret from the Kubernetes API.
+func (c *client) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
+	sl := new(kubeapi.SecretList)
+	s := make([]string, 0, len(selector))
+	for key, val := range selector {
+		s = append(s, key+"="+url.QueryEscape(val))
+	}
+	ss := strings.Join(s, ",")
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL("", TypeSecrets, ss), nil, sl); err != nil {
+		return nil, err
+	}
+	return sl, nil
+}
+
 // CreateSecret creates a secret in the Kubernetes API.
 func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error {
 	s.Namespace = c.ns
-	return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets), s, nil)
+	return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets, ""), s, nil)
 }
 
 // UpdateSecret updates a secret in the Kubernetes API.
 func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error {
-	return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets), s, nil)
+	return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets, ""), s, nil)
 }
 
 // JSONPatch is a JSON patch operation.
@@ -283,14 +298,14 @@ func (c *client) JSONPatchResource(ctx context.Context, name, typ string, patche
 			return fmt.Errorf("unsupported JSON patch operation: %q", p.Op)
 		}
 	}
-	return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
+	return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ, ""), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
 }
 
 // StrategicMergePatchSecret updates a secret in the Kubernetes API using a
 // strategic merge patch.
 // If a fieldManager is provided, it will be used to track the patch.
 func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error {
-	surl := c.resourceURL(name, TypeSecrets)
+	surl := c.resourceURL(name, TypeSecrets, "")
 	if fieldManager != "" {
 		uv := url.Values{
 			"fieldManager": {fieldManager},
@@ -342,7 +357,7 @@ func (c *client) Event(ctx context.Context, typ, reason, msg string) error {
 			LastTimestamp:  now,
 			Count:          1,
 		}
-		return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents), &ev, nil)
+		return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents, ""), &ev, nil)
 	}
 	// If the Event already exists, we patch its count and last timestamp. This ensures that when users run 'kubectl
 	// describe pod...', they see the event just once (but with a message of how many times it has appeared over
@@ -472,9 +487,13 @@ func (c *client) checkPermission(ctx context.Context, verb, typ, name string) (b
 // resourceURL returns a URL that can be used to interact with the given resource type and, if name is not empty string,
 // the named resource of that type.
 // Note that this only works for core/v1 resource types.
-func (c *client) resourceURL(name, typ string) string {
+func (c *client) resourceURL(name, typ, sel string) string {
 	if name == "" {
-		return fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
+		url := fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
+		if sel != "" {
+			url += "?labelSelector=" + sel
+		}
+		return url
 	}
 	return fmt.Sprintf("%s/api/v1/namespaces/%s/%s/%s", c.url, c.ns, typ, name)
 }
@@ -487,7 +506,7 @@ func (c *client) nameForEvent(reason string) string {
 // getEvent fetches the event from the Kubernetes API.
 func (c *client) getEvent(ctx context.Context, name string) (*kubeapi.Event, error) {
 	e := &kubeapi.Event{}
-	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents), nil, e); err != nil {
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents, ""), nil, e); err != nil {
 		return nil, err
 	}
 	return e, nil

kube/kubeclient/fake_client.go

@@ -18,6 +18,7 @@ type FakeClient struct {
 	CreateSecretImpl           func(context.Context, *kubeapi.Secret) error
 	UpdateSecretImpl           func(context.Context, *kubeapi.Secret) error
 	JSONPatchResourceImpl      func(context.Context, string, string, []JSONPatch) error
+	ListSecretsImpl            func(context.Context, map[string]string) (*kubeapi.SecretList, error)
 }
 
 func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) {
@@ -45,3 +46,9 @@ func (fc *FakeClient) UpdateSecret(ctx context.Context, secret *kubeapi.Secret)
 func (fc *FakeClient) CreateSecret(ctx context.Context, secret *kubeapi.Secret) error {
 	return fc.CreateSecretImpl(ctx, secret)
 }
+func (fc *FakeClient) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
+	if fc.ListSecretsImpl != nil {
+		return fc.ListSecretsImpl(ctx, selector)
+	}
+	return nil, nil
+}

kube/kubetypes/types.go

@@ -48,4 +48,7 @@ const (
 	PodIPv4Header string = "Pod-IPv4"
 
 	EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown"
+
+	LabelManaged    = "tailscale.com/managed"
+	LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs"
 )

licenses/android.md

@@ -29,7 +29,7 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/djherbis/times](https://pkg.go.dev/github.com/djherbis/times) ([MIT](https://github.com/djherbis/times/blob/v1.6.0/LICENSE))
  - [github.com/fxamacker/cbor/v2](https://pkg.go.dev/github.com/fxamacker/cbor/v2) ([MIT](https://github.com/fxamacker/cbor/blob/v2.7.0/LICENSE))
  - [github.com/gaissmai/bart](https://pkg.go.dev/github.com/gaissmai/bart) ([MIT](https://github.com/gaissmai/bart/blob/v0.18.0/LICENSE))
- - [github.com/go-json-experiment/json](https://pkg.go.dev/github.com/go-json-experiment/json) ([BSD-3-Clause](https://github.com/go-json-experiment/json/blob/6a9a0fde9288/LICENSE))
+ - [github.com/go-json-experiment/json](https://pkg.go.dev/github.com/go-json-experiment/json) ([BSD-3-Clause](https://github.com/go-json-experiment/json/blob/d3c622f1b874/LICENSE))
  - [github.com/godbus/dbus/v5](https://pkg.go.dev/github.com/godbus/dbus/v5) ([BSD-2-Clause](https://github.com/godbus/dbus/blob/76236955d466/LICENSE))
  - [github.com/golang/groupcache/lru](https://pkg.go.dev/github.com/golang/groupcache/lru) ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
  - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.1.2/LICENSE))
@@ -64,11 +64,11 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [go4.org/mem](https://pkg.go.dev/go4.org/mem) ([Apache-2.0](https://github.com/go4org/mem/blob/ae6ca9944745/LICENSE))
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [go4.org/unsafe/assume-no-moving-gc](https://pkg.go.dev/go4.org/unsafe/assume-no-moving-gc) ([BSD-3-Clause](https://github.com/go4org/unsafe-assume-no-moving-gc/blob/e7c30c78aeb2/LICENSE))
- - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.33.0:LICENSE))
+ - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.35.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
  - [golang.org/x/mobile](https://pkg.go.dev/golang.org/x/mobile) ([BSD-3-Clause](https://cs.opensource.google/go/x/mobile/+/81131f64:LICENSE))
  - [golang.org/x/mod/semver](https://pkg.go.dev/golang.org/x/mod/semver) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.23.0:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.29.0:LICENSE))

licenses/apple.md

@@ -70,7 +70,7 @@ See also the dependencies in the [Tailscale CLI][].
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.35.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.29.0:LICENSE))

licenses/tailscale.md

@@ -92,7 +92,7 @@ Some packages may only be included on certain architectures or operating systems
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.35.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/oauth2](https://pkg.go.dev/golang.org/x/oauth2) ([BSD-3-Clause](https://cs.opensource.google/go/x/oauth2/+/v0.26.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))

licenses/windows.md

@@ -62,7 +62,7 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/tailscale/go-winio](https://pkg.go.dev/github.com/tailscale/go-winio) ([MIT](https://github.com/tailscale/go-winio/blob/c4f33415bf55/LICENSE))
  - [github.com/tailscale/hujson](https://pkg.go.dev/github.com/tailscale/hujson) ([BSD-3-Clause](https://github.com/tailscale/hujson/blob/20486734a56a/LICENSE))
  - [github.com/tailscale/netlink](https://pkg.go.dev/github.com/tailscale/netlink) ([Apache-2.0](https://github.com/tailscale/netlink/blob/4d49adab4de7/LICENSE))
- - [github.com/tailscale/walk](https://pkg.go.dev/github.com/tailscale/walk) ([BSD-3-Clause](https://github.com/tailscale/walk/blob/04068c1cab63/LICENSE))
+ - [github.com/tailscale/walk](https://pkg.go.dev/github.com/tailscale/walk) ([BSD-3-Clause](https://github.com/tailscale/walk/blob/b2c15a420186/LICENSE))
  - [github.com/tailscale/win](https://pkg.go.dev/github.com/tailscale/win) ([BSD-3-Clause](https://github.com/tailscale/win/blob/5992cb43ca35/LICENSE))
  - [github.com/tailscale/xnet/webdav](https://pkg.go.dev/github.com/tailscale/xnet/webdav) ([BSD-3-Clause](https://github.com/tailscale/xnet/blob/8497ac4dab2e/LICENSE))
  - [github.com/tc-hib/winres](https://pkg.go.dev/github.com/tc-hib/winres) ([0BSD](https://github.com/tc-hib/winres/blob/v0.2.1/LICENSE))
@@ -74,7 +74,7 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
  - [golang.org/x/image/bmp](https://pkg.go.dev/golang.org/x/image/bmp) ([BSD-3-Clause](https://cs.opensource.google/go/x/image/+/v0.24.0:LICENSE))
  - [golang.org/x/mod](https://pkg.go.dev/golang.org/x/mod) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.23.0:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.29.0:LICENSE))

net/dns/manager.go

@@ -35,6 +35,9 @@ import (
 
 var (
 	errFullQueue = errors.New("request queue full")
+	// ErrNoDNSConfig is returned by RecompileDNSConfig when the Manager
+	// has no existing DNS configuration.
+	ErrNoDNSConfig = errors.New("no DNS configuration")
 )
 
 // maxActiveQueries returns the maximal number of DNS requests that can
@@ -91,21 +94,18 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, health *health.Tracker,
 	}
 
 	// Rate limit our attempts to correct our DNS configuration.
+	// This is done on incoming queries, we don't want to spam it.
 	limiter := rate.NewLimiter(1.0/5.0, 1)
 
 	// This will recompile the DNS config, which in turn will requery the system
 	// DNS settings. The recovery func should triggered only when we are missing
 	// upstream nameservers and require them to forward a query.
 	m.resolver.SetMissingUpstreamRecovery(func() {
-		m.mu.Lock()
-		defer m.mu.Unlock()
-		if m.config == nil {
-			return
-		}
-
 		if limiter.Allow() {
-			m.logf("DNS resolution failed due to missing upstream nameservers.  Recompiling DNS configuration.")
-			m.setLocked(*m.config)
+			m.logf("resolution failed due to missing upstream nameservers.  Recompiling DNS configuration.")
+			if err := m.RecompileDNSConfig(); err != nil {
+				m.logf("config recompilation failed: %v", err)
+			}
 		}
 	})
 
@@ -117,6 +117,26 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, health *health.Tracker,
 // Resolver returns the Manager's DNS Resolver.
 func (m *Manager) Resolver() *resolver.Resolver { return m.resolver }
 
+// RecompileDNSConfig sets the DNS config to the current value, which has
+// the side effect of re-querying the OS's interface nameservers.  This should be used
+// on platforms where the interface nameservers can change.  Darwin, for example,
+// where the nameservers aren't always available when we process a major interface
+// change event, or platforms where the nameservers may change while tunnel is up.
+//
+// This should be called if it is determined that [OSConfigurator.GetBaseConfig] may
+// give a better or different result than when [Manager.Set] was last called.  The
+// logic for making that determination is up to the caller.
+//
+// It returns [ErrNoDNSConfig] if the [Manager] has no existing DNS configuration.
+func (m *Manager) RecompileDNSConfig() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.config == nil {
+		return ErrNoDNSConfig
+	}
+	return m.setLocked(*m.config)
+}
+
 func (m *Manager) Set(cfg Config) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()

net/netmon/interfaces_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestGetState(t *testing.T) {
-	st, err := GetState()
+	st, err := getState()
 	if err != nil {
 		t.Fatal(err)
 	}

net/netmon/loghelper.go

@@ -0,0 +1,42 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package netmon
+
+import (
+	"sync"
+
+	"tailscale.com/types/logger"
+)
+
+// LinkChangeLogLimiter returns a new [logger.Logf] that logs each unique
+// format string to the underlying logger only once per major LinkChange event.
+//
+// The returned function should be called when the logger is no longer needed,
+// to release resources from the Monitor.
+func LinkChangeLogLimiter(logf logger.Logf, nm *Monitor) (_ logger.Logf, unregister func()) {
+	var formatSeen sync.Map // map[string]bool
+	unregister = nm.RegisterChangeCallback(func(cd *ChangeDelta) {
+		// If we're in a major change or a time jump, clear the seen map.
+		if cd.Major || cd.TimeJumped {
+			formatSeen.Clear()
+		}
+	})
+
+	return func(format string, args ...any) {
+		// We only store 'true' in the map, so if it's present then it
+		// means we've already logged this format string.
+		_, loaded := formatSeen.LoadOrStore(format, true)
+		if loaded {
+			// TODO(andrew-d): we may still want to log this
+			// message every N minutes (1x/hour?) even if it's been
+			// seen, so that debugging doesn't require searching
+			// back in the logs for an unbounded amount of time.
+			//
+			// See: https://github.com/tailscale/tailscale/issues/13145
+			return
+		}
+
+		logf(format, args...)
+	}, unregister
+}

net/netmon/loghelper_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package netmon
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+)
+
+func TestLinkChangeLogLimiter(t *testing.T) {
+	mon, err := New(t.Logf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer mon.Close()
+
+	var logBuffer bytes.Buffer
+	logf := func(format string, args ...any) {
+		t.Logf("captured log: "+format, args...)
+
+		if format[len(format)-1] != '\n' {
+			format += "\n"
+		}
+		fmt.Fprintf(&logBuffer, format, args...)
+	}
+
+	logf, unregister := LinkChangeLogLimiter(logf, mon)
+	defer unregister()
+
+	// Log once, which should write to our log buffer.
+	logf("hello %s", "world")
+	if got := logBuffer.String(); got != "hello world\n" {
+		t.Errorf("unexpected log buffer contents: %q", got)
+	}
+
+	// Log again, which should not write to our log buffer.
+	logf("hello %s", "andrew")
+	if got := logBuffer.String(); got != "hello world\n" {
+		t.Errorf("unexpected log buffer contents: %q", got)
+	}
+
+	// Log a different message, which should write to our log buffer.
+	logf("other message")
+	if got := logBuffer.String(); got != "hello world\nother message\n" {
+		t.Errorf("unexpected log buffer contents: %q", got)
+	}
+
+	// Synthesize a fake major change event, which should clear the format
+	// string cache and allow the next log to write to our log buffer.
+	//
+	// InjectEvent doesn't work because it's not a major event, so we
+	// instead reach into the netmon and grab the callback, and then call
+	// it ourselves.
+	mon.mu.Lock()
+	var cb func(*ChangeDelta)
+	for _, c := range mon.cbs {
+		cb = c
+		break
+	}
+	mon.mu.Unlock()
+
+	cb(&ChangeDelta{Major: true})
+
+	logf("hello %s", "world")
+	if got := logBuffer.String(); got != "hello world\nother message\nhello world\n" {
+		t.Errorf("unexpected log buffer contents: %q", got)
+	}
+
+	// Unregistering the callback should clear our 'cbs' set.
+	unregister()
+	mon.mu.Lock()
+	if len(mon.cbs) != 0 {
+		t.Errorf("expected no callbacks, got %v", mon.cbs)
+	}
+	mon.mu.Unlock()
+}

net/netmon/netmon.go

@@ -161,7 +161,7 @@ func (m *Monitor) InterfaceState() *State {
 }
 
 func (m *Monitor) interfaceStateUncached() (*State, error) {
-	return GetState()
+	return getState()
 }
 
 // SetTailscaleInterfaceName sets the name of the Tailscale interface. For

net/netmon/state.go

@@ -466,7 +466,7 @@ var getPAC func() string
 // It does not set the returned State.IsExpensive. The caller can populate that.
 //
 // Deprecated: use netmon.Monitor.InterfaceState instead.
-func GetState() (*State, error) {
+func getState() (*State, error) {
 	s := &State{
 		InterfaceIPs: make(map[string][]netip.Prefix),
 		Interface:    make(map[string]Interface),

net/packet/geneve.go

@@ -0,0 +1,104 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package packet
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+const (
+	// GeneveFixedHeaderLength is the length of the fixed size portion of the
+	// Geneve header, in bytes.
+	GeneveFixedHeaderLength = 8
+)
+
+const (
+	// GeneveProtocolDisco is the IEEE 802 Ethertype number used to represent
+	// the Tailscale Disco protocol in a Geneve header.
+	GeneveProtocolDisco uint16 = 0x7A11
+	// GeneveProtocolWireGuard is the IEEE 802 Ethertype number used to represent the
+	// WireGuard protocol in a Geneve header.
+	GeneveProtocolWireGuard uint16 = 0x7A12
+)
+
+// GeneveHeader represents the fixed size Geneve header from RFC8926.
+// TLVs/options are not implemented/supported.
+//
+//	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//	|Ver|  Opt Len  |O|C|    Rsvd.  |          Protocol Type        |
+//	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//	|        Virtual Network Identifier (VNI)       |    Reserved   |
+//	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+type GeneveHeader struct {
+	// Ver (2 bits): The current version number is 0. Packets received by a
+	// tunnel endpoint with an unknown version MUST be dropped. Transit devices
+	// interpreting Geneve packets with an unknown version number MUST treat
+	// them as UDP packets with an unknown payload.
+	Version uint8
+
+	// Protocol Type (16 bits): The type of protocol data unit appearing after
+	// the Geneve header. This follows the Ethertype [ETYPES] convention, with
+	// Ethernet itself being represented by the value 0x6558.
+	Protocol uint16
+
+	// Virtual Network Identifier (VNI) (24 bits): An identifier for a unique
+	// element of a virtual network. In many situations, this may represent an
+	// L2 segment; however, the control plane defines the forwarding semantics
+	// of decapsulated packets. The VNI MAY be used as part of ECMP forwarding
+	// decisions or MAY be used as a mechanism to distinguish between
+	// overlapping address spaces contained in the encapsulated packet when load
+	// balancing across CPUs.
+	VNI uint32
+
+	// O (1 bit): Control packet. This packet contains a control message.
+	// Control messages are sent between tunnel endpoints. Tunnel endpoints MUST
+	// NOT forward the payload, and transit devices MUST NOT attempt to
+	// interpret it. Since control messages are less frequent, it is RECOMMENDED
+	// that tunnel endpoints direct these packets to a high-priority control
+	// queue (for example, to direct the packet to a general purpose CPU from a
+	// forwarding Application-Specific Integrated Circuit (ASIC) or to separate
+	// out control traffic on a NIC). Transit devices MUST NOT alter forwarding
+	// behavior on the basis of this bit, such as ECMP link selection.
+	Control bool
+}
+
+// Encode encodes GeneveHeader into b. If len(b) < GeneveFixedHeaderLength an
+// io.ErrShortBuffer error is returned.
+func (h *GeneveHeader) Encode(b []byte) error {
+	if len(b) < GeneveFixedHeaderLength {
+		return io.ErrShortBuffer
+	}
+	if h.Version > 3 {
+		return errors.New("version must be <= 3")
+	}
+	b[0] = 0
+	b[1] = 0
+	b[0] |= h.Version << 6
+	if h.Control {
+		b[1] |= 0x80
+	}
+	binary.BigEndian.PutUint16(b[2:], h.Protocol)
+	if h.VNI > 1<<24-1 {
+		return errors.New("VNI must be <= 2^24-1")
+	}
+	binary.BigEndian.PutUint32(b[4:], h.VNI<<8)
+	return nil
+}
+
+// Decode decodes GeneveHeader from b. If len(b) < GeneveFixedHeaderLength an
+// io.ErrShortBuffer error is returned.
+func (h *GeneveHeader) Decode(b []byte) error {
+	if len(b) < GeneveFixedHeaderLength {
+		return io.ErrShortBuffer
+	}
+	h.Version = b[0] >> 6
+	if b[1]&0x80 != 0 {
+		h.Control = true
+	}
+	h.Protocol = binary.BigEndian.Uint16(b[2:])
+	h.VNI = binary.BigEndian.Uint32(b[4:]) >> 8
+	return nil
+}

net/packet/geneve_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package packet
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestGeneveHeader(t *testing.T) {
+	in := GeneveHeader{
+		Version:  3,
+		Protocol: GeneveProtocolDisco,
+		VNI:      1<<24 - 1,
+		Control:  true,
+	}
+	b := make([]byte, GeneveFixedHeaderLength)
+	err := in.Encode(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := GeneveHeader{}
+	err = out.Decode(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(out, in); diff != "" {
+		t.Fatalf("wrong results (-got +want)\n%s", diff)
+	}
+}

pkgdoc_test.go

@@ -26,6 +26,9 @@ func TestPackageDocs(t *testing.T) {
 		if err != nil {
 			return err
 		}
+		if fi.Mode().IsDir() && path == ".git" {
+			return filepath.SkipDir // No documentation lives in .git
+		}
 		if fi.Mode().IsRegular() && strings.HasSuffix(path, ".go") {
 			if strings.HasSuffix(path, "_test.go") {
 				return nil

safesocket/safesocket.go

@@ -61,7 +61,11 @@ func ConnectContext(ctx context.Context, path string) (net.Conn, error) {
 			if ctx.Err() != nil {
 				return nil, ctx.Err()
 			}
-			time.Sleep(250 * time.Millisecond)
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(250 * time.Millisecond):
+			}
 			continue
 		}
 		return c, err

tsnet/tsnet_test.go

@@ -120,6 +120,7 @@ func startControl(t *testing.T) (controlURL string, control *testcontrol.Server)
 			Proxied: true,
 		},
 		MagicDNSDomain: "tail-scale.ts.net",
+		Logf:           t.Logf,
 	}
 	control.HTTPTestServer = httptest.NewUnstartedServer(control)
 	control.HTTPTestServer.Start()
@@ -221,7 +222,7 @@ func startServer(t *testing.T, ctx context.Context, controlURL, hostname string)
 		getCertForTesting: testCertRoot.getCert,
 	}
 	if *verboseNodes {
-		s.Logf = log.Printf
+		s.Logf = t.Logf
 	}
 	t.Cleanup(func() { s.Close() })
 

tstest/integration/integration_test.go

@@ -1942,6 +1942,8 @@ func (n *testNode) AwaitIP6() netip.Addr {
 
 // AwaitRunning waits for n to reach the IPN state "Running".
 func (n *testNode) AwaitRunning() {
+	t := n.env.t
+	t.Helper()
 	n.AwaitBackendState("Running")
 }
 
@@ -2015,7 +2017,7 @@ func (n *testNode) Status() (*ipnstate.Status, error) {
 	}
 	st := new(ipnstate.Status)
 	if err := json.Unmarshal(out, st); err != nil {
-		return nil, fmt.Errorf("decoding tailscale status JSON: %w", err)
+		return nil, fmt.Errorf("decoding tailscale status JSON: %w\njson:\n%s", err, out)
 	}
 	return st, nil
 }

tstest/mts/mts.go

@@ -0,0 +1,599 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux || darwin
+
+// The mts ("Multiple Tailscale") command runs multiple tailscaled instances for
+// development, managing their directories and sockets, and lets you easily direct
+// tailscale CLI commands to them.
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"maps"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"tailscale.com/client/local"
+	"tailscale.com/types/bools"
+	"tailscale.com/types/lazy"
+	"tailscale.com/util/mak"
+)
+
+func usage(args ...any) {
+	var format string
+	if len(args) > 0 {
+		format, args = args[0].(string), args[1:]
+	}
+	if format != "" {
+		format = strings.TrimSpace(format) + "\n\n"
+		fmt.Fprintf(os.Stderr, format, args...)
+	}
+	io.WriteString(os.Stderr, strings.TrimSpace(`
+usage:
+
+   mts server <subcommand>      # manage tailscaled instances
+   mts server run               # run the mts server (parent process of all tailscaled)
+   mts server list              # list all tailscaled and their state
+   mts server list <name>       # show details of named instance
+   mts server add <name>        # add+start new named tailscaled
+   mts server start <name>      # start a previously added tailscaled
+   mts server stop <name>       # stop & remove a named tailscaled
+   mts server rm <name>         # stop & remove a named tailscaled
+   mts server logs [-f] <name>  # get/follow tailscaled logs
+
+  mts <inst-name> [tailscale CLI args] # run Tailscale CLI against a named instance
+    e.g.
+      mts gmail1 up
+      mts github2 status --json
+	`)+"\n")
+	os.Exit(1)
+}
+
+func main() {
+	// Don't use flag.Parse here; we mostly just delegate through
+	// to the Tailscale CLI.
+
+	if len(os.Args) < 2 {
+		usage()
+	}
+	firstArg, args := os.Args[1], os.Args[2:]
+	if firstArg == "server" || firstArg == "s" {
+		if err := runMTSServer(args); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		var c Client
+		inst := firstArg
+		c.RunCommand(inst, args)
+	}
+}
+
+func runMTSServer(args []string) error {
+	if len(args) == 0 {
+		usage()
+	}
+	cmd, args := args[0], args[1:]
+	if cmd == "run" {
+		var s Server
+		return s.Run()
+	}
+
+	// Commands other than "run" all use the HTTP client to
+	// hit the mts server over its unix socket.
+	var c Client
+
+	switch cmd {
+	default:
+		usage("unknown mts server subcommand %q", cmd)
+	case "list", "ls":
+		list, err := c.List()
+		if err != nil {
+			return err
+		}
+		if len(args) == 0 {
+			names := slices.Sorted(maps.Keys(list.Instances))
+			for _, name := range names {
+				running := list.Instances[name].Running
+				fmt.Printf("%10s %s\n", bools.IfElse(running, "RUNNING", "stopped"), name)
+			}
+		} else {
+			for _, name := range args {
+				inst, ok := list.Instances[name]
+				if !ok {
+					return fmt.Errorf("no instance named %q", name)
+				}
+				je := json.NewEncoder(os.Stdout)
+				je.SetIndent("", "  ")
+				if err := je.Encode(inst); err != nil {
+					return err
+				}
+			}
+		}
+
+	case "rm":
+		if len(args) == 0 {
+			return fmt.Errorf("missing instance name(s) to remove")
+		}
+		log.SetFlags(0)
+		for _, name := range args {
+			ok, err := c.Remove(name)
+			if err != nil {
+				return err
+			}
+			if ok {
+				log.Printf("%s deleted.", name)
+			} else {
+				log.Printf("%s didn't exist.", name)
+			}
+		}
+	case "stop":
+		if len(args) == 0 {
+			return fmt.Errorf("missing instance name(s) to stop")
+		}
+		log.SetFlags(0)
+		for _, name := range args {
+			ok, err := c.Stop(name)
+			if err != nil {
+				return err
+			}
+			if ok {
+				log.Printf("%s stopped.", name)
+			} else {
+				log.Printf("%s didn't exist.", name)
+			}
+		}
+	case "start", "restart":
+		list, err := c.List()
+		if err != nil {
+			return err
+		}
+		shouldStop := cmd == "restart"
+		for _, arg := range args {
+			is, ok := list.Instances[arg]
+			if !ok {
+				return fmt.Errorf("no instance named %q", arg)
+			}
+			if is.Running {
+				if shouldStop {
+					if _, err := c.Stop(arg); err != nil {
+						return fmt.Errorf("stopping %q: %w", arg, err)
+					}
+				} else {
+					log.SetFlags(0)
+					log.Printf("%s already running.", arg)
+					continue
+				}
+			}
+			// Creating an existing one starts it up.
+			if err := c.Create(arg); err != nil {
+				return fmt.Errorf("starting %q: %w", arg, err)
+			}
+		}
+	case "add":
+		if len(args) == 0 {
+			return fmt.Errorf("missing instance name(s) to add")
+		}
+		for _, name := range args {
+			if err := c.Create(name); err != nil {
+				return fmt.Errorf("creating %q: %w", name, err)
+			}
+		}
+	case "logs":
+		fs := flag.NewFlagSet("logs", flag.ExitOnError)
+		fs.Usage = func() { usage() }
+		follow := fs.Bool("f", false, "follow logs")
+		fs.Parse(args)
+		log.Printf("Parsed; following=%v, args=%q", *follow, fs.Args())
+		if fs.NArg() != 1 {
+			usage()
+		}
+		cmd := bools.IfElse(*follow, "tail", "cat")
+		args := []string{cmd}
+		if *follow {
+			args = append(args, "-f")
+		}
+		path, err := exec.LookPath(cmd)
+		if err != nil {
+			return fmt.Errorf("looking up %q: %w", cmd, err)
+		}
+		args = append(args, instLogsFile(fs.Arg(0)))
+		log.Fatal(syscall.Exec(path, args, os.Environ()))
+	}
+	return nil
+}
+
+type Client struct {
+}
+
+func (c *Client) client() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return net.Dial("unix", mtsSock())
+			},
+		},
+	}
+}
+
+func getJSON[T any](res *http.Response, err error) (T, error) {
+	var ret T
+	if err != nil {
+		return ret, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return ret, fmt.Errorf("unexpected status: %v: %s", res.Status, body)
+	}
+	if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
+		return ret, err
+	}
+	return ret, nil
+}
+
+func (c *Client) List() (listResponse, error) {
+	return getJSON[listResponse](c.client().Get("http://mts/list"))
+}
+
+func (c *Client) Remove(name string) (found bool, err error) {
+	return getJSON[bool](c.client().PostForm("http://mts/rm", url.Values{
+		"name": []string{name},
+	}))
+}
+
+func (c *Client) Stop(name string) (found bool, err error) {
+	return getJSON[bool](c.client().PostForm("http://mts/stop", url.Values{
+		"name": []string{name},
+	}))
+}
+
+func (c *Client) Create(name string) error {
+	req, err := http.NewRequest("POST", "http://mts/create/"+name, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.client().Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("unexpected status: %v: %s", resp.Status, body)
+	}
+	return nil
+}
+
+func (c *Client) RunCommand(name string, args []string) {
+	sock := instSock(name)
+	lc := &local.Client{
+		Socket:        sock,
+		UseSocketOnly: true,
+	}
+	probeCtx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
+	defer cancel()
+	if _, err := lc.StatusWithoutPeers(probeCtx); err != nil {
+		log.Fatalf("instance %q not running? start with 'mts server start %q'; got error: %v", name, name, err)
+	}
+	args = append([]string{"run", "tailscale.com/cmd/tailscale", "--socket=" + sock}, args...)
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+	err := cmd.Run()
+	if err == nil {
+		os.Exit(0)
+	}
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		os.Exit(exitErr.ExitCode())
+	}
+	panic(err)
+}
+
+type Server struct {
+	lazyTailscaled lazy.GValue[string]
+
+	mu   sync.Mutex
+	cmds map[string]*exec.Cmd // running tailscaled instances
+}
+
+func (s *Server) tailscaled() string {
+	v, err := s.lazyTailscaled.GetErr(func() (string, error) {
+		out, err := exec.Command("go", "list", "-f", "{{.Target}}", "tailscale.com/cmd/tailscaled").CombinedOutput()
+		if err != nil {
+			return "", err
+		}
+		return strings.TrimSpace(string(out)), nil
+	})
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+func (s *Server) Run() error {
+	if err := os.MkdirAll(mtsRoot(), 0700); err != nil {
+		return err
+	}
+	sock := mtsSock()
+	os.Remove(sock)
+	log.Printf("Multi-Tailscaled Server running; listening on %q ...", sock)
+	ln, err := net.Listen("unix", sock)
+	if err != nil {
+		return err
+	}
+	return http.Serve(ln, s)
+}
+
+var validNameRx = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
+
+func validInstanceName(name string) bool {
+	return validNameRx.MatchString(name)
+}
+
+func (s *Server) InstanceRunning(name string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	_, ok := s.cmds[name]
+	return ok
+}
+
+func (s *Server) Stop(name string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if cmd, ok := s.cmds[name]; ok {
+		if err := cmd.Process.Kill(); err != nil {
+			log.Printf("error killing %q: %v", name, err)
+		}
+		delete(s.cmds, name)
+	}
+}
+
+func (s *Server) RunInstance(name string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.cmds[name]; ok {
+		return fmt.Errorf("instance %q already running", name)
+	}
+
+	if !validInstanceName(name) {
+		return fmt.Errorf("invalid instance name %q", name)
+	}
+	dir := filepath.Join(mtsRoot(), name)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+
+	env := os.Environ()
+	env = append(env, "TS_DEBUG_LOG_RATE=all")
+	if ef, err := os.Open(instEnvFile(name)); err == nil {
+		defer ef.Close()
+		sc := bufio.NewScanner(ef)
+		for sc.Scan() {
+			t := strings.TrimSpace(sc.Text())
+			if strings.HasPrefix(t, "#") || !strings.Contains(t, "=") {
+				continue
+			}
+			env = append(env, t)
+		}
+	} else if os.IsNotExist(err) {
+		// Write an example one.
+		os.WriteFile(instEnvFile(name), fmt.Appendf(nil, "# Example mts env.txt file; uncomment/add stuff you want for %q\n\n#TS_DEBUG_MAP=1\n#TS_DEBUG_REGISTER=1\n#TS_NO_LOGS_NO_SUPPORT=1\n", name), 0600)
+	}
+
+	extraArgs := []string{"--verbose=1"}
+	if af, err := os.Open(instArgsFile(name)); err == nil {
+		extraArgs = nil // clear default args
+		defer af.Close()
+		sc := bufio.NewScanner(af)
+		for sc.Scan() {
+			t := strings.TrimSpace(sc.Text())
+			if strings.HasPrefix(t, "#") || t == "" {
+				continue
+			}
+			extraArgs = append(extraArgs, t)
+		}
+	} else if os.IsNotExist(err) {
+		// Write an example one.
+		os.WriteFile(instArgsFile(name), fmt.Appendf(nil, "# Example mts args.txt file for instance %q.\n# One line per extra arg to tailscaled; no magic string quoting\n\n--verbose=1\n#--socks5-server=127.0.0.1:5000\n", name), 0600)
+	}
+
+	log.Printf("Running Tailscale daemon %q in %q", name, dir)
+
+	args := []string{
+		"--tun=userspace-networking",
+		"--statedir=" + filepath.Join(dir),
+		"--socket=" + filepath.Join(dir, "tailscaled.sock"),
+	}
+	args = append(args, extraArgs...)
+
+	cmd := exec.Command(s.tailscaled(), args...)
+	cmd.Dir = dir
+	cmd.Env = env
+
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	cmd.Stderr = cmd.Stdout
+
+	logs := instLogsFile(name)
+	logFile, err := os.OpenFile(logs, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_TRUNC, 0644)
+	if err != nil {
+		return fmt.Errorf("opening logs file: %w", err)
+	}
+
+	go func() {
+		bs := bufio.NewScanner(out)
+		for bs.Scan() {
+			// TODO(bradfitz): record in memory too, serve via HTTP
+			line := strings.TrimSpace(bs.Text())
+			fmt.Fprintf(logFile, "%s\n", line)
+			fmt.Printf("tailscaled[%s]: %s\n", name, line)
+		}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	go func() {
+		err := cmd.Wait()
+		logFile.Close()
+		log.Printf("Tailscale daemon %q exited: %v", name, err)
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		delete(s.cmds, name)
+	}()
+
+	mak.Set(&s.cmds, name, cmd)
+	return nil
+}
+
+type listResponse struct {
+	// Instances maps instance name to its details.
+	Instances map[string]listResponseInstance `json:"instances"`
+}
+
+type listResponseInstance struct {
+	Name    string `json:"name"`
+	Dir     string `json:"dir"`
+	Sock    string `json:"sock"`
+	Running bool   `json:"running"`
+	Env     string `json:"env"`
+	Args    string `json:"args"`
+	Logs    string `json:"logs"`
+}
+
+func writeJSON(w http.ResponseWriter, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "  ")
+	e.Encode(v)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path == "/list" {
+		var res listResponse
+		for _, name := range s.InstanceNames() {
+			mak.Set(&res.Instances, name, listResponseInstance{
+				Name:    name,
+				Dir:     instDir(name),
+				Sock:    instSock(name),
+				Running: s.InstanceRunning(name),
+				Env:     instEnvFile(name),
+				Args:    instArgsFile(name),
+				Logs:    instLogsFile(name),
+			})
+		}
+		writeJSON(w, res)
+		return
+	}
+	if r.URL.Path == "/rm" || r.URL.Path == "/stop" {
+		shouldRemove := r.URL.Path == "/rm"
+		if r.Method != "POST" {
+			http.Error(w, "POST required", http.StatusMethodNotAllowed)
+			return
+		}
+		target := r.FormValue("name")
+		var ok bool
+		for _, name := range s.InstanceNames() {
+			if name != target {
+				continue
+			}
+			ok = true
+			s.Stop(name)
+			if shouldRemove {
+				if err := os.RemoveAll(instDir(name)); err != nil {
+					http.Error(w, err.Error(), http.StatusInternalServerError)
+					return
+				}
+			}
+			break
+		}
+		writeJSON(w, ok)
+		return
+	}
+	if inst, ok := strings.CutPrefix(r.URL.Path, "/create/"); ok {
+		if !s.InstanceRunning(inst) {
+			if err := s.RunInstance(inst); err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+		fmt.Fprintf(w, "OK\n")
+		return
+	}
+	if r.URL.Path == "/" {
+		fmt.Fprintf(w, "This is mts, the multi-tailscaled server.\n")
+		return
+	}
+	http.NotFound(w, r)
+}
+
+func (s *Server) InstanceNames() []string {
+	var ret []string
+	des, err := os.ReadDir(mtsRoot())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		panic(err)
+	}
+	for _, de := range des {
+		if !de.IsDir() {
+			continue
+		}
+		ret = append(ret, de.Name())
+	}
+	return ret
+}
+
+func mtsRoot() string {
+	dir, err := os.UserConfigDir()
+	if err != nil {
+		panic(err)
+	}
+	return filepath.Join(dir, "multi-tailscale-dev")
+}
+
+func instDir(name string) string {
+	return filepath.Join(mtsRoot(), name)
+}
+
+func instSock(name string) string {
+	return filepath.Join(instDir(name), "tailscaled.sock")
+}
+
+func instEnvFile(name string) string {
+	return filepath.Join(mtsRoot(), name, "env.txt")
+}
+
+func instArgsFile(name string) string {
+	return filepath.Join(mtsRoot(), name, "args.txt")
+}
+
+func instLogsFile(name string) string {
+	return filepath.Join(mtsRoot(), name, "logs.txt")
+}
+
+func mtsSock() string {
+	return filepath.Join(mtsRoot(), "mts.sock")
+}

tsweb/debug.go

@@ -14,7 +14,7 @@ import (
 	"os"
 	"runtime"
 
-	"tailscale.com/tsweb/promvarz"
+	"tailscale.com/feature"
 	"tailscale.com/tsweb/varz"
 	"tailscale.com/version"
 )
@@ -34,8 +34,14 @@ type DebugHandler struct {
 	kvs      []func(io.Writer)                // output one <li>...</li> each, see KV()
 	urls     []string                         // one <li>...</li> block with link each
 	sections []func(io.Writer, *http.Request) // invoked in registration order prior to outputting </body>
+	title    string                           // title displayed on index page
 }
 
+// PrometheusHandler is an optional hook to enable native Prometheus
+// support in the debug handler. It is disabled by default. Import the
+// tailscale.com/tsweb/promvarz package to enable this feature.
+var PrometheusHandler feature.Hook[func(*DebugHandler)]
+
 // Debugger returns the DebugHandler registered on mux at /debug/,
 // creating it if necessary.
 func Debugger(mux *http.ServeMux) *DebugHandler {
@@ -44,14 +50,19 @@ func Debugger(mux *http.ServeMux) *DebugHandler {
 		return d
 	}
 	ret := &DebugHandler{
-		mux: mux,
+		mux:   mux,
+		title: fmt.Sprintf("%s debug", version.CmdName()),
 	}
 	mux.Handle("/debug/", ret)
 
 	ret.KVFunc("Uptime", func() any { return varz.Uptime() })
 	ret.KV("Version", version.Long())
 	ret.Handle("vars", "Metrics (Go)", expvar.Handler())
-	ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(promvarz.Handler))
+	if PrometheusHandler.IsSet() {
+		PrometheusHandler.Get()(ret)
+	} else {
+		ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(varz.Handler))
+	}
 
 	// pprof.Index serves everything that runtime/pprof.Lookup finds:
 	// goroutine, threadcreate, heap, allocs, block, mutex
@@ -85,7 +96,7 @@ func (d *DebugHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	AddBrowserHeaders(w)
 	f := func(format string, args ...any) { fmt.Fprintf(w, format, args...) }
-	f("<html><body><h1>%s debug</h1><ul>", version.CmdName())
+	f("<html><body><h1>%s</h1><ul>", html.EscapeString(d.title))
 	for _, kv := range d.kvs {
 		kv(w)
 	}
@@ -103,14 +114,20 @@ func (d *DebugHandler) handle(slug string, handler http.Handler) string {
 	return href
 }
 
-// Handle registers handler at /debug/<slug> and creates a descriptive
-// entry in /debug/ for it.
+// Handle registers handler at /debug/<slug> and adds a link to it
+// on /debug/ with the provided description.
 func (d *DebugHandler) Handle(slug, desc string, handler http.Handler) {
 	href := d.handle(slug, handler)
 	d.URL(href, desc)
 }
 
-// HandleSilent registers handler at /debug/<slug>. It does not create
+// Handle registers handler at /debug/<slug> and adds a link to it
+// on /debug/ with the provided description.
+func (d *DebugHandler) HandleFunc(slug, desc string, handler http.HandlerFunc) {
+	d.Handle(slug, desc, handler)
+}
+
+// HandleSilent registers handler at /debug/<slug>. It does not add
 // a descriptive entry in /debug/ for it. This should be used
 // sparingly, for things that need to be registered but would pollute
 // the list of debug links.
@@ -118,6 +135,14 @@ func (d *DebugHandler) HandleSilent(slug string, handler http.Handler) {
 	d.handle(slug, handler)
 }
 
+// HandleSilent registers handler at /debug/<slug>. It does not add
+// a descriptive entry in /debug/ for it. This should be used
+// sparingly, for things that need to be registered but would pollute
+// the list of debug links.
+func (d *DebugHandler) HandleSilentFunc(slug string, handler http.HandlerFunc) {
+	d.HandleSilent(slug, handler)
+}
+
 // KV adds a key/value list item to /debug/.
 func (d *DebugHandler) KV(k string, v any) {
 	val := html.EscapeString(fmt.Sprintf("%v", v))
@@ -149,6 +174,11 @@ func (d *DebugHandler) Section(f func(w io.Writer, r *http.Request)) {
 	d.sections = append(d.sections, f)
 }
 
+// Title sets the title at the top of the debug page.
+func (d *DebugHandler) Title(title string) {
+	d.title = title
+}
+
 func gcHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("running GC...\n"))
 	if f, ok := w.(http.Flusher); ok {

tsweb/promvarz/promvarz.go

@@ -11,12 +11,21 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/expfmt"
+	"tailscale.com/tsweb"
 	"tailscale.com/tsweb/varz"
 )
 
-// Handler returns Prometheus metrics exported by our expvar converter
+func init() {
+	tsweb.PrometheusHandler.Set(registerVarz)
+}
+
+func registerVarz(debug *tsweb.DebugHandler) {
+	debug.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(handler))
+}
+
+// handler returns Prometheus metrics exported by our expvar converter
 // and the official Prometheus client.
-func Handler(w http.ResponseWriter, r *http.Request) {
+func handler(w http.ResponseWriter, r *http.Request) {
 	if err := gatherNativePrometheusMetrics(w); err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte(err.Error()))

tsweb/promvarz/promvarz_test.go

@@ -23,7 +23,7 @@ func TestHandler(t *testing.T) {
 	testVar1.Set(42)
 	testVar2.Set(4242)
 
-	svr := httptest.NewServer(http.HandlerFunc(Handler))
+	svr := httptest.NewServer(http.HandlerFunc(handler))
 	defer svr.Close()
 
 	want := `

util/eventbus/assets/event.html

@@ -0,0 +1,6 @@
+<li id="monitor" hx-swap-oob="afterbegin">
+    <details>
+        <summary>{{.Count}}: {{.Type}} from {{.Event.From.Name}}, {{len .Event.To}} recipients</summary>
+        {{.Event.Event}}
+    </details>
+</li>

util/eventbus/assets/main.html

@@ -0,0 +1,97 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <script src="bus/htmx.min.js"></script>
+        <script src="bus/htmx-websocket.min.js"></script>
+        <link rel="stylesheet" href="bus/style.css">
+    </head>
+    <body hx-ext="ws">
+        <h1>Event bus</h1>
+
+        <section>
+            <h2>General</h2>
+            {{with $.PublishQueue}}
+            {{len .}} pending
+            {{end}}
+
+            <button hx-post="bus/monitor" hx-swap="outerHTML">Monitor all events</button>
+        </section>
+
+        <section>
+            <h2>Clients</h2>
+
+            <table>
+                <thead>
+                    <tr>
+                        <th>Name</th>
+                        <th>Publishing</th>
+                        <th>Subscribing</th>
+                        <th>Pending</th>
+                    </tr>
+                </thead>
+                {{range .Clients}}
+                <tr id="{{.Name}}">
+                    <td>{{.Name}}</td>
+                    <td class="list">
+                        <ul>
+                            {{range .Publish}}
+                            <li><a href="#{{.}}">{{.}}</a></li>
+                            {{end}}
+                        </ul>
+                    </td>
+                    <td class="list">
+                        <ul>
+                            {{range .Subscribe}}
+                            <li><a href="#{{.}}">{{.}}</a></li>
+                            {{end}}
+                        </ul>
+                    </td>
+                    <td>
+                        {{len ($.SubscribeQueue .Client)}}
+                    </td>
+                </tr>
+                {{end}}
+            </table>
+        </section>
+
+        <section>
+            <h2>Types</h2>
+
+            {{range .Types}}
+
+            <section id="{{.}}">
+            <h3>{{.Name}}</h3>
+            <h4>Definition</h4>
+            <code>{{prettyPrintStruct .}}</code>
+
+            <h4>Published by:</h4>
+            {{if len (.Publish)}}
+            <ul>
+                {{range .Publish}}
+                <li><a href="#{{.Name}}">{{.Name}}</a></li>
+                {{end}}
+            </ul>
+            {{else}}
+            <ul>
+                <li>No publishers.</li>
+            </ul>
+            {{end}}
+
+            <h4>Received by:</h4>
+            {{if len (.Subscribe)}}
+            <ul>
+                {{range .Subscribe}}
+                <li><a href="#{{.Name}}">{{.Name}}</a></li>
+                {{end}}
+            </ul>
+            {{else}}
+            <ul>
+                <li>No subscribers.</li>
+            </ul>
+            {{end}}
+            </section>
+            {{end}}
+
+        </section>
+    </body>
+</html>

util/eventbus/assets/monitor.html

@@ -0,0 +1,5 @@
+<div>
+<ul id="monitor" ws-connect="bus/monitor">
+</ul>
+<button hx-get="bus" hx-target="body">Stop monitoring</button>
+</div>

util/eventbus/assets/style.css

@@ -0,0 +1,90 @@
+/* CSS reset, thanks Josh Comeau: https://www.joshwcomeau.com/css/custom-css-reset/ */
+*, *::before, *::after { box-sizing: border-box; }
+* { margin: 0; }
+input, button, textarea, select { font: inherit; }
+p, h1, h2, h3, h4, h5, h6 { overflow-wrap: break-word; }
+p { text-wrap: pretty; }
+h1, h2, h3, h4, h5, h6 { text-wrap: balance; }
+#root, #__next { isolation: isolate; }
+body {
+    line-height: 1.5;
+    -webkit-font-smoothing: antialiased;
+}
+img, picture, video, canvas, svg {
+  display: block;
+  max-width: 100%;
+}
+
+/* Local styling begins */
+
+body {
+    padding: 12px;
+}
+
+div {
+    width: 100%;
+}
+
+section {
+    display: flex;
+    flex-direction: column;
+    flex-gap: 6px;
+    align-items: flex-start;
+    padding: 12px 0;
+}
+
+section > * {
+    margin-left: 24px;
+}
+
+section > h2, section > h3 {
+    margin-left: 0;
+    padding-bottom: 6px;
+    padding-top: 12px;
+}
+
+details {
+    padding-bottom: 12px;
+}
+
+table {
+    table-layout: fixed;
+    width: calc(100% - 48px);
+    border-collapse: collapse;
+    border: 1px solid black;
+}
+
+th, td {
+    padding: 12px;
+    border: 1px solid black;
+}
+
+td.list {
+    vertical-align: top;
+}
+
+ul {
+    list-style: none;
+}
+
+td ul {
+    margin: 0;
+    padding: 0;
+}
+
+code {
+    padding: 12px;
+    white-space: pre;
+}
+
+#monitor {
+    width: calc(100% - 48px);
+    resize: vertical;
+    padding: 12px;
+    overflow: scroll;
+    height: 15lh;
+    border: 1px inset;
+    min-height: 1em;
+    display: flex;
+    flex-direction: column-reverse;
+}

util/eventbus/bench_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package eventbus_test
+
+import (
+	"math/rand/v2"
+	"testing"
+
+	"tailscale.com/util/eventbus"
+)
+
+func BenchmarkBasicThroughput(b *testing.B) {
+	bus := eventbus.New()
+	pcli := bus.Client(b.Name() + "-pub")
+	scli := bus.Client(b.Name() + "-sub")
+
+	type emptyEvent [0]byte
+
+	// One publisher and a corresponding subscriber shoveling events as fast as
+	// they can through the plumbing.
+	pub := eventbus.Publish[emptyEvent](pcli)
+	sub := eventbus.Subscribe[emptyEvent](scli)
+
+	go func() {
+		for {
+			select {
+			case <-sub.Events():
+				continue
+			case <-sub.Done():
+				return
+			}
+		}
+	}()
+
+	for b.Loop() {
+		pub.Publish(emptyEvent{})
+	}
+	bus.Close()
+}
+
+func BenchmarkSubsThroughput(b *testing.B) {
+	bus := eventbus.New()
+	pcli := bus.Client(b.Name() + "-pub")
+	scli1 := bus.Client(b.Name() + "-sub1")
+	scli2 := bus.Client(b.Name() + "-sub2")
+
+	type emptyEvent [0]byte
+
+	// One publisher and two subscribers shoveling events as fast as they can
+	// through the plumbing.
+	pub := eventbus.Publish[emptyEvent](pcli)
+	sub1 := eventbus.Subscribe[emptyEvent](scli1)
+	sub2 := eventbus.Subscribe[emptyEvent](scli2)
+
+	for _, sub := range []*eventbus.Subscriber[emptyEvent]{sub1, sub2} {
+		go func() {
+			for {
+				select {
+				case <-sub.Events():
+					continue
+				case <-sub.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	for b.Loop() {
+		pub.Publish(emptyEvent{})
+	}
+	bus.Close()
+}
+
+func BenchmarkMultiThroughput(b *testing.B) {
+	bus := eventbus.New()
+	cli := bus.Client(b.Name())
+
+	type eventA struct{}
+	type eventB struct{}
+
+	// Two disjoint event streams routed through the global order.
+	apub := eventbus.Publish[eventA](cli)
+	asub := eventbus.Subscribe[eventA](cli)
+	bpub := eventbus.Publish[eventB](cli)
+	bsub := eventbus.Subscribe[eventB](cli)
+
+	go func() {
+		for {
+			select {
+			case <-asub.Events():
+				continue
+			case <-asub.Done():
+				return
+			}
+		}
+	}()
+	go func() {
+		for {
+			select {
+			case <-bsub.Events():
+				continue
+			case <-bsub.Done():
+				return
+			}
+		}
+	}()
+
+	var rng uint64
+	var bits int
+	for b.Loop() {
+		if bits == 0 {
+			rng = rand.Uint64()
+			bits = 64
+		}
+		if rng&1 == 0 {
+			apub.Publish(eventA{})
+		} else {
+			bpub.Publish(eventB{})
+		}
+		rng >>= 1
+		bits--
+	}
+	bus.Close()
+}

util/eventbus/bus.go

@@ -73,8 +73,8 @@ func (b *Bus) Client(name string) *Client {
 }
 
 // Debugger returns the debugging facility for the bus.
-func (b *Bus) Debugger() Debugger {
-	return Debugger{b}
+func (b *Bus) Debugger() *Debugger {
+	return &Debugger{b}
 }
 
 // Close closes the bus. Implicitly closes all clients, publishers and

util/eventbus/debug-demo/main.go

@@ -0,0 +1,103 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// debug-demo is a program that serves a bus's debug interface over
+// HTTP, then generates some fake traffic from a handful of
+// clients. It is an aid to development, to have something to present
+// on the debug interfaces while writing them.
+package main
+
+import (
+	"log"
+	"math/rand/v2"
+	"net/http"
+	"net/netip"
+	"time"
+
+	"tailscale.com/tsweb"
+	"tailscale.com/types/key"
+	"tailscale.com/util/eventbus"
+)
+
+func main() {
+	b := eventbus.New()
+	c := b.Client("RouteMonitor")
+	go testPub[RouteAdded](c, 5*time.Second)
+	go testPub[RouteRemoved](c, 5*time.Second)
+	c = b.Client("ControlClient")
+	go testPub[PeerAdded](c, 3*time.Second)
+	go testPub[PeerRemoved](c, 6*time.Second)
+	c = b.Client("Portmapper")
+	go testPub[PortmapAcquired](c, 10*time.Second)
+	go testPub[PortmapLost](c, 15*time.Second)
+	go testSub[RouteAdded](c)
+	c = b.Client("WireguardConfig")
+	go testSub[PeerAdded](c)
+	go testSub[PeerRemoved](c)
+	c = b.Client("Magicsock")
+	go testPub[PeerPathChanged](c, 5*time.Second)
+	go testSub[RouteAdded](c)
+	go testSub[RouteRemoved](c)
+	go testSub[PortmapAcquired](c)
+	go testSub[PortmapLost](c)
+
+	m := http.NewServeMux()
+	d := tsweb.Debugger(m)
+	b.Debugger().RegisterHTTP(d)
+
+	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/debug/bus", http.StatusFound)
+	})
+	log.Printf("Serving debug interface at http://localhost:8185/debug/bus")
+	http.ListenAndServe(":8185", m)
+}
+
+func testPub[T any](c *eventbus.Client, every time.Duration) {
+	p := eventbus.Publish[T](c)
+	for {
+		jitter := time.Duration(rand.N(2000)) * time.Millisecond
+		time.Sleep(jitter)
+		var zero T
+		log.Printf("%s publish: %T", c.Name(), zero)
+		p.Publish(zero)
+		time.Sleep(every)
+	}
+}
+
+func testSub[T any](c *eventbus.Client) {
+	s := eventbus.Subscribe[T](c)
+	for v := range s.Events() {
+		log.Printf("%s received: %T", c.Name(), v)
+	}
+}
+
+type RouteAdded struct {
+	Prefix   netip.Prefix
+	Via      netip.Addr
+	Priority int
+}
+type RouteRemoved struct {
+	Prefix netip.Addr
+}
+
+type PeerAdded struct {
+	ID  int
+	Key key.NodePublic
+}
+type PeerRemoved struct {
+	ID  int
+	Key key.NodePublic
+}
+
+type PortmapAcquired struct {
+	Endpoint netip.Addr
+}
+type PortmapLost struct {
+	Endpoint netip.Addr
+}
+
+type PeerPathChanged struct {
+	ID         int
+	EndpointID int
+	Quality    int
+}

util/eventbus/debug.go

@@ -4,11 +4,14 @@
 package eventbus
 
 import (
+	"cmp"
 	"fmt"
 	"reflect"
 	"slices"
 	"sync"
 	"sync/atomic"
+
+	"tailscale.com/tsweb"
 )
 
 // A Debugger offers access to a bus's privileged introspection and
@@ -29,7 +32,11 @@ type Debugger struct {
 
 // Clients returns a list of all clients attached to the bus.
 func (d *Debugger) Clients() []*Client {
-	return d.bus.listClients()
+	ret := d.bus.listClients()
+	slices.SortFunc(ret, func(a, b *Client) int {
+		return cmp.Compare(a.Name(), b.Name())
+	})
+	return ret
 }
 
 // PublishQueue returns the contents of the publish queue.
@@ -130,6 +137,8 @@ func (d *Debugger) SubscribeTypes(client *Client) []reflect.Type {
 	return client.subscribeTypes()
 }
 
+func (d *Debugger) RegisterHTTP(td *tsweb.DebugHandler) { registerHTTPDebugger(d, td) }
+
 // A hook collects hook functions that can be run as a group.
 type hook[T any] struct {
 	sync.Mutex

util/eventbus/debughttp.go

@@ -0,0 +1,240 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !ios
+
+package eventbus
+
+import (
+	"bytes"
+	"cmp"
+	"embed"
+	"fmt"
+	"html/template"
+	"io"
+	"io/fs"
+	"log"
+	"net/http"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/coder/websocket"
+	"tailscale.com/tsweb"
+)
+
+type httpDebugger struct {
+	*Debugger
+}
+
+func registerHTTPDebugger(d *Debugger, td *tsweb.DebugHandler) {
+	dh := httpDebugger{d}
+	td.Handle("bus", "Event bus", dh)
+	td.HandleSilent("bus/monitor", http.HandlerFunc(dh.serveMonitor))
+	td.HandleSilent("bus/style.css", serveStatic("style.css"))
+	td.HandleSilent("bus/htmx.min.js", serveStatic("htmx.min.js.gz"))
+	td.HandleSilent("bus/htmx-websocket.min.js", serveStatic("htmx-websocket.min.js.gz"))
+}
+
+//go:embed assets/*.html
+var templatesSrc embed.FS
+
+var templates = sync.OnceValue(func() *template.Template {
+	d, err := fs.Sub(templatesSrc, "assets")
+	if err != nil {
+		panic(fmt.Errorf("getting eventbus debughttp templates subdir: %w", err))
+	}
+	ret := template.New("").Funcs(map[string]any{
+		"prettyPrintStruct": prettyPrintStruct,
+	})
+	return template.Must(ret.ParseFS(d, "*"))
+})
+
+//go:generate go run fetch-htmx.go
+
+//go:embed assets/*.css assets/*.min.js.gz
+var static embed.FS
+
+func serveStatic(name string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasSuffix(name, ".css"):
+			w.Header().Set("Content-Type", "text/css")
+		case strings.HasSuffix(name, ".min.js.gz"):
+			w.Header().Set("Content-Type", "text/javascript")
+			w.Header().Set("Content-Encoding", "gzip")
+		case strings.HasSuffix(name, ".js"):
+			w.Header().Set("Content-Type", "text/javascript")
+		default:
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+
+		f, err := static.Open(filepath.Join("assets", name))
+		if err != nil {
+			http.Error(w, fmt.Sprintf("opening asset: %v", err), http.StatusInternalServerError)
+			return
+		}
+		defer f.Close()
+		if _, err := io.Copy(w, f); err != nil {
+			http.Error(w, fmt.Sprintf("serving asset: %v", err), http.StatusInternalServerError)
+			return
+		}
+	})
+}
+
+func render(w http.ResponseWriter, name string, data any) {
+	err := templates().ExecuteTemplate(w, name+".html", data)
+	if err != nil {
+		err := fmt.Errorf("rendering template: %v", err)
+		log.Print(err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+func (h httpDebugger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	type clientInfo struct {
+		*Client
+		Publish   []reflect.Type
+		Subscribe []reflect.Type
+	}
+	type typeInfo struct {
+		reflect.Type
+		Publish   []*Client
+		Subscribe []*Client
+	}
+	type info struct {
+		*Debugger
+		Clients map[string]*clientInfo
+		Types   map[string]*typeInfo
+	}
+
+	data := info{
+		Debugger: h.Debugger,
+		Clients:  map[string]*clientInfo{},
+		Types:    map[string]*typeInfo{},
+	}
+
+	getTypeInfo := func(t reflect.Type) *typeInfo {
+		if data.Types[t.Name()] == nil {
+			data.Types[t.Name()] = &typeInfo{
+				Type: t,
+			}
+		}
+		return data.Types[t.Name()]
+	}
+
+	for _, c := range h.Clients() {
+		ci := &clientInfo{
+			Client:    c,
+			Publish:   h.PublishTypes(c),
+			Subscribe: h.SubscribeTypes(c),
+		}
+		slices.SortFunc(ci.Publish, func(a, b reflect.Type) int { return cmp.Compare(a.Name(), b.Name()) })
+		slices.SortFunc(ci.Subscribe, func(a, b reflect.Type) int { return cmp.Compare(a.Name(), b.Name()) })
+		data.Clients[c.Name()] = ci
+
+		for _, t := range ci.Publish {
+			ti := getTypeInfo(t)
+			ti.Publish = append(ti.Publish, c)
+		}
+		for _, t := range ci.Subscribe {
+			ti := getTypeInfo(t)
+			ti.Subscribe = append(ti.Subscribe, c)
+		}
+	}
+
+	render(w, "main", data)
+}
+
+func (h httpDebugger) serveMonitor(w http.ResponseWriter, r *http.Request) {
+	if r.Header.Get("Upgrade") == "websocket" {
+		h.serveMonitorStream(w, r)
+		return
+	}
+
+	render(w, "monitor", nil)
+}
+
+func (h httpDebugger) serveMonitorStream(w http.ResponseWriter, r *http.Request) {
+	conn, err := websocket.Accept(w, r, nil)
+	if err != nil {
+		return
+	}
+	defer conn.CloseNow()
+	wsCtx := conn.CloseRead(r.Context())
+
+	mon := h.WatchBus()
+	defer mon.Close()
+
+	i := 0
+	for {
+		select {
+		case <-r.Context().Done():
+			return
+		case <-wsCtx.Done():
+			return
+		case <-mon.Done():
+			return
+		case event := <-mon.Events():
+			msg, err := conn.Writer(r.Context(), websocket.MessageText)
+			if err != nil {
+				return
+			}
+			data := map[string]any{
+				"Count": i,
+				"Type":  reflect.TypeOf(event.Event),
+				"Event": event,
+			}
+			i++
+			if err := templates().ExecuteTemplate(msg, "event.html", data); err != nil {
+				log.Println(err)
+				return
+			}
+			if err := msg.Close(); err != nil {
+				return
+			}
+		}
+	}
+}
+
+func prettyPrintStruct(t reflect.Type) string {
+	if t.Kind() != reflect.Struct {
+		return t.String()
+	}
+	var rec func(io.Writer, int, reflect.Type)
+	rec = func(out io.Writer, indent int, t reflect.Type) {
+		ind := strings.Repeat("    ", indent)
+		fmt.Fprintf(out, "%s", t.String())
+		fs := collectFields(t)
+		if len(fs) > 0 {
+			io.WriteString(out, " {\n")
+			for _, f := range fs {
+				fmt.Fprintf(out, "%s    %s ", ind, f.Name)
+				if f.Type.Kind() == reflect.Struct {
+					rec(out, indent+1, f.Type)
+				} else {
+					fmt.Fprint(out, f.Type)
+				}
+				io.WriteString(out, "\n")
+			}
+			fmt.Fprintf(out, "%s}", ind)
+		}
+	}
+
+	var ret bytes.Buffer
+	rec(&ret, 0, t)
+	return ret.String()
+}
+
+func collectFields(t reflect.Type) (ret []reflect.StructField) {
+	for _, f := range reflect.VisibleFields(t) {
+		if !f.IsExported() {
+			continue
+		}
+		ret = append(ret, f)
+	}
+	return ret
+}

util/eventbus/debughttp_ios.go

@@ -0,0 +1,18 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build ios
+
+package eventbus
+
+import "tailscale.com/tsweb"
+
+func registerHTTPDebugger(d *Debugger, td *tsweb.DebugHandler) {
+	// The event bus debugging UI uses html/template, which uses
+	// reflection for method lookups. This forces the compiler to
+	// retain a lot more code and information to make dynamic method
+	// dispatch work, which is unacceptable bloat for the iOS build.
+	//
+	// TODO: https://github.com/tailscale/tailscale/issues/15297 to
+	// bring the debug UI back to iOS somehow.
+}

util/eventbus/fetch-htmx.go

@@ -0,0 +1,93 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build ignore
+
+// Program fetch-htmx fetches and installs local copies of the HTMX
+// library and its dependencies, used by the debug UI. It is meant to
+// be run via go generate.
+package main
+
+import (
+	"compress/gzip"
+	"crypto/sha512"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+)
+
+func main() {
+	// Hash from https://htmx.org/docs/#installing
+	htmx, err := fetchHashed("https://unpkg.com/htmx.org@2.0.4", "HGfztofotfshcF7+8n44JQL2oJmowVChPTg48S+jvZoztPfvwD79OC/LTtG6dMp+")
+	if err != nil {
+		log.Fatalf("fetching htmx: %v", err)
+	}
+
+	// Hash SHOULD be from https://htmx.org/extensions/ws/ , but the
+	// hash is currently incorrect, see
+	// https://github.com/bigskysoftware/htmx-extensions/issues/153
+	//
+	// Until that bug is resolved, hash was obtained by rebuilding the
+	// extension from git source, and verifying that the hash matches
+	// what unpkg is serving.
+	ws, err := fetchHashed("https://unpkg.com/htmx-ext-ws@2.0.2", "932iIqjARv+Gy0+r6RTGrfCkCKS5MsF539Iqf6Vt8L4YmbnnWI2DSFoMD90bvXd0")
+	if err != nil {
+		log.Fatalf("fetching htmx-websockets: %v", err)
+	}
+
+	if err := writeGz("assets/htmx.min.js.gz", htmx); err != nil {
+		log.Fatalf("writing htmx.min.js.gz: %v", err)
+	}
+	if err := writeGz("assets/htmx-websocket.min.js.gz", ws); err != nil {
+		log.Fatalf("writing htmx-websocket.min.js.gz: %v", err)
+	}
+}
+
+func writeGz(path string, bs []byte) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	g, err := gzip.NewWriterLevel(f, gzip.BestCompression)
+	if err != nil {
+		return err
+	}
+
+	if _, err := g.Write(bs); err != nil {
+		return err
+	}
+
+	if err := g.Flush(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func fetchHashed(url, wantHash string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetching %q returned error status: %s", url, resp.Status)
+	}
+	ret, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading file from %q: %v", url, err)
+	}
+	h := sha512.Sum384(ret)
+	got := base64.StdEncoding.EncodeToString(h[:])
+	if got != wantHash {
+		return nil, fmt.Errorf("wrong hash for %q: got %q, want %q", url, got, wantHash)
+	}
+	return ret, nil
+}

wgengine/magicsock/magicsock.go

@@ -177,6 +177,10 @@ type Conn struct {
 	// port mappings from NAT devices.
 	portMapper *portmapper.Client
 
+	// portMapperLogfUnregister is the function to call to unregister
+	// the portmapper log limiter.
+	portMapperLogfUnregister func()
+
 	// derpRecvCh is used by receiveDERP to read DERP messages.
 	// It must have buffer size > 0; see issue 3736.
 	derpRecvCh chan derpReadResult
@@ -532,10 +536,15 @@ func NewConn(opts Options) (*Conn, error) {
 	c.idleFunc = opts.IdleFunc
 	c.testOnlyPacketListener = opts.TestOnlyPacketListener
 	c.noteRecvActivity = opts.NoteRecvActivity
+
+	// Don't log the same log messages possibly every few seconds in our
+	// portmapper.
+	portmapperLogf := logger.WithPrefix(c.logf, "portmapper: ")
+	portmapperLogf, c.portMapperLogfUnregister = netmon.LinkChangeLogLimiter(portmapperLogf, opts.NetMon)
 	portMapOpts := &portmapper.DebugKnobs{
 		DisableAll: func() bool { return opts.DisablePortMapper || c.onlyTCP443.Load() },
 	}
-	c.portMapper = portmapper.NewClient(logger.WithPrefix(c.logf, "portmapper: "), opts.NetMon, portMapOpts, opts.ControlKnobs, c.onPortMapChanged)
+	c.portMapper = portmapper.NewClient(portmapperLogf, opts.NetMon, portMapOpts, opts.ControlKnobs, c.onPortMapChanged)
 	c.portMapper.SetGatewayLookupFunc(opts.NetMon.GatewayAndSelfIP)
 	c.netMon = opts.NetMon
 	c.health = opts.HealthTracker
@@ -2481,6 +2490,7 @@ func (c *Conn) Close() error {
 	}
 	c.stopPeriodicReSTUNTimerLocked()
 	c.portMapper.Close()
+	c.portMapperLogfUnregister()
 
 	c.peerMap.forEachEndpoint(func(ep *endpoint) {
 		ep.stopAndReset()

wgengine/userspace.go

@@ -1580,6 +1580,11 @@ type fwdDNSLinkSelector struct {
 }
 
 func (ls fwdDNSLinkSelector) PickLink(ip netip.Addr) (linkName string) {
+	// sandboxed macOS needs some extra hand-holding for loopback addresses.
+	if ip.IsLoopback() && version.IsSandboxedMacOS() {
+		return "lo0"
+	}
+
 	if ls.ue.isDNSIPOverTailscale.Load()(ip) {
 		return ls.tunName
 	}