tailcfg: change MapResponse.PacketFilter from omitempty to omitzero

Updates #test

.github/workflows/govulncheck.yml

@@ -30,7 +30,7 @@ jobs:
           token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
           payload: |
             {
-              "channel": "C05PXRM304B",
+              "channel": "C08FGKZCQTW",
               "blocks": [
                 {
                   "type": "section",

client/tailscale/devices.go

@@ -79,6 +79,13 @@ type Device struct {
 	// Tailscale have attempted to collect this from the device but it has not
 	// opted in, PostureIdentity will have Disabled=true.
 	PostureIdentity *DevicePostureIdentity `json:"postureIdentity"`
+
+	// TailnetLockKey is the tailnet lock public key of the node as a hex string.
+	TailnetLockKey string `json:"tailnetLockKey,omitempty"`
+
+	// TailnetLockErr indicates an issue with the tailnet lock node-key signature
+	// on this device. This field is only populated when tailnet lock is enabled.
+	TailnetLockErr string `json:"tailnetLockError,omitempty"`
 }
 
 type DevicePostureIdentity struct {

client/web/web.go

@@ -335,7 +335,8 @@ func (s *Server) requireTailscaleIP(w http.ResponseWriter, r *http.Request) (han
 		ipv6ServiceHost = "[" + tsaddr.TailscaleServiceIPv6String + "]"
 	)
 	// allow requests on quad-100 (or ipv6 equivalent)
-	if r.Host == ipv4ServiceHost || r.Host == ipv6ServiceHost {
+	host := strings.TrimSuffix(r.Host, ":80")
+	if host == ipv4ServiceHost || host == ipv6ServiceHost {
 		return false
 	}
 

client/web/web_test.go

@@ -1177,6 +1177,16 @@ func TestRequireTailscaleIP(t *testing.T) {
 			target:      "http://[fd7a:115c:a1e0::53]/",
 			wantHandled: false,
 		},
+		{
+			name:        "quad-100:80",
+			target:      "http://100.100.100.100:80/",
+			wantHandled: false,
+		},
+		{
+			name:        "ipv6-service-addr:80",
+			target:      "http://[fd7a:115c:a1e0::53]:80/",
+			wantHandled: false,
+		},
 	}
 
 	for _, tt := range tests {

cmd/containerboot/certs.go

@@ -0,0 +1,147 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/util/goroutines"
+	"tailscale.com/util/mak"
+)
+
+// certManager is responsible for issuing certificates for known domains and for
+// maintaining a loop that re-attempts issuance daily.
+// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for
+// HA Ingress HTTPS endpoints ('write' replicas).
+type certManager struct {
+	lc      localClient
+	tracker goroutines.Tracker // tracks running goroutines
+	mu      sync.Mutex         // guards the following
+	// certLoops contains a map of DNS names, for which we currently need to
+	// manage certs to cancel functions that allow stopping a goroutine when
+	// we no longer need to manage certs for the DNS name.
+	certLoops map[string]context.CancelFunc
+}
+
+// ensureCertLoops ensures that, for all currently managed Service HTTPS
+// endpoints, there is a cert loop responsible for issuing and ensuring the
+// renewal of the TLS certs.
+// ServeConfig must not be nil.
+func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error {
+	if sc == nil {
+		return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
+	}
+	currentDomains := make(map[string]bool)
+	const httpsPort = "443"
+	for _, service := range sc.Services {
+		for hostPort := range service.Web {
+			domain, port, err := net.SplitHostPort(string(hostPort))
+			if err != nil {
+				return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
+			}
+			if port != httpsPort { // HA Ingress' HTTP endpoint
+				continue
+			}
+			currentDomains[domain] = true
+		}
+	}
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	for domain := range currentDomains {
+		if _, exists := cm.certLoops[domain]; !exists {
+			cancelCtx, cancel := context.WithCancel(ctx)
+			mak.Set(&cm.certLoops, domain, cancel)
+			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
+		}
+	}
+
+	// Stop goroutines for domain names that are no longer in the config.
+	for domain, cancel := range cm.certLoops {
+		if !currentDomains[domain] {
+			cancel()
+			delete(cm.certLoops, domain)
+		}
+	}
+	return nil
+}
+
+// runCertLoop:
+// - calls localAPI certificate endpoint to ensure that certs are issued for the
+// given domain name
+// - calls localAPI certificate endpoint daily to ensure that certs are renewed
+// - if certificate issuance failed retries after an exponential backoff period
+// starting at 1 minute and capped at 24 hours. Reset the backoff once issuance succeeds.
+// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get
+// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write'
+// replica does not get any HTTPS requests.
+// https://letsencrypt.org/docs/integration-guide/#retrying-failures
+func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
+	const (
+		normalInterval   = 24 * time.Hour  // regular renewal check
+		initialRetry     = 1 * time.Minute // initial backoff after a failure
+		maxRetryInterval = 24 * time.Hour  // max backoff period
+	)
+	timer := time.NewTimer(0) // fire off timer immediately
+	defer timer.Stop()
+	retryCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			// We call the certificate endpoint, but don't do anything
+			// with the returned certs here.
+			// The call to the certificate endpoint will ensure that
+			// certs are issued/renewed as needed and stored in the
+			// relevant state store. For example, for HA Ingress
+			// 'write' replica, the cert and key will be stored in a
+			// Kubernetes Secret named after the domain for which we
+			// are issuing.
+			// Note that renewals triggered by the call to the
+			// certificates endpoint here and by renewal check
+			// triggered during a call to node's HTTPS endpoint
+			// share the same state/renewal lock mechanism, so we
+			// should not run into redundant issuances during
+			// concurrent renewal checks.
+			// TODO(irbekrm): maybe it is worth adding a new
+			// issuance endpoint that explicitly only triggers
+			// issuance and stores certs in the relevant store, but
+			// does not return certs to the caller?
+			_, _, err := cm.lc.CertPair(ctx, domain)
+			if err != nil {
+				log.Printf("error refreshing certificate for %s: %v", domain, err)
+			}
+			var nextInterval time.Duration
+			// TODO(irbekrm): distinguish between LE rate limit
+			// errors and other error types like transient network
+			// errors.
+			if err == nil {
+				retryCount = 0
+				nextInterval = normalInterval
+			} else {
+				retryCount++
+				// Calculate backoff: initialRetry * 2^(retryCount-1)
+				// For retryCount=1: 1min * 2^0 = 1min
+				// For retryCount=2: 1min * 2^1 = 2min
+				// For retryCount=3: 1min * 2^2 = 4min
+				backoff := initialRetry * time.Duration(1<<(retryCount-1))
+				if backoff > maxRetryInterval {
+					backoff = maxRetryInterval
+				}
+				nextInterval = backoff
+				log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n",
+					domain, retryCount, err, nextInterval)
+			}
+			timer.Reset(nextInterval)
+		}
+	}
+}

cmd/containerboot/certs_test.go

@@ -0,0 +1,229 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+)
+
+// TestEnsureCertLoops tests that the certManager correctly starts and stops
+// update loops for certs when the serve config changes. It tracks goroutine
+// count and uses that as a validator that the expected number of cert loops are
+// running.
+func TestEnsureCertLoops(t *testing.T) {
+	tests := []struct {
+		name              string
+		initialConfig     *ipn.ServeConfig
+		updatedConfig     *ipn.ServeConfig
+		initialGoroutines int64 // after initial serve config is applied
+		updatedGoroutines int64 // after updated serve config is applied
+		wantErr           bool
+	}{
+		{
+			name:              "empty_serve_config",
+			initialConfig:     &ipn.ServeConfig{},
+			initialGoroutines: 0,
+		},
+		{
+			name:              "nil_serve_config",
+			initialConfig:     nil,
+			initialGoroutines: 0,
+			wantErr:           true,
+		},
+		{
+			name:          "empty_to_one_service",
+			initialConfig: &ipn.ServeConfig{},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 0,
+			updatedGoroutines: 1,
+		},
+		{
+			name: "single_service",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+		},
+		{
+			name: "multiple_services",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // one loop per domain across all services
+		},
+		{
+			name: "ignore_non_https_ports",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+							"my-app.tailnetxyz.ts.net:80":  {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1, // only one loop for the 443 endpoint
+		},
+		{
+			name: "remove_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 2, // initially two loops (one per service)
+			updatedGoroutines: 1, // one loop after removing service2
+		},
+		{
+			name: "add_domain",
+			initialConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			updatedConfig: &ipn.ServeConfig{
+				Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
+					"svc:my-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+					"svc:my-other-app": {
+						Web: map[ipn.HostPort]*ipn.WebServerConfig{
+							"my-other-app.tailnetxyz.ts.net:443": {},
+						},
+					},
+				},
+			},
+			initialGoroutines: 1,
+			updatedGoroutines: 2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			cm := &certManager{
+				lc:        &fakeLocalClient{},
+				certLoops: make(map[string]context.CancelFunc),
+			}
+
+			allDone := make(chan bool, 1)
+			defer cm.tracker.AddDoneCallback(func() {
+				cm.mu.Lock()
+				defer cm.mu.Unlock()
+				if cm.tracker.RunningGoroutines() > 0 {
+					return
+				}
+				select {
+				case allDone <- true:
+				default:
+				}
+			})()
+
+			err := cm.ensureCertLoops(ctx, tt.initialConfig)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ensureCertLoops() error = %v", err)
+			}
+
+			if got := cm.tracker.RunningGoroutines(); got != tt.initialGoroutines {
+				t.Errorf("after initial config: got %d running goroutines, want %d", got, tt.initialGoroutines)
+			}
+
+			if tt.updatedConfig != nil {
+				if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil {
+					t.Fatalf("ensureCertLoops() error on update = %v", err)
+				}
+
+				// Although starting goroutines and cancelling
+				// the context happens in the main goroutine, it
+				// the actual goroutine exit when a context is
+				// cancelled does not- so wait for a bit for the
+				// running goroutine count to reach the expected
+				// number.
+				deadline := time.After(5 * time.Second)
+				for {
+					if got := cm.tracker.RunningGoroutines(); got == tt.updatedGoroutines {
+						break
+					}
+					select {
+					case <-deadline:
+						t.Fatalf("timed out waiting for goroutine count to reach %d, currently at %d",
+							tt.updatedGoroutines, cm.tracker.RunningGoroutines())
+					case <-time.After(10 * time.Millisecond):
+						continue
+					}
+				}
+			}
+
+			if tt.updatedGoroutines == 0 {
+				return // no goroutines to wait for
+			}
+			// cancel context to make goroutines exit
+			cancel()
+			select {
+			case <-time.After(5 * time.Second):
+				t.Fatal("timed out waiting for goroutine to finish")
+			case <-allDone:
+			}
+		})
+	}
+}

cmd/containerboot/main.go

@@ -646,7 +646,7 @@ runLoop:
 
 				if cfg.ServeConfigPath != "" {
 					triggerWatchServeConfigChanges.Do(func() {
-						go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc)
+						go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg)
 					})
 				}
 

cmd/containerboot/serve.go

@@ -28,10 +28,11 @@ import (
 // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
 // is written to when the certDomain changes, causing the serve config to be
 // re-read and applied.
-func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient) {
+func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings) {
 	if certDomainAtomic == nil {
 		panic("certDomainAtomic must not be nil")
 	}
+
 	var tickChan <-chan time.Time
 	var eventChan <-chan fsnotify.Event
 	if w, err := fsnotify.NewWatcher(); err != nil {
@@ -43,7 +44,7 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 		tickChan = ticker.C
 	} else {
 		defer w.Close()
-		if err := w.Add(filepath.Dir(path)); err != nil {
+		if err := w.Add(filepath.Dir(cfg.ServeConfigPath)); err != nil {
 			log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
 		}
 		eventChan = w.Events
@@ -51,6 +52,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 
 	var certDomain string
 	var prevServeConfig *ipn.ServeConfig
+	var cm certManager
+	if cfg.CertShareMode == "rw" {
+		cm = certManager{
+			lc: lc,
+		}
+	}
 	for {
 		select {
 		case <-ctx.Done():
@@ -63,12 +70,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			// k8s handles these mounts. So just re-read the file and apply it
 			// if it's changed.
 		}
-		sc, err := readServeConfig(path, certDomain)
+		sc, err := readServeConfig(cfg.ServeConfigPath, certDomain)
 		if err != nil {
 			log.Fatalf("serve proxy: failed to read serve config: %v", err)
 		}
 		if sc == nil {
-			log.Printf("serve proxy: no serve config at %q, skipping", path)
+			log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath)
 			continue
 		}
 		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
@@ -83,6 +90,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
 			}
 		}
 		prevServeConfig = sc
+		if cfg.CertShareMode != "rw" {
+			continue
+		}
+		if err := cm.ensureCertLoops(ctx, sc); err != nil {
+			log.Fatalf("serve proxy: error ensuring cert loops: %v", err)
+		}
 	}
 }
 
@@ -96,6 +109,7 @@ func certDomainFromNetmap(nm *netmap.NetworkMap) string {
 // localClient is a subset of [local.Client] that can be mocked for testing.
 type localClient interface {
 	SetServeConfig(context.Context, *ipn.ServeConfig) error
+	CertPair(context.Context, string) ([]byte, []byte, error)
 }
 
 func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error {

cmd/containerboot/serve_test.go

@@ -206,6 +206,10 @@ func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConf
 	return nil
 }
 
+func (m *fakeLocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) {
+	return nil, nil, nil
+}
+
 func TestHasHTTPSEndpoint(t *testing.T) {
 	tests := []struct {
 		name string

cmd/containerboot/settings.go

@@ -74,6 +74,12 @@ type settings struct {
 	HealthCheckEnabled   bool
 	DebugAddrPort        string
 	EgressProxiesCfgPath string
+	// CertShareMode is set for Kubernetes Pods running cert share mode.
+	// Possible values are empty (containerboot doesn't run any certs
+	// logic),  'ro' (for Pods that shold never attempt to issue/renew
+	// certs) and 'rw' for Pods that should manage the TLS certs shared
+	// amongst the replicas.
+	CertShareMode string
 }
 
 func configFromEnv() (*settings, error) {
@@ -128,6 +134,17 @@ func configFromEnv() (*settings, error) {
 			cfg.PodIPv6 = parsed.String()
 		}
 	}
+	// If cert share is enabled, set the replica as read or write. Only 0th
+	// replica should be able to write.
+	isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false)
+	if isInCertShareMode {
+		cfg.CertShareMode = "ro"
+		podName := os.Getenv("POD_NAME")
+		if strings.HasSuffix(podName, "-0") {
+			cfg.CertShareMode = "rw"
+		}
+	}
+
 	if err := cfg.validate(); err != nil {
 		return nil, fmt.Errorf("invalid configuration: %v", err)
 	}

cmd/containerboot/tailscaled.go

@@ -33,6 +33,9 @@ func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Pro
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
+	if cfg.CertShareMode != "" {
+		cmd.Env = append(os.Environ(), "TS_CERT_SHARE_MODE="+cfg.CertShareMode)
+	}
 	log.Printf("Starting tailscaled")
 	if err := cmd.Start(); err != nil {
 		return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err)

cmd/derper/depaware.txt

@@ -96,6 +96,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/drive                                          from tailscale.com/client/local+
         tailscale.com/envknob                                        from tailscale.com/client/local+
+        tailscale.com/feature                                        from tailscale.com/tsweb
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/local
@@ -128,8 +129,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/derp
-        tailscale.com/tsweb                                          from tailscale.com/cmd/derper
-        tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb                                          from tailscale.com/cmd/derper+
+        tailscale.com/tsweb/promvarz                                 from tailscale.com/cmd/derper
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
@@ -309,7 +310,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         html                                                         from net/http/pprof+
         html/template                                                from tailscale.com/cmd/derper
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -319,12 +320,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         internal/filepathlite                                        from os+
         internal/fmtsort                                             from fmt+
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
-        internal/godebug                                             from crypto/tls+
+        internal/godebug                                             from crypto/internal/fips140deps/godebug+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/derper/derper.go

@@ -49,6 +49,9 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/derpprobe/derpprobe.go

@@ -15,6 +15,9 @@ import (
 	"tailscale.com/prober"
 	"tailscale.com/tsweb"
 	"tailscale.com/version"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/k8s-operator/depaware.txt

@@ -1151,7 +1151,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -1163,11 +1163,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
         internal/godebug                                             from archive/tar+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
         internal/lazyregexp                                          from go/doc
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -75,7 +75,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
-  verbs: ["get", "create", "patch", "update", "list", "watch"]
+  verbs: ["get", "create", "patch", "update", "list", "watch", "deletecollection"]
 - apiGroups: ["monitoring.coreos.com"]
   resources: ["servicemonitors"]
   verbs: ["get", "list", "update", "create", "delete"]

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -4898,6 +4898,7 @@ rules:
         - update
         - list
         - watch
+        - deletecollection
     - apiGroups:
         - monitoring.coreos.com
       resources:

cmd/k8s-operator/dnsrecords_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 )
@@ -163,10 +164,10 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 			Name:      o.GetName(),
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      o.GetName(),
-				LabelParentNamespace: o.GetNamespace(),
-				LabelParentType:      typ,
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        o.GetName(),
+				LabelParentNamespace:   o.GetNamespace(),
+				LabelParentType:        typ,
 			},
 		},
 		Spec: corev1.ServiceSpec{

cmd/k8s-operator/egress-pod-readiness.go

@@ -112,9 +112,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
 	}
 	// Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup.
 	lbls := map[string]string{
-		LabelManaged:    "true",
-		labelProxyGroup: proxyGroupName,
-		labelSvcType:    typeEgress,
+		kubetypes.LabelManaged: "true",
+		labelProxyGroup:        proxyGroupName,
+		labelSvcType:           typeEgress,
 	}
 	svcs := &corev1.ServiceList{}
 	if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil {

cmd/k8s-operator/egress-pod-readiness_test.go

@@ -450,9 +450,9 @@ func newSvc(name string, port int32) (*corev1.Service, string) {
 			Namespace: "operator-ns",
 			Name:      name,
 			Labels: map[string]string{
-				LabelManaged:    "true",
-				labelProxyGroup: "dev",
-				labelSvcType:    typeEgress,
+				kubetypes.LabelManaged: "true",
+				labelProxyGroup:        "dev",
+				labelSvcType:           typeEgress,
 			},
 		},
 		Spec: corev1.ServiceSpec{},

cmd/k8s-operator/egress-services.go

@@ -680,12 +680,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 // should probably validate and truncate (?) the names is they are too long.
 func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentType:      "svc",
-		LabelParentName:      svc.Name,
-		LabelParentNamespace: svc.Namespace,
-		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
-		labelSvcType:         typeEgress,
+		kubetypes.LabelManaged: "true",
+		LabelParentType:        "svc",
+		LabelParentName:        svc.Name,
+		LabelParentNamespace:   svc.Namespace,
+		labelProxyGroup:        svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:           typeEgress,
 	}
 }
 

cmd/k8s-operator/ingress-for-pg.go

@@ -22,6 +22,7 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -154,13 +155,13 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 	pg := &tsapi.ProxyGroup{}
 	if err := r.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil {
 		if apierrors.IsNotFound(err) {
-			logger.Infof("ProxyGroup %q does not exist", pgName)
+			logger.Infof("ProxyGroup does not exist")
 			return false, nil
 		}
 		return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
 	}
 	if !tsoperator.ProxyGroupIsReady(pg) {
-		logger.Infof("ProxyGroup %q is not (yet) ready", pgName)
+		logger.Infof("ProxyGroup is not (yet) ready")
 		return false, nil
 	}
 
@@ -175,8 +176,6 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		r.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
 	}
 
-	logger = logger.With("proxy-group", pg.Name)
-
 	if !slices.Contains(ing.Finalizers, FinalizerNamePG) {
 		// This log line is printed exactly once during initial provisioning,
 		// because once the finalizer is in place this block gets skipped. So,
@@ -242,8 +241,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidVIPService", msg)
 		return false, nil
 	}
+	// 3. Ensure that TLS Secret and RBAC exists
+	if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil {
+		return false, fmt.Errorf("error ensuring cert resources: %w", err)
+	}
 
-	// 3. Ensure that the serve config for the ProxyGroup contains the VIPService.
+	// 4. Ensure that the serve config for the ProxyGroup contains the VIPService.
 	cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
 	if err != nil {
 		return false, fmt.Errorf("error getting Ingress serve config: %w", err)
@@ -326,7 +329,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		!reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) ||
 		!reflect.DeepEqual(vipSvc.Ports, existingVIPSvc.Ports) ||
 		!strings.EqualFold(vipSvc.Comment, existingVIPSvc.Comment) {
-		logger.Infof("Ensuring VIPService %q exists and is up to date", hostname)
+		logger.Infof("Ensuring VIPService exists and is up to date")
 		if err := r.tsClient.CreateOrUpdateVIPService(ctx, vipSvc); err != nil {
 			return false, fmt.Errorf("error creating VIPService: %w", err)
 		}
@@ -338,31 +341,48 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		return false, fmt.Errorf("failed to update tailscaled config: %w", err)
 	}
 
-	// TODO(irbekrm): check that the replicas are ready to route traffic for the VIPService before updating Ingress
-	// status.
-	// 6. Update Ingress status
+	// 6. Update Ingress status if ProxyGroup Pods are ready.
+	count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName)
+	if err != nil {
+		return false, fmt.Errorf("failed to check if any Pods are configured: %w", err)
+	}
+
 	oldStatus := ing.Status.DeepCopy()
-	ports := []networkingv1.IngressPortStatus{
-		{
-			Protocol: "TCP",
-			Port:     443,
-		},
+
+	switch count {
+	case 0:
+		ing.Status.LoadBalancer.Ingress = nil
+	default:
+		ports := []networkingv1.IngressPortStatus{
+			{
+				Protocol: "TCP",
+				Port:     443,
+			},
+		}
+		if isHTTPEndpointEnabled(ing) {
+			ports = append(ports, networkingv1.IngressPortStatus{
+				Protocol: "TCP",
+				Port:     80,
+			})
+		}
+		ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
+			{
+				Hostname: dnsName,
+				Ports:    ports,
+			},
+		}
 	}
-	if isHTTPEndpointEnabled(ing) {
-		ports = append(ports, networkingv1.IngressPortStatus{
-			Protocol: "TCP",
-			Port:     80,
-		})
-	}
-	ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
-		{
-			Hostname: dnsName,
-			Ports:    ports,
-		},
-	}
-	if apiequality.Semantic.DeepEqual(oldStatus, ing.Status) {
+	if apiequality.Semantic.DeepEqual(oldStatus, &ing.Status) {
 		return svcsChanged, nil
 	}
+
+	const prefix = "Updating Ingress status"
+	if count == 0 {
+		logger.Infof("%s. No Pods are advertising VIPService yet", prefix)
+	} else {
+		logger.Infof("%s. %d Pod(s) advertising VIPService", prefix, count)
+	}
+
 	if err := r.Status().Update(ctx, ing); err != nil {
 		return false, fmt.Errorf("failed to update Ingress status: %w", err)
 	}
@@ -402,24 +422,24 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
 			logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipServiceName)
 
 			// Delete the VIPService from control if necessary.
-			svc, _ := r.tsClient.GetVIPService(ctx, vipServiceName)
-			if svc != nil && isVIPServiceForAnyIngress(svc) {
-				logger.Infof("cleaning up orphaned VIPService %q", vipServiceName)
-				svcsChanged, err = r.cleanupVIPService(ctx, vipServiceName, logger)
-				if err != nil {
-					errResp := &tailscale.ErrResponse{}
-					if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
-						return false, fmt.Errorf("deleting VIPService %q: %w", vipServiceName, err)
-					}
-				}
+			svcsChanged, err = r.cleanupVIPService(ctx, vipServiceName, logger)
+			if err != nil {
+				return false, fmt.Errorf("deleting VIPService %q: %w", vipServiceName, err)
 			}
 
 			// Make sure the VIPService is not advertised in tailscaled or serve config.
 			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
 				return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 			}
-			delete(cfg.Services, vipServiceName)
-			serveConfigChanged = true
+			_, ok := cfg.Services[vipServiceName]
+			if ok {
+				logger.Infof("Removing VIPService %q from serve config", vipServiceName)
+				delete(cfg.Services, vipServiceName)
+				serveConfigChanged = true
+			}
+			if err := r.cleanupCertResources(ctx, proxyGroupName, vipServiceName); err != nil {
+				return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+			}
 		}
 	}
 
@@ -480,16 +500,22 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
 	if err != nil {
 		return false, fmt.Errorf("error deleting VIPService: %w", err)
 	}
+
+	// 3. Clean up any cluster resources
+	if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil {
+		return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+	}
+
 	if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup
 		return svcChanged, nil
 	}
 
-	// 3. Unadvertise the VIPService in tailscaled config.
+	// 4. Unadvertise the VIPService in tailscaled config.
 	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 	}
 
-	// 4. Remove the VIPService from the serve config for the ProxyGroup.
+	// 5. Remove the VIPService from the serve config for the ProxyGroup.
 	logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
 	delete(cfg.Services, serviceName)
 	cfgBytes, err := json.Marshal(cfg)
@@ -570,13 +596,6 @@ func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
 	return isTSIngress && pgAnnot != ""
 }
 
-func isVIPServiceForAnyIngress(svc *tailscale.VIPService) bool {
-	if svc == nil {
-		return false
-	}
-	return strings.HasPrefix(svc.Comment, "tailscale.com/k8s-operator:owned-by:")
-}
-
 // validateIngress validates that the Ingress is properly configured.
 // Currently validates:
 // - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags
@@ -740,6 +759,30 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
 	return nil
 }
 
+func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) {
+	// Get all state Secrets for this ProxyGroup.
+	secrets := &corev1.SecretList{}
+	if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil {
+		return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err)
+	}
+
+	var count int
+	for _, secret := range secrets.Items {
+		prefs, ok, err := getDevicePrefs(&secret)
+		if err != nil {
+			return 0, fmt.Errorf("error getting node metadata: %w", err)
+		}
+		if !ok {
+			continue
+		}
+		if slices.Contains(prefs.AdvertiseServices, serviceName.String()) {
+			count++
+		}
+	}
+
+	return count, nil
+}
+
 // OwnerRef is an owner reference that uniquely identifies a Tailscale
 // Kubernetes operator instance.
 type OwnerRef struct {
@@ -791,6 +834,49 @@ func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (strin
 	return string(json), nil
 }
 
+// ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC
+// resources that allow proxies to manage the Secret are created.
+// Note that Tailscale VIPService name validation matches Kubernetes
+// resource name validation, so we can be certain that the VIPService name
+// (domain) is a valid Kubernetes resource name.
+// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
+// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error {
+	secret := certSecret(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
+		return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
+	}
+	role := certSecretRole(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil {
+		return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
+	}
+	rb := certSecretRoleBinding(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil {
+		return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err)
+	}
+	return nil
+}
+
+// cleanupCertResources ensures that the TLS Secret and associated RBAC
+// resources that allow proxies to read/write to the Secret are deleted.
+func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
+	domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
+	if err != nil {
+		return fmt.Errorf("error getting DNS name for VIPService %s: %w", name, err)
+	}
+	labels := certResourceLabels(pgName, domainName)
+	if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err)
+	}
+	if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err)
+	}
+	if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err)
+	}
+	return nil
+}
+
 // parseComment returns VIPService comment or nil if none found or not matching the expected format.
 func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
 	if vipSvc.Comment == "" {
@@ -811,3 +897,93 @@ func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
 func requeueInterval() time.Duration {
 	return time.Duration(rand.N(5)+5) * time.Minute
 }
+
+// certSecretRole creates a Role that will allow proxies to manage the TLS
+// Secret for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecretRole(pgName, namespace, domain string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    certResourceLabels(pgName, domain),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{""},
+				Resources:     []string{"secrets"},
+				ResourceNames: []string{domain},
+				Verbs: []string{
+					"get",
+					"list",
+					"patch",
+					"update",
+				},
+			},
+		},
+	}
+}
+
+// certSecretRoleBinding creates a RoleBinding for Role that will allow proxies
+// to manage the TLS Secret for the given domain. Domain must be a valid
+// Kubernetes resource name.
+func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    certResourceLabels(pgName, domain),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      pgName,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: domain,
+		},
+	}
+}
+
+// certSecret creates a Secret that will store the TLS certificate and private
+// key for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecret(pgName, namespace, domain string) *corev1.Secret {
+	labels := certResourceLabels(pgName, domain)
+	labels[kubetypes.LabelSecretType] = "certs"
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       nil,
+			corev1.TLSPrivateKeyKey: nil,
+		},
+		Type: corev1.SecretTypeTLS,
+	}
+}
+
+func certResourceLabels(pgName, domain string) map[string]string {
+	return map[string]string{
+		kubetypes.LabelManaged:      "true",
+		"tailscale.com/proxy-group": pgName,
+		"tailscale.com/domain":      domain,
+	}
+}
+
+// dnsNameForService returns the DNS name for the given VIPService name.
+func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
+	s := svc.WithoutPrefix()
+	tcd, err := r.tailnetCertDomain(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error determining DNS name base: %w", err)
+	}
+	return s + "." + tcd, nil
+}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -8,8 +8,10 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"maps"
+	"net/http"
 	"reflect"
 	"testing"
 
@@ -18,6 +20,7 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -68,6 +71,11 @@ func TestIngressPGReconciler(t *testing.T) {
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
+	// Verify cert resources were created for the first Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
+
 	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
 		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
 	})
@@ -122,6 +130,11 @@ func TestIngressPGReconciler(t *testing.T) {
 	verifyServeConfig(t, fc, "svc:my-other-svc", false)
 	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
 
+	// Verify cert resources were created for the second Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
+
 	// Verify first Ingress is still working
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
@@ -158,6 +171,9 @@ func TestIngressPGReconciler(t *testing.T) {
 	}
 
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
 
 	// Delete the first Ingress and verify cleanup
 	if err := fc.Delete(context.Background(), ing); err != nil {
@@ -184,6 +200,66 @@ func TestIngressPGReconciler(t *testing.T) {
 		t.Error("serve config not cleaned up")
 	}
 	verifyTailscaledConfig(t, fc, nil)
+
+	// Add verification that cert resources were cleaned up
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-svc.ts.net")
+}
+
+func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
+	ingPGR, fc, ft := setupIngressTest(t)
+
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test-pg",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+
+	// Verify initial reconciliation
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:my-svc", false)
+	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+
+	// Update the Ingress hostname and make sure the original VIPService is deleted.
+	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
+		ing.Spec.TLS[0].Hosts[0] = "updated-svc.tailnetxyz.ts.net"
+	})
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	verifyServeConfig(t, fc, "svc:updated-svc", false)
+	verifyVIPService(t, ft, "svc:updated-svc", []string{"443"})
+	verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})
+
+	_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc"))
+	if err == nil {
+		t.Fatalf("svc:my-svc not cleaned up")
+	}
+	var errResp *tailscale.ErrResponse
+	if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
+		t.Fatalf("unexpected error: %v", err)
+	}
 }
 
 func TestValidateIngress(t *testing.T) {
@@ -404,6 +480,31 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Status will be empty until the VIPService shows up in prefs.
+	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress, []networkingv1.IngressLoadBalancerIngress(nil)) {
+		t.Errorf("incorrect Ingress status: got %v, want empty",
+			ing.Status.LoadBalancer.Ingress)
+	}
+
+	// Add the VIPService to prefs to have the Ingress recognised as ready.
+	mustCreate(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg-0",
+			Namespace: "operator-ns",
+			Labels:    pgSecretLabels("test-pg", "state"),
+		},
+		Data: map[string][]byte{
+			"_current-profile": []byte("profile-foo"),
+			"profile-foo":      []byte(`{"AdvertiseServices":["svc:my-svc"],"Config":{"NodeID":"node-foo"}}`),
+		},
+	})
+
+	// Reconcile and re-fetch Ingress.
+	expectReconciled(t, ingPGR, "default", "test-ingress")
+	if err := fc.Get(context.Background(), client.ObjectKeyFromObject(ing), ing); err != nil {
+		t.Fatal(err)
+	}
+
 	wantStatus := []networkingv1.IngressPortStatus{
 		{Port: 443, Protocol: "TCP"},
 		{Port: 80, Protocol: "TCP"},

cmd/k8s-operator/metrics_resources.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 )
 
 const (
@@ -222,7 +223,7 @@ func metricsResourceName(stsName string) string {
 // proxy.
 func metricsResourceLabels(opts *metricsOpts) map[string]string {
 	lbls := map[string]string{
-		LabelManaged:             "true",
+		kubetypes.LabelManaged:   "true",
 		labelMetricsTarget:       opts.proxyStsName,
 		labelPromProxyType:       opts.proxyType,
 		labelPromProxyParentName: opts.proxyLabels[LabelParentName],

cmd/k8s-operator/operator.go

@@ -347,6 +347,7 @@ func runReconcilers(opts reconcilerOpts) {
 		For(&networkingv1.Ingress{}).
 		Named("ingress-pg-reconciler").
 		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(ingressesFromPGStateSecret(mgr.GetClient(), startlog))).
 		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
 		Complete(&HAIngressReconciler{
 			recorder:    eventRecorder,
@@ -636,8 +637,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Service.
 		svcProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "svc",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "svc",
 		}
 		svcHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
@@ -650,8 +651,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Ingress.
 		ingProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "ingress",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "ingress",
 		}
 		ingHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
@@ -718,7 +719,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 func isManagedResource(o client.Object) bool {
 	ls := o.GetLabels()
-	return ls[LabelManaged] == "true"
+	return ls[kubetypes.LabelManaged] == "true"
 }
 
 func isManagedByType(o client.Object, typ string) bool {
@@ -955,7 +956,7 @@ func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
 		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
@@ -975,15 +976,13 @@ func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
-		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		// have ingress ProxyGroups.
 		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
 			return nil
 		}
-		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
+		if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
 			return nil
 		}
 		pg, ok := o.GetLabels()[LabelParentName]
@@ -1000,7 +999,7 @@ func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
 	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
 		return nil
 	}
-	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+	if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 		return nil
 	}
 	svcName, ok := o.GetLabels()[LabelParentName]
@@ -1040,6 +1039,45 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
 	return reqs
 }
 
+func ingressesFromPGStateSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		secret, ok := o.(*corev1.Secret)
+		if !ok {
+			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
+			return nil
+		}
+		if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" {
+			return nil
+		}
+		if secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
+			return nil
+		}
+		if secret.ObjectMeta.Labels[kubetypes.LabelSecretType] != "state" {
+			return nil
+		}
+		pgName, ok := secret.ObjectMeta.Labels[LabelParentName]
+		if !ok {
+			return nil
+		}
+
+		ingList := &networkingv1.IngressList{}
+		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pgName}); err != nil {
+			logger.Infof("error listing Ingresses, skipping a reconcile for event on Secret %s: %v", secret.Name, err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, ing := range ingList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: ing.Namespace,
+					Name:      ing.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
 // user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1145,9 +1183,9 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h
 			return nil
 		}
 		podLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "proxygroup",
-			LabelParentName: eps.Labels[labelProxyGroup],
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "proxygroup",
+			LabelParentName:        eps.Labels[labelProxyGroup],
 		}
 		podList := &corev1.PodList{}
 		if err := cl.List(ctx, podList, client.InNamespace(ns),

cmd/k8s-operator/operator_test.go

@@ -1387,10 +1387,10 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 			Name:      "headless-1",
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      "ing-1",
-				LabelParentNamespace: "ns-1",
-				LabelParentType:      "ingress",
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        "ing-1",
+				LabelParentNamespace:   "ns-1",
+				LabelParentType:        "ingress",
 			},
 		},
 	}

cmd/k8s-operator/proxygroup.go

@@ -461,7 +461,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 
 		var existingCfgSecret *corev1.Secret // unmodified copy of secret
 		if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
-			logger.Debugf("secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
+			logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
 			existingCfgSecret = cfgSecret.DeepCopy()
 		} else if !apierrors.IsNotFound(err) {
 			return "", err
@@ -469,7 +469,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 
 		var authKey string
 		if existingCfgSecret == nil {
-			logger.Debugf("creating authkey for new ProxyGroup proxy")
+			logger.Debugf("Creating authkey for new ProxyGroup proxy")
 			tags := pg.Spec.Tags.Stringify()
 			if len(tags) == 0 {
 				tags = r.defaultTags
@@ -490,7 +490,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			if err != nil {
 				return "", fmt.Errorf("error marshalling tailscaled config: %w", err)
 			}
-			mak.Set(&cfgSecret.StringData, tsoperator.TailscaledConfigFileName(cap), string(cfgJSON))
+			mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
 		}
 
 		// The config sha256 sum is a value for a hash annotation used to trigger
@@ -520,12 +520,14 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 		}
 
 		if existingCfgSecret != nil {
-			logger.Debugf("patching the existing ProxyGroup config Secret %s", cfgSecret.Name)
-			if err := r.Patch(ctx, cfgSecret, client.MergeFrom(existingCfgSecret)); err != nil {
-				return "", err
+			if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) {
+				logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name)
+				if err := r.Update(ctx, cfgSecret); err != nil {
+					return "", err
+				}
 			}
 		} else {
-			logger.Debugf("creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
+			logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
 			if err := r.Create(ctx, cfgSecret); err != nil {
 				return "", err
 			}
@@ -645,7 +647,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 			return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err)
 		}
 
-		id, dnsName, ok, err := getNodeMetadata(ctx, &secret)
+		prefs, ok, err := getDevicePrefs(&secret)
 		if err != nil {
 			return nil, err
 		}
@@ -656,8 +658,8 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 		nm := nodeMetadata{
 			ordinal:     ordinal,
 			stateSecret: &secret,
-			tsID:        id,
-			dnsName:     dnsName,
+			tsID:        prefs.Config.NodeID,
+			dnsName:     prefs.Config.UserProfile.LoginName,
 		}
 		pod := &corev1.Pod{}
 		if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) {

cmd/k8s-operator/proxygroup_specs.go

@@ -178,7 +178,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				corev1.EnvVar{
 					Name:  "TS_SERVE_CONFIG",
 					Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),
-				})
+				},
+				corev1.EnvVar{
+					// Run proxies in cert share mode to
+					// ensure that only one TLS cert is
+					// issued for an HA Ingress.
+					Name:  "TS_EXPERIMENTAL_CERT_SHARE",
+					Value: "true",
+				},
+			)
 		}
 		return append(c.Env, envs...)
 	}()
@@ -225,6 +233,13 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 			OwnerReferences: pgOwnerReference(pg),
 		},
 		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"list",
+				},
+			},
 			{
 				APIGroups: []string{""},
 				Resources: []string{"secrets"},
@@ -318,9 +333,9 @@ func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
 	}
 }
 
-func pgSecretLabels(pgName, typ string) map[string]string {
+func pgSecretLabels(pgName, secretType string) map[string]string {
 	return pgLabels(pgName, map[string]string{
-		labelSecretType: typ, // "config" or "state".
+		kubetypes.LabelSecretType: secretType, // "config" or "state".
 	})
 }
 
@@ -330,7 +345,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
 		l[k] = v
 	}
 
-	l[LabelManaged] = "true"
+	l[kubetypes.LabelManaged] = "true"
 	l[LabelParentType] = "proxygroup"
 	l[LabelParentName] = pgName
 

cmd/k8s-operator/proxygroup_test.go

@@ -247,7 +247,6 @@ func TestProxyGroup(t *testing.T) {
 		// The fake client does not clean up objects whose owner has been
 		// deleted, so we can't test for the owned resources getting deleted.
 	})
-
 }
 
 func TestProxyGroupTypes(t *testing.T) {
@@ -417,6 +416,7 @@ func TestProxyGroupTypes(t *testing.T) {
 		}
 		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
 		verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
+		verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
 
 		// Verify ConfigMap volume mount
 		cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
@@ -475,8 +475,6 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
 			Name:      pgConfigSecretName(pgName, 0),
 			Namespace: tsNamespace,
 		},
-		// Write directly to Data because the fake client doesn't copy the write-only
-		// StringData field across to Data for us.
 		Data: map[string][]byte{
 			tsoperator.TailscaledConfigFileName(106): existingConfigBytes,
 		},
@@ -514,10 +512,10 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
 			Namespace:       tsNamespace,
 			ResourceVersion: "2",
 		},
-		StringData: map[string]string{
-			tsoperator.TailscaledConfigFileName(106): string(expectedConfigBytes),
+		Data: map[string][]byte{
+			tsoperator.TailscaledConfigFileName(106): expectedConfigBytes,
 		},
-	}, omitSecretData)
+	})
 }
 
 func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
@@ -620,11 +618,3 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
 		})
 	}
 }
-
-// The operator mostly writes to StringData and reads from Data, but the fake
-// client doesn't copy StringData across to Data on write. When comparing actual
-// vs expected Secrets, use this function to only check what the operator writes
-// to StringData.
-func omitSecretData(secret *corev1.Secret) {
-	secret.Data = nil
-}

cmd/k8s-operator/sts.go

@@ -44,11 +44,9 @@ const (
 	// Labels that the operator sets on StatefulSets and Pods. If you add a
 	// new label here, do also add it to tailscaleManagedLabels var to
 	// ensure that it does not get overwritten by ProxyClass configuration.
-	LabelManaged         = "tailscale.com/managed"
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
 	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
 	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
@@ -108,7 +106,7 @@ const (
 
 var (
 	// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
-	tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
+	tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
 	// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
 	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
 )

cmd/k8s-operator/sts_test.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
 
@@ -156,8 +157,8 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// Set a couple additional fields so we can test that we don't
 	// mistakenly override those.
 	labels := map[string]string{
-		LabelManaged:    "true",
-		LabelParentName: "foo",
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        "foo",
 	}
 	annots := map[string]string{
 		podAnnotationLastSetClusterIP: "1.2.3.4",
@@ -303,28 +304,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 	}{
 		{
 			name:    "no custom labels specified and none present in current labels, return current labels",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels",
-			current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels only contain tailscale managed labels, return a union of both",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels",
-			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{

cmd/k8s-operator/svc.go

@@ -84,10 +84,10 @@ func childResourceLabels(name, ns, typ string) map[string]string {
 	// proxying. Instead, we have to do our own filtering and tracking with
 	// labels.
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 }
 

cmd/k8s-operator/testutils_test.go

@@ -32,6 +32,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
@@ -563,10 +564,10 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
 	t.Helper()
 	labels := map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 	s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
 	if err != nil {

cmd/k8s-operator/tsrecorder.go

@@ -230,7 +230,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
 func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
 	logger := r.logger(tsr.Name)
 
-	id, _, ok, err := r.getNodeMetadata(ctx, tsr.Name)
+	prefs, ok, err := r.getDevicePrefs(ctx, tsr.Name)
 	if err != nil {
 		return false, err
 	}
@@ -243,6 +243,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
 		return true, nil
 	}
 
+	id := string(prefs.Config.NodeID)
 	logger.Debugf("deleting device %s from control", string(id))
 	if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
 		errResp := &tailscale.ErrResponse{}
@@ -327,34 +328,33 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string)
 	return secret, nil
 }
 
-func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+func (r *RecorderReconciler) getDevicePrefs(ctx context.Context, tsrName string) (prefs prefs, ok bool, err error) {
 	secret, err := r.getStateSecret(ctx, tsrName)
 	if err != nil || secret == nil {
-		return "", "", false, err
+		return prefs, false, err
 	}
 
-	return getNodeMetadata(ctx, secret)
+	return getDevicePrefs(secret)
 }
 
-// getNodeMetadata returns 'ok == true' iff the node ID is found. The dnsName
+// getDevicePrefs returns 'ok == true' iff the node ID is found. The dnsName
 // is expected to always be non-empty if the node ID is, but not required.
-func getNodeMetadata(ctx context.Context, secret *corev1.Secret) (id tailcfg.StableNodeID, dnsName string, ok bool, err error) {
+func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) {
 	// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
 	currentProfile, ok := secret.Data[currentProfileKey]
 	if !ok {
-		return "", "", false, nil
+		return prefs, false, nil
 	}
 	profileBytes, ok := secret.Data[string(currentProfile)]
 	if !ok {
-		return "", "", false, nil
+		return prefs, false, nil
 	}
-	var profile profile
-	if err := json.Unmarshal(profileBytes, &profile); err != nil {
-		return "", "", false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
+	if err := json.Unmarshal(profileBytes, &prefs); err != nil {
+		return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
 	}
 
-	ok = profile.Config.NodeID != ""
-	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
+	ok = prefs.Config.NodeID != ""
+	return prefs, ok, nil
 }
 
 func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
@@ -367,14 +367,14 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string)
 }
 
 func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
-	nodeID, dnsName, ok, err := getNodeMetadata(ctx, secret)
+	prefs, ok, err := getDevicePrefs(secret)
 	if !ok || err != nil {
 		return tsapi.RecorderTailnetDevice{}, false, err
 	}
 
 	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
 	// need the API. Should we instead update the profile to include addresses?
-	device, err := tsClient.Device(ctx, string(nodeID), nil)
+	device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil)
 	if err != nil {
 		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
 	}
@@ -383,20 +383,25 @@ func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret
 		Hostname:   device.Hostname,
 		TailnetIPs: device.Addresses,
 	}
-	if dnsName != "" {
+	if dnsName := prefs.Config.UserProfile.LoginName; dnsName != "" {
 		d.URL = fmt.Sprintf("https://%s", dnsName)
 	}
 
 	return d, true, nil
 }
 
-type profile struct {
+// [prefs] is a subset of the ipn.Prefs struct used for extracting information
+// from the state Secret of Tailscale devices.
+type prefs struct {
 	Config struct {
-		NodeID      string `json:"NodeID"`
+		NodeID      tailcfg.StableNodeID `json:"NodeID"`
 		UserProfile struct {
+			// LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net.
 			LoginName string `json:"LoginName"`
 		} `json:"UserProfile"`
 	} `json:"Config"`
+
+	AdvertiseServices []string `json:"AdvertiseServices"`
 }
 
 func markedForDeletion(obj metav1.Object) bool {

cmd/proxy-to-grafana/proxy-to-grafana.go

@@ -19,8 +19,25 @@
 //	header_property = username
 //	auto_sign_up = true
 //	whitelist = 127.0.0.1
-//	headers = Name:X-WEBAUTH-NAME
+//	headers = Email:X-Webauth-User, Name:X-Webauth-Name, Role:X-Webauth-Role
 //	enable_login_token = true
+//
+// You can use grants in Tailscale ACL to give users different roles in Grafana.
+// For example, to give group:eng the Editor role, add the following to your ACLs:
+//
+//	 "grants": [
+//			{
+//				"src": ["group:eng"],
+//				"dst": ["tag:grafana"],
+//				"app": {
+//					"tailscale.com/cap/proxy-to-grafana": [{
+//						"role": "editor",
+//					}],
+//				},
+//			},
+//	 ],
+//
+// If multiple roles are specified, the most permissive role is used.
 package main
 
 import (
@@ -49,6 +66,57 @@ var (
 	loginServer  = flag.String("login-server", "", "URL to alternative control server. If empty, the default Tailscale control is used.")
 )
 
+// aclCap is the Tailscale ACL capability used to configure proxy-to-grafana.
+const aclCap tailcfg.PeerCapability = "tailscale.com/cap/proxy-to-grafana"
+
+// aclGrant is an access control rule that assigns Grafana permissions
+// while provisioning a user.
+type aclGrant struct {
+	// Role is one of: "viewer", "editor", "admin".
+	Role string `json:"role"`
+}
+
+// grafanaRole defines possible Grafana roles.
+type grafanaRole int
+
+const (
+	// Roles are ordered by their permissions, with the least permissive role first.
+	// If a user has multiple roles, the most permissive role is used.
+	ViewerRole grafanaRole = iota
+	EditorRole
+	AdminRole
+)
+
+// String returns the string representation of a grafanaRole.
+// It is used as a header value in the HTTP request to Grafana.
+func (r grafanaRole) String() string {
+	switch r {
+	case ViewerRole:
+		return "Viewer"
+	case EditorRole:
+		return "Editor"
+	case AdminRole:
+		return "Admin"
+	default:
+		// A safe default.
+		return "Viewer"
+	}
+}
+
+// roleFromString converts a string to a grafanaRole.
+// It is used to parse the role from the ACL grant.
+func roleFromString(s string) (grafanaRole, error) {
+	switch strings.ToLower(s) {
+	case "viewer":
+		return ViewerRole, nil
+	case "editor":
+		return EditorRole, nil
+	case "admin":
+		return AdminRole, nil
+	}
+	return ViewerRole, fmt.Errorf("unknown role: %q", s)
+}
+
 func main() {
 	flag.Parse()
 	if *hostname == "" || strings.Contains(*hostname, ".") {
@@ -134,7 +202,15 @@ func modifyRequest(req *http.Request, localClient *local.Client) {
 		return
 	}
 
-	user, err := getTailscaleUser(req.Context(), localClient, req.RemoteAddr)
+	// Delete any existing X-Webauth-* headers to prevent possible spoofing
+	// if getting Tailnet identity fails.
+	for h := range req.Header {
+		if strings.HasPrefix(h, "X-Webauth-") {
+			req.Header.Del(h)
+		}
+	}
+
+	user, role, err := getTailscaleIdentity(req.Context(), localClient, req.RemoteAddr)
 	if err != nil {
 		log.Printf("error getting Tailscale user: %v", err)
 		return
@@ -142,19 +218,33 @@ func modifyRequest(req *http.Request, localClient *local.Client) {
 
 	req.Header.Set("X-Webauth-User", user.LoginName)
 	req.Header.Set("X-Webauth-Name", user.DisplayName)
+	req.Header.Set("X-Webauth-Role", role.String())
 }
 
-func getTailscaleUser(ctx context.Context, localClient *local.Client, ipPort string) (*tailcfg.UserProfile, error) {
+func getTailscaleIdentity(ctx context.Context, localClient *local.Client, ipPort string) (*tailcfg.UserProfile, grafanaRole, error) {
 	whois, err := localClient.WhoIs(ctx, ipPort)
 	if err != nil {
-		return nil, fmt.Errorf("failed to identify remote host: %w", err)
+		return nil, ViewerRole, fmt.Errorf("failed to identify remote host: %w", err)
 	}
 	if whois.Node.IsTagged() {
-		return nil, fmt.Errorf("tagged nodes are not users")
+		return nil, ViewerRole, fmt.Errorf("tagged nodes are not users")
 	}
 	if whois.UserProfile == nil || whois.UserProfile.LoginName == "" {
-		return nil, fmt.Errorf("failed to identify remote user")
+		return nil, ViewerRole, fmt.Errorf("failed to identify remote user")
 	}
 
-	return whois.UserProfile, nil
+	role := ViewerRole
+	grants, err := tailcfg.UnmarshalCapJSON[aclGrant](whois.CapMap, aclCap)
+	if err != nil {
+		return nil, ViewerRole, fmt.Errorf("failed to unmarshal ACL grants: %w", err)
+	}
+	for _, g := range grants {
+		r, err := roleFromString(g.Role)
+		if err != nil {
+			return nil, ViewerRole, fmt.Errorf("failed to parse role: %w", err)
+		}
+		role = max(role, r)
+	}
+
+	return whois.UserProfile, role, nil
 }

cmd/stund/depaware.txt

@@ -49,6 +49,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         google.golang.org/protobuf/types/known/timestamppb           from github.com/prometheus/client_golang/prometheus+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/envknob                                        from tailscale.com/tsweb+
+        tailscale.com/feature                                        from tailscale.com/tsweb
         tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
         tailscale.com/metrics                                        from tailscale.com/net/stunserver+
         tailscale.com/net/netaddr                                    from tailscale.com/net/tsaddr
@@ -57,8 +58,8 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/net/tsaddr                                     from tailscale.com/tsweb
         tailscale.com/syncs                                          from tailscale.com/metrics
         tailscale.com/tailcfg                                        from tailscale.com/version
-        tailscale.com/tsweb                                          from tailscale.com/cmd/stund
-        tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
+        tailscale.com/tsweb                                          from tailscale.com/cmd/stund+
+        tailscale.com/tsweb/promvarz                                 from tailscale.com/cmd/stund
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg
@@ -194,7 +195,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -204,12 +205,12 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         internal/filepathlite                                        from os+
         internal/fmtsort                                             from fmt
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
-        internal/godebug                                             from crypto/tls+
+        internal/godebug                                             from crypto/internal/fips140deps/godebug+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/stund/stund.go

@@ -15,6 +15,9 @@ import (
 
 	"tailscale.com/net/stunserver"
 	"tailscale.com/tsweb"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

cmd/tailscale/depaware.txt

@@ -333,7 +333,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         image/color                                                  from github.com/skip2/go-qrcode+
         image/png                                                    from github.com/skip2/go-qrcode
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -345,10 +345,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
         internal/godebug                                             from archive/tar+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/tailscaled/depaware.txt

@@ -286,7 +286,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/ipnlocal+
    L    tailscale.com/kube/kubeapi                                   from tailscale.com/ipn/store/kubestore+
    L    tailscale.com/kube/kubeclient                                from tailscale.com/ipn/store/kubestore
-        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob
+        tailscale.com/kube/kubetypes                                 from tailscale.com/envknob+
         tailscale.com/licenses                                       from tailscale.com/client/web
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
@@ -589,7 +589,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
         internal/abi                                                 from crypto/x509/internal/macos+
-        internal/asan                                                from syscall+
+        internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug
         internal/bytealg                                             from bytes+
         internal/byteorder                                           from crypto/cipher+
@@ -601,10 +601,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         internal/goarch                                              from crypto/internal/fips140deps/cpu+
         internal/godebug                                             from archive/tar+
         internal/godebugs                                            from internal/godebug+
-        internal/goexperiment                                        from runtime+
+        internal/goexperiment                                        from hash/maphash+
         internal/goos                                                from crypto/x509+
         internal/itoa                                                from internal/poll+
-        internal/msan                                                from syscall+
+        internal/msan                                                from internal/runtime/maps+
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+

cmd/xdpderper/xdpderper.go

@@ -18,6 +18,9 @@ import (
 	"tailscale.com/derp/xdp"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tsweb"
+
+	// Support for prometheus varz in tsweb
+	_ "tailscale.com/tsweb/promvarz"
 )
 
 var (

envknob/envknob.go

@@ -429,10 +429,16 @@ func App() string {
 // is a shared cert available.
 func IsCertShareReadOnlyMode() bool {
 	m := String("TS_CERT_SHARE_MODE")
-	return m == modeRO
+	return m == "ro"
 }
 
-const modeRO = "ro"
+// IsCertShareReadWriteMode returns true if this instance is the replica
+// responsible for issuing and renewing TLS certs in an HA setup with certs
+// shared between multiple replicas.
+func IsCertShareReadWriteMode() bool {
+	m := String("TS_CERT_SHARE_MODE")
+	return m == "rw"
+}
 
 // CrashOnUnexpected reports whether the Tailscale client should panic
 // on unexpected conditions. If TS_DEBUG_CRASH_ON_UNEXPECTED is set, that's

ipn/conf.go

@@ -145,9 +145,15 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) {
 		mp.AppConnector = *c.AppConnector
 		mp.AppConnectorSet = true
 	}
+	// Configfile should be the source of truth for whether this node
+	// advertises any services.  We need to ensure that each reload updates
+	// currently advertised services as else the transition from 'some
+	// services are advertised' to 'advertised services are empty/unset in
+	// conffile' would have no effect (especially given that an empty
+	// service slice would be omitted from the JSON config).
+	mp.AdvertiseServicesSet = true
 	if c.AdvertiseServices != nil {
 		mp.AdvertiseServices = c.AdvertiseServices
-		mp.AdvertiseServicesSet = true
 	}
 	return mp, nil
 }

ipn/ipnlocal/local_test.go

@@ -44,6 +44,7 @@ import (
 	"tailscale.com/tsd"
 	"tailscale.com/tstest"
 	"tailscale.com/types/dnstype"
+	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/logid"
@@ -60,6 +61,7 @@ import (
 	"tailscale.com/util/syspolicy/source"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
+	"tailscale.com/wgengine/filter/filtertype"
 	"tailscale.com/wgengine/wgcfg"
 )
 
@@ -4743,32 +4745,133 @@ func TestLoginNotifications(t *testing.T) {
 // TestConfigFileReload tests that the LocalBackend reloads its configuration
 // when the configuration file changes.
 func TestConfigFileReload(t *testing.T) {
-	cfg1 := `{"Hostname": "foo", "Version": "alpha0"}`
-	f := filepath.Join(t.TempDir(), "cfg")
-	must.Do(os.WriteFile(f, []byte(cfg1), 0600))
-	sys := new(tsd.System)
-	sys.InitialConfig = must.Get(conffile.Load(f))
-	lb := newTestLocalBackendWithSys(t, sys)
-	must.Do(lb.Start(ipn.Options{}))
-
-	lb.mu.Lock()
-	hn := lb.hostinfo.Hostname
-	lb.mu.Unlock()
-	if hn != "foo" {
-		t.Fatalf("got %q; want %q", hn, "foo")
+	type testCase struct {
+		name    string
+		initial *conffile.Config
+		updated *conffile.Config
+		checkFn func(*testing.T, *LocalBackend)
 	}
 
-	cfg2 := `{"Hostname": "bar", "Version": "alpha0"}`
-	must.Do(os.WriteFile(f, []byte(cfg2), 0600))
-	if !must.Get(lb.ReloadConfig()) {
-		t.Fatal("reload failed")
+	tests := []testCase{
+		{
+			name: "hostname_change",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:  "alpha0",
+					Hostname: ptr.To("initial-host"),
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:  "alpha0",
+					Hostname: ptr.To("updated-host"),
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if got := b.Prefs().Hostname(); got != "updated-host" {
+					t.Errorf("hostname = %q; want updated-host", got)
+				}
+			},
+		},
+		{
+			name: "start_advertising_services",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version: "alpha0",
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc", "svc:def"},
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if got := b.Prefs().AdvertiseServices().AsSlice(); !reflect.DeepEqual(got, []string{"svc:abc", "svc:def"}) {
+					t.Errorf("AdvertiseServices = %v; want [svc:abc, svc:def]", got)
+				}
+			},
+		},
+		{
+			name: "change_advertised_services",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc", "svc:def"},
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc", "svc:ghi"},
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if got := b.Prefs().AdvertiseServices().AsSlice(); !reflect.DeepEqual(got, []string{"svc:abc", "svc:ghi"}) {
+					t.Errorf("AdvertiseServices = %v; want [svc:abc, svc:ghi]", got)
+				}
+			},
+		},
+		{
+			name: "unset_advertised_services",
+			initial: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version:           "alpha0",
+					AdvertiseServices: []string{"svc:abc"},
+				},
+			},
+			updated: &conffile.Config{
+				Parsed: ipn.ConfigVAlpha{
+					Version: "alpha0",
+				},
+			},
+			checkFn: func(t *testing.T, b *LocalBackend) {
+				if b.Prefs().AdvertiseServices().Len() != 0 {
+					t.Errorf("got %d AdvertiseServices wants none", b.Prefs().AdvertiseServices().Len())
+				}
+			},
+		},
 	}
 
-	lb.mu.Lock()
-	hn = lb.hostinfo.Hostname
-	lb.mu.Unlock()
-	if hn != "bar" {
-		t.Fatalf("got %q; want %q", hn, "bar")
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "tailscale.conf")
+
+			// Write initial config
+			initialJSON, err := json.Marshal(tc.initial.Parsed)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(path, initialJSON, 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create backend with initial config
+			tc.initial.Path = path
+			tc.initial.Raw = initialJSON
+			sys := &tsd.System{
+				InitialConfig: tc.initial,
+			}
+			b := newTestLocalBackendWithSys(t, sys)
+
+			// Update config file
+			updatedJSON, err := json.Marshal(tc.updated.Parsed)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(path, updatedJSON, 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			// Trigger reload
+			if ok, err := b.ReloadConfig(); !ok || err != nil {
+				t.Fatalf("ReloadConfig() = %v, %v; want true, nil", ok, err)
+			}
+
+			// Check outcome
+			tc.checkFn(t, b)
+		})
 	}
 }
 
@@ -5206,3 +5309,60 @@ func TestUpdateIngressLocked(t *testing.T) {
 		})
 	}
 }
+
+// TestSrcCapPacketFilter tests that LocalBackend handles packet filters with
+// SrcCaps instead of Srcs (IPs)
+func TestSrcCapPacketFilter(t *testing.T) {
+	lb := newLocalBackendWithTestControl(t, false, func(tb testing.TB, opts controlclient.Options) controlclient.Client {
+		return newClient(tb, opts)
+	})
+	if err := lb.Start(ipn.Options{}); err != nil {
+		t.Fatalf("(*LocalBackend).Start(): %v", err)
+	}
+
+	var k key.NodePublic
+	must.Do(k.UnmarshalText([]byte("nodekey:5c8f86d5fc70d924e55f02446165a5dae8f822994ad26bcf4b08fd841f9bf261")))
+
+	controlClient := lb.cc.(*mockControl)
+	controlClient.send(nil, "", false, &netmap.NetworkMap{
+		SelfNode: (&tailcfg.Node{
+			Addresses: []netip.Prefix{netip.MustParsePrefix("1.1.1.1/32")},
+		}).View(),
+		Peers: []tailcfg.NodeView{
+			(&tailcfg.Node{
+				Addresses: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")},
+				ID:        2,
+				Key:       k,
+				CapMap:    tailcfg.NodeCapMap{"cap-X": nil}, // node 2 has cap
+			}).View(),
+			(&tailcfg.Node{
+				Addresses: []netip.Prefix{netip.MustParsePrefix("3.3.3.3/32")},
+				ID:        3,
+				Key:       k,
+				CapMap:    tailcfg.NodeCapMap{}, // node 3 does not have the cap
+			}).View(),
+		},
+		PacketFilter: []filtertype.Match{{
+			IPProto: views.SliceOf([]ipproto.Proto{ipproto.TCP}),
+			SrcCaps: []tailcfg.NodeCapability{"cap-X"}, // cap in packet filter rule
+			Dsts: []filtertype.NetPortRange{{
+				Net: netip.MustParsePrefix("1.1.1.1/32"),
+				Ports: filtertype.PortRange{
+					First: 22,
+					Last:  22,
+				},
+			}},
+		}},
+	})
+
+	f := lb.GetFilterForTest()
+	res := f.Check(netip.MustParseAddr("2.2.2.2"), netip.MustParseAddr("1.1.1.1"), 22, ipproto.TCP)
+	if res != filter.Accept {
+		t.Errorf("Check(2.2.2.2, ...) = %s, want %s", res, filter.Accept)
+	}
+
+	res = f.Check(netip.MustParseAddr("3.3.3.3"), netip.MustParseAddr("1.1.1.1"), 22, ipproto.TCP)
+	if !res.IsDrop() {
+		t.Error("IsDrop() for node without cap = false, want true")
+	}
+}

ipn/store/kubestore/store_kube.go

@@ -13,11 +13,14 @@ import (
 	"strings"
 	"time"
 
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
 )
 
@@ -32,21 +35,37 @@ const (
 	reasonTailscaleStateLoadFailed   = "TailscaleStateLoadFailed"
 	eventTypeWarning                 = "Warning"
 	eventTypeNormal                  = "Normal"
+
+	keyTLSCert = "tls.crt"
+	keyTLSKey  = "tls.key"
 )
 
 // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
 type Store struct {
-	client     kubeclient.Client
-	canPatch   bool
-	secretName string
+	client        kubeclient.Client
+	canPatch      bool
+	secretName    string // state Secret
+	certShareMode string // 'ro', 'rw', or empty
+	podName       string
 
-	// memory holds the latest tailscale state. Writes write state to a kube Secret and memory, Reads read from
-	// memory.
+	// memory holds the latest tailscale state. Writes write state to a kube
+	// Secret and memory, Reads read from memory.
 	memory mem.Store
 }
 
-// New returns a new Store that persists to the named Secret.
-func New(_ logger.Logf, secretName string) (*Store, error) {
+// New returns a new Store that persists state to Kubernets Secret(s).
+// Tailscale state is stored in a Secret named by the secretName parameter.
+// TLS certs are stored and retrieved from state Secret or separate Secrets
+// named after TLS endpoints if running in cert share mode.
+func New(logf logger.Logf, secretName string) (*Store, error) {
+	c, err := newClient()
+	if err != nil {
+		return nil, err
+	}
+	return newWithClient(logf, c, secretName)
+}
+
+func newClient() (kubeclient.Client, error) {
 	c, err := kubeclient.New("tailscale-state-store")
 	if err != nil {
 		return nil, err
@@ -55,6 +74,10 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 		// Derive the API server address from the environment variables
 		c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
 	}
+	return c, nil
+}
+
+func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*Store, error) {
 	canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
 	if err != nil {
 		return nil, err
@@ -63,11 +86,30 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 		client:     c,
 		canPatch:   canPatch,
 		secretName: secretName,
+		podName:    os.Getenv("POD_NAME"),
 	}
+	if envknob.IsCertShareReadWriteMode() {
+		s.certShareMode = "rw"
+	} else if envknob.IsCertShareReadOnlyMode() {
+		s.certShareMode = "ro"
+	}
+
 	// Load latest state from kube Secret if it already exists.
 	if err := s.loadState(); err != nil && err != ipn.ErrStateNotExist {
 		return nil, fmt.Errorf("error loading state from kube Secret: %w", err)
 	}
+	// If we are in cert share mode, pre-load existing shared certs.
+	if s.certShareMode == "rw" || s.certShareMode == "ro" {
+		sel := s.certSecretSelector()
+		if err := s.loadCerts(context.Background(), sel); err != nil {
+			// We will attempt to again retrieve the certs from Secrets when a request for an HTTPS endpoint
+			// is received.
+			log.Printf("[unexpected] error loading TLS certs: %v", err)
+		}
+	}
+	if s.certShareMode == "ro" {
+		go s.runCertReload(context.Background(), logf)
+	}
 	return s, nil
 }
 
@@ -84,27 +126,101 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
 
 // WriteState implements the StateStore interface.
 func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
-	return s.updateStateSecret(map[string][]byte{string(id): bs})
-}
-
-// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields of a Tailscale Kubernetes node's state
-// Secret.
-func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) error {
-	return s.updateStateSecret(map[string][]byte{domain + ".crt": cert, domain + ".key": key})
-}
-
-func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer func() {
 		if err == nil {
-			for id, bs := range data {
-				// The in-memory store does not distinguish between values read from state Secret on
-				// init and values written to afterwards. Values read from the state
-				// Secret will always be sanitized, so we also need to sanitize values written to store
-				// later, so that the Read logic can just lookup keys in sanitized form.
-				s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
-			}
+			s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
 		}
+	}()
+	return s.updateSecret(map[string][]byte{string(id): bs}, s.secretName)
+}
+
+// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields
+// of a Tailscale Kubernetes node's state Secret.
+func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) {
+	if s.certShareMode == "ro" {
+		log.Printf("[unexpected] TLS cert and key write in read-only mode")
+	}
+	if err := dnsname.ValidHostname(domain); err != nil {
+		return fmt.Errorf("invalid domain name %q: %w", domain, err)
+	}
+	defer func() {
+		// TODO(irbekrm): a read between these two separate writes would
+		// get a mismatched cert and key.  Allow writing both cert and
+		// key to the memory store in a single, lock-protected operation.
+		if err == nil {
+			s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
+			s.memory.WriteState(ipn.StateKey(domain+".key"), key)
+		}
+	}()
+	secretName := s.secretName
+	data := map[string][]byte{
+		domain + ".crt": cert,
+		domain + ".key": key,
+	}
+	// If we run in cert share mode, cert and key for a DNS name are written
+	// to a separate Secret.
+	if s.certShareMode == "rw" {
+		secretName = domain
+		data = map[string][]byte{
+			keyTLSCert: cert,
+			keyTLSKey:  key,
+		}
+	}
+	return s.updateSecret(data, secretName)
+}
+
+// ReadTLSCertAndKey reads a TLS cert and key from memory or from a
+// domain-specific Secret. It first checks the in-memory store, if not found in
+// memory and running cert store in read-only mode, looks up a Secret.
+func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
+	if err := dnsname.ValidHostname(domain); err != nil {
+		return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
+	}
+	certKey := domain + ".crt"
+	keyKey := domain + ".key"
+
+	cert, err = s.memory.ReadState(ipn.StateKey(certKey))
+	if err == nil {
+		key, err = s.memory.ReadState(ipn.StateKey(keyKey))
+		if err == nil {
+			return cert, key, nil
+		}
+	}
+	if s.certShareMode != "ro" {
+		return nil, nil, ipn.ErrStateNotExist
+	}
+	// If we are in cert share read only mode, it is possible that a write
+	// replica just issued the TLS cert for this DNS name and it has not
+	// been loaded to store yet, so check the Secret.
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, domain)
+	if err != nil {
+		if kubeclient.IsNotFoundErr(err) {
+			// TODO(irbekrm): we should return a more specific error
+			// that wraps ipn.ErrStateNotExist here.
+			return nil, nil, ipn.ErrStateNotExist
+		}
+		return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
+	}
+	cert = secret.Data[keyTLSCert]
+	key = secret.Data[keyTLSKey]
+	if len(cert) == 0 || len(key) == 0 {
+		return nil, nil, ipn.ErrStateNotExist
+	}
+	// TODO(irbekrm): a read between these two separate writes would
+	// get a mismatched cert and key.  Allow writing both cert and
+	// key to the memory store in a single lock-protected operation.
+	s.memory.WriteState(ipn.StateKey(certKey), cert)
+	s.memory.WriteState(ipn.StateKey(keyKey), key)
+	return cert, key, nil
+}
+
+func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer func() {
 		if err != nil {
 			if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil {
 				log.Printf("kubestore: error creating tailscaled state update Event: %v", err)
@@ -116,17 +232,17 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 		}
 		cancel()
 	}()
-	secret, err := s.client.GetSecret(ctx, s.secretName)
+	secret, err := s.client.GetSecret(ctx, secretName)
 	if err != nil {
 		// If the Secret does not exist, create it with the required data.
-		if kubeclient.IsNotFoundErr(err) {
+		if kubeclient.IsNotFoundErr(err) && s.canCreateSecret(secretName) {
 			return s.client.CreateSecret(ctx, &kubeapi.Secret{
 				TypeMeta: kubeapi.TypeMeta{
 					APIVersion: "v1",
 					Kind:       "Secret",
 				},
 				ObjectMeta: kubeapi.ObjectMeta{
-					Name: s.secretName,
+					Name: secretName,
 				},
 				Data: func(m map[string][]byte) map[string][]byte {
 					d := make(map[string][]byte, len(m))
@@ -137,9 +253,9 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 				}(data),
 			})
 		}
-		return err
+		return fmt.Errorf("error getting Secret %s: %w", secretName, err)
 	}
-	if s.canPatch {
+	if s.canPatchSecret(secretName) {
 		var m []kubeclient.JSONPatch
 		// If the user has pre-created a Secret with no data, we need to ensure the top level /data field.
 		if len(secret.Data) == 0 {
@@ -166,8 +282,8 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 				})
 			}
 		}
-		if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
-			return fmt.Errorf("error patching Secret %s: %w", s.secretName, err)
+		if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
+			return fmt.Errorf("error patching Secret %s: %w", secretName, err)
 		}
 		return nil
 	}
@@ -176,9 +292,9 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
 		mak.Set(&secret.Data, sanitizeKey(key), val)
 	}
 	if err := s.client.UpdateSecret(ctx, secret); err != nil {
-		return err
+		return fmt.Errorf("error updating Secret %s: %w", s.secretName, err)
 	}
-	return err
+	return nil
 }
 
 func (s *Store) loadState() (err error) {
@@ -202,6 +318,96 @@ func (s *Store) loadState() (err error) {
 	return nil
 }
 
+// runCertReload relists and reloads all TLS certs for endpoints shared by this
+// node from Secrets other than the state Secret to ensure that renewed certs get eventually loaded.
+// It is not critical to reload a cert immediately after
+// renewal, so a daily check is acceptable.
+// Currently (3/2025) this is only used for the shared HA Ingress certs on 'read' replicas.
+// Note that if shared certs are not found in memory on an HTTPS request, we
+// do a Secret lookup, so this mechanism does not need to ensure that newly
+// added Ingresses' certs get loaded.
+func (s *Store) runCertReload(ctx context.Context, logf logger.Logf) {
+	ticker := time.NewTicker(time.Hour * 24)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			sel := s.certSecretSelector()
+			if err := s.loadCerts(ctx, sel); err != nil {
+				logf("[unexpected] error reloading TLS certs: %v", err)
+			}
+		}
+	}
+}
+
+// loadCerts lists all Secrets matching the provided selector and loads TLS
+// certs and keys from those.
+func (s *Store) loadCerts(ctx context.Context, sel map[string]string) error {
+	ss, err := s.client.ListSecrets(ctx, sel)
+	if err != nil {
+		return fmt.Errorf("error listing TLS Secrets: %w", err)
+	}
+	for _, secret := range ss.Items {
+		if !hasTLSData(&secret) {
+			continue
+		}
+		// Only load secrets that have valid domain names (ending in .ts.net)
+		if !strings.HasSuffix(secret.Name, ".ts.net") {
+			continue
+		}
+		s.memory.WriteState(ipn.StateKey(secret.Name)+".crt", secret.Data[keyTLSCert])
+		s.memory.WriteState(ipn.StateKey(secret.Name)+".key", secret.Data[keyTLSKey])
+	}
+	return nil
+}
+
+// canCreateSecret returns true if this node should be allowed to create the given
+// Secret in its namespace.
+func (s *Store) canCreateSecret(secret string) bool {
+	// Only allow creating the state Secret (and not TLS Secrets).
+	return secret == s.secretName
+}
+
+// canPatchSecret returns true if this node should be allowed to patch the given
+// Secret.
+func (s *Store) canPatchSecret(secret string) bool {
+	// For backwards compatibility reasons, setups where the proxies are not
+	// given PATCH permissions for state Secrets are allowed. For TLS
+	// Secrets, we should always have PATCH permissions.
+	if secret == s.secretName {
+		return s.canPatch
+	}
+	return true
+}
+
+// certSecretSelector returns a label selector that can be used to list all
+// Secrets that aren't Tailscale state Secrets and contain TLS certificates for
+// HTTPS endpoints that this node serves.
+// Currently (3/2025) this only applies to the Kubernetes Operator's ingress
+// ProxyGroup.
+func (s *Store) certSecretSelector() map[string]string {
+	if s.podName == "" {
+		return map[string]string{}
+	}
+	p := strings.LastIndex(s.podName, "-")
+	if p == -1 {
+		return map[string]string{}
+	}
+	pgName := s.podName[:p]
+	return map[string]string{
+		kubetypes.LabelSecretType:   "certs",
+		kubetypes.LabelManaged:      "true",
+		"tailscale.com/proxy-group": pgName,
+	}
+}
+
+// hasTLSData returns true if the provided Secret contains non-empty TLS cert and key.
+func hasTLSData(s *kubeapi.Secret) bool {
+	return len(s.Data[keyTLSCert]) != 0 && len(s.Data[keyTLSKey]) != 0
+}
+
 // sanitizeKey converts any value that can be converted to a string into a valid Kubernetes Secret key.
 // Valid characters are alphanumeric, -, _, and .
 // https://kubernetes.io/docs/concepts/configuration/secret/#restriction-names-data.

ipn/store/kubestore/store_kube_test.go

@@ -4,33 +4,37 @@
 package kubestore
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/kube/kubeapi"
 	"tailscale.com/kube/kubeclient"
 )
 
-func TestUpdateStateSecret(t *testing.T) {
+func TestWriteState(t *testing.T) {
 	tests := []struct {
 		name       string
 		initial    map[string][]byte
-		updates    map[string][]byte
+		key        ipn.StateKey
+		value      []byte
 		wantData   map[string][]byte
 		allowPatch bool
 	}{
 		{
-			name: "basic_update",
+			name: "basic_write",
 			initial: map[string][]byte{
 				"existing": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"foo": []byte("bar"),
-			},
+			key:   "foo",
+			value: []byte("bar"),
 			wantData: map[string][]byte{
 				"existing": []byte("old"),
 				"foo":      []byte("bar"),
@@ -42,35 +46,17 @@ func TestUpdateStateSecret(t *testing.T) {
 			initial: map[string][]byte{
 				"foo": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"foo": []byte("new"),
-			},
+			key:   "foo",
+			value: []byte("new"),
 			wantData: map[string][]byte{
 				"foo": []byte("new"),
 			},
 			allowPatch: true,
 		},
 		{
-			name: "multiple_updates",
-			initial: map[string][]byte{
-				"keep": []byte("keep"),
-			},
-			updates: map[string][]byte{
-				"foo": []byte("bar"),
-				"baz": []byte("qux"),
-			},
-			wantData: map[string][]byte{
-				"keep": []byte("keep"),
-				"foo":  []byte("bar"),
-				"baz":  []byte("qux"),
-			},
-			allowPatch: true,
-		},
-		{
-			name: "create_new_secret",
-			updates: map[string][]byte{
-				"foo": []byte("bar"),
-			},
+			name:  "create_new_secret",
+			key:   "foo",
+			value: []byte("bar"),
 			wantData: map[string][]byte{
 				"foo": []byte("bar"),
 			},
@@ -81,29 +67,23 @@ func TestUpdateStateSecret(t *testing.T) {
 			initial: map[string][]byte{
 				"foo": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"foo": []byte("new"),
-			},
+			key:   "foo",
+			value: []byte("new"),
 			wantData: map[string][]byte{
 				"foo": []byte("new"),
 			},
 			allowPatch: false,
 		},
 		{
-			name: "sanitize_keys",
+			name: "sanitize_key",
 			initial: map[string][]byte{
 				"clean-key": []byte("old"),
 			},
-			updates: map[string][]byte{
-				"dirty@key": []byte("new"),
-				"also/bad":  []byte("value"),
-				"good.key":  []byte("keep"),
-			},
+			key:   "dirty@key",
+			value: []byte("new"),
 			wantData: map[string][]byte{
 				"clean-key": []byte("old"),
 				"dirty_key": []byte("new"),
-				"also_bad":  []byte("value"),
-				"good.key":  []byte("keep"),
 			},
 			allowPatch: true,
 		},
@@ -152,13 +132,13 @@ func TestUpdateStateSecret(t *testing.T) {
 			s := &Store{
 				client:     client,
 				canPatch:   tt.allowPatch,
-				secretName: "test-secret",
+				secretName: "ts-state",
 				memory:     mem.Store{},
 			}
 
-			err := s.updateStateSecret(tt.updates)
+			err := s.WriteState(tt.key, tt.value)
 			if err != nil {
-				t.Errorf("updateStateSecret() error = %v", err)
+				t.Errorf("WriteState() error = %v", err)
 				return
 			}
 
@@ -168,16 +148,576 @@ func TestUpdateStateSecret(t *testing.T) {
 			}
 
 			// Verify memory store was updated
-			for k, v := range tt.updates {
-				got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(k)))
+			got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(string(tt.key))))
+			if err != nil {
+				t.Errorf("reading from memory store: %v", err)
+			}
+			if !cmp.Equal(got, tt.value) {
+				t.Errorf("memory store key %q = %v, want %v", tt.key, got, tt.value)
+			}
+		})
+	}
+}
+
+func TestWriteTLSCertAndKey(t *testing.T) {
+	const (
+		testDomain = "my-app.tailnetxyz.ts.net"
+		testCert   = "fake-cert"
+		testKey    = "fake-key"
+	)
+
+	tests := []struct {
+		name            string
+		initial         map[string][]byte // pre-existing cert and key
+		certShareMode   string
+		allowPatch      bool   // whether client can patch the Secret
+		wantSecretName  string // name of the Secret where cert and key should be written
+		wantSecretData  map[string][]byte
+		wantMemoryStore map[ipn.StateKey][]byte
+	}{
+		{
+			name: "basic_write",
+			initial: map[string][]byte{
+				"existing": []byte("old"),
+			},
+			allowPatch:     true,
+			wantSecretName: "ts-state",
+			wantSecretData: map[string][]byte{
+				"existing":                     []byte("old"),
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:           "cert_share_mode_write",
+			certShareMode:  "rw",
+			allowPatch:     true,
+			wantSecretName: "my-app.tailnetxyz.ts.net",
+			wantSecretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name: "cert_share_mode_write_update_existing",
+			initial: map[string][]byte{
+				"tls.crt": []byte("old-cert"),
+				"tls.key": []byte("old-key"),
+			},
+			certShareMode:  "rw",
+			allowPatch:     true,
+			wantSecretName: "my-app.tailnetxyz.ts.net",
+			wantSecretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name: "update_existing",
+			initial: map[string][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte("old-cert"),
+				"my-app.tailnetxyz.ts.net.key": []byte("old-key"),
+			},
+			certShareMode:  "",
+			allowPatch:     true,
+			wantSecretName: "ts-state",
+			wantSecretData: map[string][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:           "patch_denied",
+			certShareMode:  "",
+			allowPatch:     false,
+			wantSecretName: "ts-state",
+			wantSecretData: map[string][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			// Set POD_NAME for testing selectors
+			envknob.Setenv("POD_NAME", "ingress-proxies-1")
+			defer envknob.Setenv("POD_NAME", "")
+
+			secret := tt.initial // track current state
+			client := &kubeclient.FakeClient{
+				GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
+					if secret == nil {
+						return nil, &kubeapi.Status{Code: 404}
+					}
+					return &kubeapi.Secret{Data: secret}, nil
+				},
+				CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
+					return tt.allowPatch, true, nil
+				},
+				CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
+					if s.Name != tt.wantSecretName {
+						t.Errorf("CreateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
+					}
+					secret = s.Data
+					return nil
+				},
+				UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
+					if s.Name != tt.wantSecretName {
+						t.Errorf("UpdateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
+					}
+					secret = s.Data
+					return nil
+				},
+				JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
+					if !tt.allowPatch {
+						return &kubeapi.Status{Reason: "Forbidden"}
+					}
+					if name != tt.wantSecretName {
+						t.Errorf("JSONPatchResource called with wrong name, got %q, want %q", name, tt.wantSecretName)
+					}
+					if secret == nil {
+						secret = make(map[string][]byte)
+					}
+					for _, p := range patches {
+						if p.Op == "add" && p.Path == "/data" {
+							secret = p.Value.(map[string][]byte)
+						} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
+							key := strings.TrimPrefix(p.Path, "/data/")
+							secret[key] = p.Value.([]byte)
+						}
+					}
+					return nil
+				},
+			}
+
+			s := &Store{
+				client:        client,
+				canPatch:      tt.allowPatch,
+				secretName:    tt.wantSecretName,
+				certShareMode: tt.certShareMode,
+				memory:        mem.Store{},
+			}
+
+			err := s.WriteTLSCertAndKey(testDomain, []byte(testCert), []byte(testKey))
+			if err != nil {
+				t.Errorf("WriteTLSCertAndKey() error = '%v'", err)
+				return
+			}
+
+			// Verify secret data
+			if diff := cmp.Diff(secret, tt.wantSecretData); diff != "" {
+				t.Errorf("secret data mismatch (-got +want):\n%s", diff)
+			}
+
+			// Verify memory store was updated
+			for key, want := range tt.wantMemoryStore {
+				got, err := s.memory.ReadState(key)
 				if err != nil {
 					t.Errorf("reading from memory store: %v", err)
 					continue
 				}
-				if !cmp.Equal(got, v) {
-					t.Errorf("memory store key %q = %v, want %v", k, got, v)
+				if !cmp.Equal(got, want) {
+					t.Errorf("memory store key %q = %v, want %v", key, got, want)
 				}
 			}
 		})
 	}
 }
+
+func TestReadTLSCertAndKey(t *testing.T) {
+	const (
+		testDomain = "my-app.tailnetxyz.ts.net"
+		testCert   = "fake-cert"
+		testKey    = "fake-key"
+	)
+
+	tests := []struct {
+		name          string
+		memoryStore   map[ipn.StateKey][]byte // pre-existing memory store state
+		certShareMode string
+		domain        string
+		secretData    map[string][]byte // data to return from mock GetSecret
+		secretGetErr  error             // error to return from mock GetSecret
+		wantCert      []byte
+		wantKey       []byte
+		wantErr       error
+		// what should end up in memory store after the store is created
+		wantMemoryStore map[ipn.StateKey][]byte
+	}{
+		{
+			name: "found",
+			memoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			domain:   testDomain,
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:    "not_found",
+			domain:  testDomain,
+			wantErr: ipn.ErrStateNotExist,
+		},
+		{
+			name:          "cert_share_ro_mode_found_in_secret",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:          "cert_share_ro_mode_found_in_memory",
+			certShareMode: "ro",
+			memoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+			domain:   testDomain,
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
+		},
+		{
+			name:          "cert_share_ro_mode_not_found",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretGetErr:  &kubeapi.Status{Code: 404},
+			wantErr:       ipn.ErrStateNotExist,
+		},
+		{
+			name:          "cert_share_ro_mode_empty_cert_in_secret",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretData: map[string][]byte{
+				"tls.crt": {},
+				"tls.key": []byte(testKey),
+			},
+			wantErr: ipn.ErrStateNotExist,
+		},
+		{
+			name:          "cert_share_ro_mode_kube_api_error",
+			certShareMode: "ro",
+			domain:        testDomain,
+			secretGetErr:  fmt.Errorf("api error"),
+			wantErr:       fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			client := &kubeclient.FakeClient{
+				GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
+					if tt.secretGetErr != nil {
+						return nil, tt.secretGetErr
+					}
+					return &kubeapi.Secret{Data: tt.secretData}, nil
+				},
+			}
+
+			s := &Store{
+				client:        client,
+				secretName:    "ts-state",
+				certShareMode: tt.certShareMode,
+				memory:        mem.Store{},
+			}
+
+			// Initialize memory store
+			for k, v := range tt.memoryStore {
+				s.memory.WriteState(k, v)
+			}
+
+			gotCert, gotKey, err := s.ReadTLSCertAndKey(tt.domain)
+			if tt.wantErr != nil {
+				if err == nil {
+					t.Errorf("ReadTLSCertAndKey() error = nil, want error containing %v", tt.wantErr)
+					return
+				}
+				if !strings.Contains(err.Error(), tt.wantErr.Error()) {
+					t.Errorf("ReadTLSCertAndKey() error = %v, want error containing %v", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Errorf("ReadTLSCertAndKey() unexpected error: %v", err)
+				return
+			}
+
+			if !bytes.Equal(gotCert, tt.wantCert) {
+				t.Errorf("ReadTLSCertAndKey() gotCert = %v, want %v", gotCert, tt.wantCert)
+			}
+			if !bytes.Equal(gotKey, tt.wantKey) {
+				t.Errorf("ReadTLSCertAndKey() gotKey = %v, want %v", gotKey, tt.wantKey)
+			}
+
+			// Verify memory store contents after operation
+			if tt.wantMemoryStore != nil {
+				for key, want := range tt.wantMemoryStore {
+					got, err := s.memory.ReadState(key)
+					if err != nil {
+						t.Errorf("reading from memory store: %v", err)
+						continue
+					}
+					if !bytes.Equal(got, want) {
+						t.Errorf("memory store key %q = %v, want %v", key, got, want)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestNewWithClient(t *testing.T) {
+	const (
+		secretName = "ts-state"
+		testCert   = "fake-cert"
+		testKey    = "fake-key"
+	)
+
+	certSecretsLabels := map[string]string{
+		"tailscale.com/secret-type": "certs",
+		"tailscale.com/managed":     "true",
+		"tailscale.com/proxy-group": "ingress-proxies",
+	}
+
+	// Helper function to create Secret objects for testing
+	makeSecret := func(name string, labels map[string]string, certSuffix string) kubeapi.Secret {
+		return kubeapi.Secret{
+			ObjectMeta: kubeapi.ObjectMeta{
+				Name:   name,
+				Labels: labels,
+			},
+			Data: map[string][]byte{
+				"tls.crt": []byte(testCert + certSuffix),
+				"tls.key": []byte(testKey + certSuffix),
+			},
+		}
+	}
+
+	tests := []struct {
+		name                    string
+		stateSecretContents     map[string][]byte // data in state Secret
+		TLSSecrets              []kubeapi.Secret  // list of TLS cert Secrets
+		certMode                string
+		secretGetErr            error // error to return from GetSecret
+		secretsListErr          error // error to return from ListSecrets
+		wantMemoryStoreContents map[ipn.StateKey][]byte
+		wantErr                 error
+	}{
+		{
+			name:                    "empty_state_secret",
+			stateSecretContents:     map[string][]byte{},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{},
+		},
+		{
+			name:                    "state_secret_not_found",
+			secretGetErr:            &kubeapi.Status{Code: 404},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{},
+		},
+		{
+			name:         "state_secret_get_error",
+			secretGetErr: fmt.Errorf("some error"),
+			wantErr:      fmt.Errorf("error loading state from kube Secret: some error"),
+		},
+		{
+			name: "load_existing_state",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+				"baz": []byte("qux"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo": []byte("bar"),
+				"baz": []byte("qux"),
+			},
+		},
+		{
+			name:     "load_select_certs_in_read_only_mode",
+			certMode: "ro",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			TLSSecrets: []kubeapi.Secret{
+				makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
+				makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
+				makeSecret("some-other-secret", nil, "3"),
+				makeSecret("app3.other-proxies.ts.net", map[string]string{
+					"tailscale.com/secret-type": "certs",
+					"tailscale.com/managed":     "true",
+					"tailscale.com/proxy-group": "some-other-proxygroup",
+				}, "4"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo":                        []byte("bar"),
+				"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
+				"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
+				"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
+				"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
+			},
+		},
+		{
+			name:     "load_select_certs_in_read_write_mode",
+			certMode: "rw",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			TLSSecrets: []kubeapi.Secret{
+				makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
+				makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
+				makeSecret("some-other-secret", nil, "3"),
+				makeSecret("app3.other-proxies.ts.net", map[string]string{
+					"tailscale.com/secret-type": "certs",
+					"tailscale.com/managed":     "true",
+					"tailscale.com/proxy-group": "some-other-proxygroup",
+				}, "4"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo":                        []byte("bar"),
+				"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
+				"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
+				"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
+				"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
+			},
+		},
+		{
+			name:     "list_cert_secrets_fails",
+			certMode: "ro",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			secretsListErr: fmt.Errorf("list error"),
+			// The error is logged but not returned, and state is still loaded
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo": []byte("bar"),
+			},
+		},
+		{
+			name:     "cert_secrets_not_loaded_when_not_in_share_mode",
+			certMode: "",
+			stateSecretContents: map[string][]byte{
+				"foo": []byte("bar"),
+			},
+			TLSSecrets: []kubeapi.Secret{
+				makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
+			},
+			wantMemoryStoreContents: map[ipn.StateKey][]byte{
+				"foo": []byte("bar"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			envknob.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
+
+			t.Setenv("POD_NAME", "ingress-proxies-1")
+
+			client := &kubeclient.FakeClient{
+				GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
+					if tt.secretGetErr != nil {
+						return nil, tt.secretGetErr
+					}
+					if name == secretName {
+						return &kubeapi.Secret{Data: tt.stateSecretContents}, nil
+					}
+					return nil, &kubeapi.Status{Code: 404}
+				},
+				CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
+					return true, true, nil
+				},
+				ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
+					if tt.secretsListErr != nil {
+						return nil, tt.secretsListErr
+					}
+					var matchingSecrets []kubeapi.Secret
+					for _, secret := range tt.TLSSecrets {
+						matches := true
+						for k, v := range selector {
+							if secret.Labels[k] != v {
+								matches = false
+								break
+							}
+						}
+						if matches {
+							matchingSecrets = append(matchingSecrets, secret)
+						}
+					}
+					return &kubeapi.SecretList{Items: matchingSecrets}, nil
+				},
+			}
+
+			s, err := newWithClient(t.Logf, client, secretName)
+			if tt.wantErr != nil {
+				if err == nil {
+					t.Errorf("NewWithClient() error = nil, want error containing %v", tt.wantErr)
+					return
+				}
+				if !strings.Contains(err.Error(), tt.wantErr.Error()) {
+					t.Errorf("NewWithClient() error = %v, want error containing %v", err, tt.wantErr)
+				}
+				return
+			}
+
+			if err != nil {
+				t.Errorf("NewWithClient() unexpected error: %v", err)
+				return
+			}
+
+			// Verify memory store contents
+			gotJSON, err := s.memory.ExportToJSON()
+			if err != nil {
+				t.Errorf("ExportToJSON failed: %v", err)
+				return
+			}
+			var got map[ipn.StateKey][]byte
+			if err := json.Unmarshal(gotJSON, &got); err != nil {
+				t.Errorf("failed to unmarshal memory store JSON: %v", err)
+				return
+			}
+			want := tt.wantMemoryStoreContents
+			if want == nil {
+				want = map[ipn.StateKey][]byte{}
+			}
+			if diff := cmp.Diff(got, want); diff != "" {
+				t.Errorf("memory store contents mismatch (-got +want):\n%s", diff)
+			}
+		})
+	}
+}

kube/kubeapi/api.go

@@ -153,6 +153,14 @@ type Secret struct {
 	Data map[string][]byte `json:"data,omitempty"`
 }
 
+// SecretList is a list of Secret objects.
+type SecretList struct {
+	TypeMeta   `json:",inline"`
+	ObjectMeta `json:"metadata"`
+
+	Items []Secret `json:"items,omitempty"`
+}
+
 // Event contains a subset of fields from corev1.Event.
 // https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7034
 // It is copied here to avoid having to import kube libraries.

kube/kubeclient/client.go

@@ -60,6 +60,7 @@ func readFile(n string) ([]byte, error) {
 // It expects to be run inside a cluster.
 type Client interface {
 	GetSecret(context.Context, string) (*kubeapi.Secret, error)
+	ListSecrets(context.Context, map[string]string) (*kubeapi.SecretList, error)
 	UpdateSecret(context.Context, *kubeapi.Secret) error
 	CreateSecret(context.Context, *kubeapi.Secret) error
 	// Event attempts to ensure an event with the specified options associated with the Pod in which we are
@@ -248,21 +249,35 @@ func (c *client) newRequest(ctx context.Context, method, url string, in any) (*h
 // GetSecret fetches the secret from the Kubernetes API.
 func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) {
 	s := &kubeapi.Secret{Data: make(map[string][]byte)}
-	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets), nil, s); err != nil {
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets, ""), nil, s); err != nil {
 		return nil, err
 	}
 	return s, nil
 }
 
+// ListSecrets fetches the secret from the Kubernetes API.
+func (c *client) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
+	sl := new(kubeapi.SecretList)
+	s := make([]string, 0, len(selector))
+	for key, val := range selector {
+		s = append(s, key+"="+url.QueryEscape(val))
+	}
+	ss := strings.Join(s, ",")
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL("", TypeSecrets, ss), nil, sl); err != nil {
+		return nil, err
+	}
+	return sl, nil
+}
+
 // CreateSecret creates a secret in the Kubernetes API.
 func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error {
 	s.Namespace = c.ns
-	return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets), s, nil)
+	return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets, ""), s, nil)
 }
 
 // UpdateSecret updates a secret in the Kubernetes API.
 func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error {
-	return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets), s, nil)
+	return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets, ""), s, nil)
 }
 
 // JSONPatch is a JSON patch operation.
@@ -283,14 +298,14 @@ func (c *client) JSONPatchResource(ctx context.Context, name, typ string, patche
 			return fmt.Errorf("unsupported JSON patch operation: %q", p.Op)
 		}
 	}
-	return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
+	return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ, ""), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
 }
 
 // StrategicMergePatchSecret updates a secret in the Kubernetes API using a
 // strategic merge patch.
 // If a fieldManager is provided, it will be used to track the patch.
 func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error {
-	surl := c.resourceURL(name, TypeSecrets)
+	surl := c.resourceURL(name, TypeSecrets, "")
 	if fieldManager != "" {
 		uv := url.Values{
 			"fieldManager": {fieldManager},
@@ -342,7 +357,7 @@ func (c *client) Event(ctx context.Context, typ, reason, msg string) error {
 			LastTimestamp:  now,
 			Count:          1,
 		}
-		return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents), &ev, nil)
+		return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents, ""), &ev, nil)
 	}
 	// If the Event already exists, we patch its count and last timestamp. This ensures that when users run 'kubectl
 	// describe pod...', they see the event just once (but with a message of how many times it has appeared over
@@ -472,9 +487,13 @@ func (c *client) checkPermission(ctx context.Context, verb, typ, name string) (b
 // resourceURL returns a URL that can be used to interact with the given resource type and, if name is not empty string,
 // the named resource of that type.
 // Note that this only works for core/v1 resource types.
-func (c *client) resourceURL(name, typ string) string {
+func (c *client) resourceURL(name, typ, sel string) string {
 	if name == "" {
-		return fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
+		url := fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
+		if sel != "" {
+			url += "?labelSelector=" + sel
+		}
+		return url
 	}
 	return fmt.Sprintf("%s/api/v1/namespaces/%s/%s/%s", c.url, c.ns, typ, name)
 }
@@ -487,7 +506,7 @@ func (c *client) nameForEvent(reason string) string {
 // getEvent fetches the event from the Kubernetes API.
 func (c *client) getEvent(ctx context.Context, name string) (*kubeapi.Event, error) {
 	e := &kubeapi.Event{}
-	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents), nil, e); err != nil {
+	if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents, ""), nil, e); err != nil {
 		return nil, err
 	}
 	return e, nil

kube/kubeclient/fake_client.go

@@ -18,6 +18,7 @@ type FakeClient struct {
 	CreateSecretImpl           func(context.Context, *kubeapi.Secret) error
 	UpdateSecretImpl           func(context.Context, *kubeapi.Secret) error
 	JSONPatchResourceImpl      func(context.Context, string, string, []JSONPatch) error
+	ListSecretsImpl            func(context.Context, map[string]string) (*kubeapi.SecretList, error)
 }
 
 func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) {
@@ -45,3 +46,9 @@ func (fc *FakeClient) UpdateSecret(ctx context.Context, secret *kubeapi.Secret)
 func (fc *FakeClient) CreateSecret(ctx context.Context, secret *kubeapi.Secret) error {
 	return fc.CreateSecretImpl(ctx, secret)
 }
+func (fc *FakeClient) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
+	if fc.ListSecretsImpl != nil {
+		return fc.ListSecretsImpl(ctx, selector)
+	}
+	return nil, nil
+}

kube/kubetypes/types.go

@@ -48,4 +48,7 @@ const (
 	PodIPv4Header string = "Pod-IPv4"
 
 	EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown"
+
+	LabelManaged    = "tailscale.com/managed"
+	LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs"
 )

licenses/android.md

@@ -29,7 +29,7 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/djherbis/times](https://pkg.go.dev/github.com/djherbis/times) ([MIT](https://github.com/djherbis/times/blob/v1.6.0/LICENSE))
  - [github.com/fxamacker/cbor/v2](https://pkg.go.dev/github.com/fxamacker/cbor/v2) ([MIT](https://github.com/fxamacker/cbor/blob/v2.7.0/LICENSE))
  - [github.com/gaissmai/bart](https://pkg.go.dev/github.com/gaissmai/bart) ([MIT](https://github.com/gaissmai/bart/blob/v0.18.0/LICENSE))
- - [github.com/go-json-experiment/json](https://pkg.go.dev/github.com/go-json-experiment/json) ([BSD-3-Clause](https://github.com/go-json-experiment/json/blob/6a9a0fde9288/LICENSE))
+ - [github.com/go-json-experiment/json](https://pkg.go.dev/github.com/go-json-experiment/json) ([BSD-3-Clause](https://github.com/go-json-experiment/json/blob/d3c622f1b874/LICENSE))
  - [github.com/godbus/dbus/v5](https://pkg.go.dev/github.com/godbus/dbus/v5) ([BSD-2-Clause](https://github.com/godbus/dbus/blob/76236955d466/LICENSE))
  - [github.com/golang/groupcache/lru](https://pkg.go.dev/github.com/golang/groupcache/lru) ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
  - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.1.2/LICENSE))
@@ -64,11 +64,11 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [go4.org/mem](https://pkg.go.dev/go4.org/mem) ([Apache-2.0](https://github.com/go4org/mem/blob/ae6ca9944745/LICENSE))
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [go4.org/unsafe/assume-no-moving-gc](https://pkg.go.dev/go4.org/unsafe/assume-no-moving-gc) ([BSD-3-Clause](https://github.com/go4org/unsafe-assume-no-moving-gc/blob/e7c30c78aeb2/LICENSE))
- - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.33.0:LICENSE))
+ - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.35.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
  - [golang.org/x/mobile](https://pkg.go.dev/golang.org/x/mobile) ([BSD-3-Clause](https://cs.opensource.google/go/x/mobile/+/81131f64:LICENSE))
  - [golang.org/x/mod/semver](https://pkg.go.dev/golang.org/x/mod/semver) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.23.0:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.29.0:LICENSE))

licenses/apple.md

@@ -70,7 +70,7 @@ See also the dependencies in the [Tailscale CLI][].
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.35.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.29.0:LICENSE))

licenses/tailscale.md

@@ -92,7 +92,7 @@ Some packages may only be included on certain architectures or operating systems
  - [go4.org/netipx](https://pkg.go.dev/go4.org/netipx) ([BSD-3-Clause](https://github.com/go4org/netipx/blob/fdeea329fbba/LICENSE))
  - [golang.org/x/crypto](https://pkg.go.dev/golang.org/x/crypto) ([BSD-3-Clause](https://cs.opensource.google/go/x/crypto/+/v0.35.0:LICENSE))
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/oauth2](https://pkg.go.dev/golang.org/x/oauth2) ([BSD-3-Clause](https://cs.opensource.google/go/x/oauth2/+/v0.26.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))

licenses/windows.md

@@ -62,7 +62,7 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/tailscale/go-winio](https://pkg.go.dev/github.com/tailscale/go-winio) ([MIT](https://github.com/tailscale/go-winio/blob/c4f33415bf55/LICENSE))
  - [github.com/tailscale/hujson](https://pkg.go.dev/github.com/tailscale/hujson) ([BSD-3-Clause](https://github.com/tailscale/hujson/blob/20486734a56a/LICENSE))
  - [github.com/tailscale/netlink](https://pkg.go.dev/github.com/tailscale/netlink) ([Apache-2.0](https://github.com/tailscale/netlink/blob/4d49adab4de7/LICENSE))
- - [github.com/tailscale/walk](https://pkg.go.dev/github.com/tailscale/walk) ([BSD-3-Clause](https://github.com/tailscale/walk/blob/04068c1cab63/LICENSE))
+ - [github.com/tailscale/walk](https://pkg.go.dev/github.com/tailscale/walk) ([BSD-3-Clause](https://github.com/tailscale/walk/blob/b2c15a420186/LICENSE))
  - [github.com/tailscale/win](https://pkg.go.dev/github.com/tailscale/win) ([BSD-3-Clause](https://github.com/tailscale/win/blob/5992cb43ca35/LICENSE))
  - [github.com/tailscale/xnet/webdav](https://pkg.go.dev/github.com/tailscale/xnet/webdav) ([BSD-3-Clause](https://github.com/tailscale/xnet/blob/8497ac4dab2e/LICENSE))
  - [github.com/tc-hib/winres](https://pkg.go.dev/github.com/tc-hib/winres) ([0BSD](https://github.com/tc-hib/winres/blob/v0.2.1/LICENSE))
@@ -74,7 +74,7 @@ Windows][].  See also the dependencies in the [Tailscale CLI][].
  - [golang.org/x/exp](https://pkg.go.dev/golang.org/x/exp) ([BSD-3-Clause](https://cs.opensource.google/go/x/exp/+/939b2ce7:LICENSE))
  - [golang.org/x/image/bmp](https://pkg.go.dev/golang.org/x/image/bmp) ([BSD-3-Clause](https://cs.opensource.google/go/x/image/+/v0.24.0:LICENSE))
  - [golang.org/x/mod](https://pkg.go.dev/golang.org/x/mod) ([BSD-3-Clause](https://cs.opensource.google/go/x/mod/+/v0.23.0:LICENSE))
- - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.35.0:LICENSE))
+ - [golang.org/x/net](https://pkg.go.dev/golang.org/x/net) ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.36.0:LICENSE))
  - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) ([BSD-3-Clause](https://cs.opensource.google/go/x/sync/+/v0.11.0:LICENSE))
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.30.0:LICENSE))
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.29.0:LICENSE))

net/dns/manager.go

@@ -35,6 +35,9 @@ import (
 
 var (
 	errFullQueue = errors.New("request queue full")
+	// ErrNoDNSConfig is returned by RecompileDNSConfig when the Manager
+	// has no existing DNS configuration.
+	ErrNoDNSConfig = errors.New("no DNS configuration")
 )
 
 // maxActiveQueries returns the maximal number of DNS requests that can
@@ -91,21 +94,18 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, health *health.Tracker,
 	}
 
 	// Rate limit our attempts to correct our DNS configuration.
+	// This is done on incoming queries, we don't want to spam it.
 	limiter := rate.NewLimiter(1.0/5.0, 1)
 
 	// This will recompile the DNS config, which in turn will requery the system
 	// DNS settings. The recovery func should triggered only when we are missing
 	// upstream nameservers and require them to forward a query.
 	m.resolver.SetMissingUpstreamRecovery(func() {
-		m.mu.Lock()
-		defer m.mu.Unlock()
-		if m.config == nil {
-			return
-		}
-
 		if limiter.Allow() {
-			m.logf("DNS resolution failed due to missing upstream nameservers.  Recompiling DNS configuration.")
-			m.setLocked(*m.config)
+			m.logf("resolution failed due to missing upstream nameservers.  Recompiling DNS configuration.")
+			if err := m.RecompileDNSConfig(); err != nil {
+				m.logf("config recompilation failed: %v", err)
+			}
 		}
 	})
 
@@ -117,6 +117,26 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, health *health.Tracker,
 // Resolver returns the Manager's DNS Resolver.
 func (m *Manager) Resolver() *resolver.Resolver { return m.resolver }
 
+// RecompileDNSConfig sets the DNS config to the current value, which has
+// the side effect of re-querying the OS's interface nameservers.  This should be used
+// on platforms where the interface nameservers can change.  Darwin, for example,
+// where the nameservers aren't always available when we process a major interface
+// change event, or platforms where the nameservers may change while tunnel is up.
+//
+// This should be called if it is determined that [OSConfigurator.GetBaseConfig] may
+// give a better or different result than when [Manager.Set] was last called.  The
+// logic for making that determination is up to the caller.
+//
+// It returns [ErrNoDNSConfig] if the [Manager] has no existing DNS configuration.
+func (m *Manager) RecompileDNSConfig() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.config == nil {
+		return ErrNoDNSConfig
+	}
+	return m.setLocked(*m.config)
+}
+
 func (m *Manager) Set(cfg Config) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()

net/packet/geneve.go

@@ -0,0 +1,104 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package packet
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+const (
+	// GeneveFixedHeaderLength is the length of the fixed size portion of the
+	// Geneve header, in bytes.
+	GeneveFixedHeaderLength = 8
+)
+
+const (
+	// GeneveProtocolDisco is the IEEE 802 Ethertype number used to represent
+	// the Tailscale Disco protocol in a Geneve header.
+	GeneveProtocolDisco uint16 = 0x7A11
+	// GeneveProtocolWireGuard is the IEEE 802 Ethertype number used to represent the
+	// WireGuard protocol in a Geneve header.
+	GeneveProtocolWireGuard uint16 = 0x7A12
+)
+
+// GeneveHeader represents the fixed size Geneve header from RFC8926.
+// TLVs/options are not implemented/supported.
+//
+//	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//	|Ver|  Opt Len  |O|C|    Rsvd.  |          Protocol Type        |
+//	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//	|        Virtual Network Identifier (VNI)       |    Reserved   |
+//	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+type GeneveHeader struct {
+	// Ver (2 bits): The current version number is 0. Packets received by a
+	// tunnel endpoint with an unknown version MUST be dropped. Transit devices
+	// interpreting Geneve packets with an unknown version number MUST treat
+	// them as UDP packets with an unknown payload.
+	Version uint8
+
+	// Protocol Type (16 bits): The type of protocol data unit appearing after
+	// the Geneve header. This follows the Ethertype [ETYPES] convention, with
+	// Ethernet itself being represented by the value 0x6558.
+	Protocol uint16
+
+	// Virtual Network Identifier (VNI) (24 bits): An identifier for a unique
+	// element of a virtual network. In many situations, this may represent an
+	// L2 segment; however, the control plane defines the forwarding semantics
+	// of decapsulated packets. The VNI MAY be used as part of ECMP forwarding
+	// decisions or MAY be used as a mechanism to distinguish between
+	// overlapping address spaces contained in the encapsulated packet when load
+	// balancing across CPUs.
+	VNI uint32
+
+	// O (1 bit): Control packet. This packet contains a control message.
+	// Control messages are sent between tunnel endpoints. Tunnel endpoints MUST
+	// NOT forward the payload, and transit devices MUST NOT attempt to
+	// interpret it. Since control messages are less frequent, it is RECOMMENDED
+	// that tunnel endpoints direct these packets to a high-priority control
+	// queue (for example, to direct the packet to a general purpose CPU from a
+	// forwarding Application-Specific Integrated Circuit (ASIC) or to separate
+	// out control traffic on a NIC). Transit devices MUST NOT alter forwarding
+	// behavior on the basis of this bit, such as ECMP link selection.
+	Control bool
+}
+
+// Encode encodes GeneveHeader into b. If len(b) < GeneveFixedHeaderLength an
+// io.ErrShortBuffer error is returned.
+func (h *GeneveHeader) Encode(b []byte) error {
+	if len(b) < GeneveFixedHeaderLength {
+		return io.ErrShortBuffer
+	}
+	if h.Version > 3 {
+		return errors.New("version must be <= 3")
+	}
+	b[0] = 0
+	b[1] = 0
+	b[0] |= h.Version << 6
+	if h.Control {
+		b[1] |= 0x80
+	}
+	binary.BigEndian.PutUint16(b[2:], h.Protocol)
+	if h.VNI > 1<<24-1 {
+		return errors.New("VNI must be <= 2^24-1")
+	}
+	binary.BigEndian.PutUint32(b[4:], h.VNI<<8)
+	return nil
+}
+
+// Decode decodes GeneveHeader from b. If len(b) < GeneveFixedHeaderLength an
+// io.ErrShortBuffer error is returned.
+func (h *GeneveHeader) Decode(b []byte) error {
+	if len(b) < GeneveFixedHeaderLength {
+		return io.ErrShortBuffer
+	}
+	h.Version = b[0] >> 6
+	if b[1]&0x80 != 0 {
+		h.Control = true
+	}
+	h.Protocol = binary.BigEndian.Uint16(b[2:])
+	h.VNI = binary.BigEndian.Uint32(b[4:]) >> 8
+	return nil
+}

net/packet/geneve_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package packet
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestGeneveHeader(t *testing.T) {
+	in := GeneveHeader{
+		Version:  3,
+		Protocol: GeneveProtocolDisco,
+		VNI:      1<<24 - 1,
+		Control:  true,
+	}
+	b := make([]byte, GeneveFixedHeaderLength)
+	err := in.Encode(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := GeneveHeader{}
+	err = out.Decode(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(out, in); diff != "" {
+		t.Fatalf("wrong results (-got +want)\n%s", diff)
+	}
+}

tailcfg/tailcfg.go

@@ -1942,12 +1942,8 @@ type MapResponse struct {
 	// the same HTTP response. A non-nil but empty list always means
 	// no PacketFilter (that is, to block everything).
 	//
-	// Note that this package's type, due its use of a slice and omitempty, is
-	// unable to marshal a zero-length non-nil slice. The control server needs
-	// to marshal this type using a separate type. See MapResponse docs.
-	//
 	// See PacketFilters for the newer way to send PacketFilter updates.
-	PacketFilter []FilterRule `json:",omitempty"`
+	PacketFilter []FilterRule `json:",omitzero"`
 
 	// PacketFilters encodes incremental packet filter updates to the client
 	// without having to send the entire packet filter on any changes as

tsweb/debug.go

@@ -14,7 +14,7 @@ import (
 	"os"
 	"runtime"
 
-	"tailscale.com/tsweb/promvarz"
+	"tailscale.com/feature"
 	"tailscale.com/tsweb/varz"
 	"tailscale.com/version"
 )
@@ -37,6 +37,11 @@ type DebugHandler struct {
 	title    string                           // title displayed on index page
 }
 
+// PrometheusHandler is an optional hook to enable native Prometheus
+// support in the debug handler. It is disabled by default. Import the
+// tailscale.com/tsweb/promvarz package to enable this feature.
+var PrometheusHandler feature.Hook[func(*DebugHandler)]
+
 // Debugger returns the DebugHandler registered on mux at /debug/,
 // creating it if necessary.
 func Debugger(mux *http.ServeMux) *DebugHandler {
@@ -53,7 +58,11 @@ func Debugger(mux *http.ServeMux) *DebugHandler {
 	ret.KVFunc("Uptime", func() any { return varz.Uptime() })
 	ret.KV("Version", version.Long())
 	ret.Handle("vars", "Metrics (Go)", expvar.Handler())
-	ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(promvarz.Handler))
+	if PrometheusHandler.IsSet() {
+		PrometheusHandler.Get()(ret)
+	} else {
+		ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(varz.Handler))
+	}
 
 	// pprof.Index serves everything that runtime/pprof.Lookup finds:
 	// goroutine, threadcreate, heap, allocs, block, mutex

tsweb/promvarz/promvarz.go

@@ -11,12 +11,21 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/expfmt"
+	"tailscale.com/tsweb"
 	"tailscale.com/tsweb/varz"
 )
 
-// Handler returns Prometheus metrics exported by our expvar converter
+func init() {
+	tsweb.PrometheusHandler.Set(registerVarz)
+}
+
+func registerVarz(debug *tsweb.DebugHandler) {
+	debug.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(handler))
+}
+
+// handler returns Prometheus metrics exported by our expvar converter
 // and the official Prometheus client.
-func Handler(w http.ResponseWriter, r *http.Request) {
+func handler(w http.ResponseWriter, r *http.Request) {
 	if err := gatherNativePrometheusMetrics(w); err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte(err.Error()))

tsweb/promvarz/promvarz_test.go

@@ -23,7 +23,7 @@ func TestHandler(t *testing.T) {
 	testVar1.Set(42)
 	testVar2.Set(4242)
 
-	svr := httptest.NewServer(http.HandlerFunc(Handler))
+	svr := httptest.NewServer(http.HandlerFunc(handler))
 	defer svr.Close()
 
 	want := `

util/eventbus/debughttp.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !ios
+
 package eventbus
 
 import (

util/eventbus/debughttp_ios.go

@@ -0,0 +1,18 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build ios
+
+package eventbus
+
+import "tailscale.com/tsweb"
+
+func registerHTTPDebugger(d *Debugger, td *tsweb.DebugHandler) {
+	// The event bus debugging UI uses html/template, which uses
+	// reflection for method lookups. This forces the compiler to
+	// retain a lot more code and information to make dynamic method
+	// dispatch work, which is unacceptable bloat for the iOS build.
+	//
+	// TODO: https://github.com/tailscale/tailscale/issues/15297 to
+	// bring the debug UI back to iOS somehow.
+}