wgengine/magicsock: subscribe to portmapper updates

When an event bus is plumbed in, use it to subscribe and react to port mapping
updates instead of using the client's callback mechanism. For now, the callback
remains available as a fallback when an event bus is not provided.

Updates #15160

Change-Id: I026adca44bf6187692ee87ae8ec02641c12f7774
Signed-off-by: M. J. Fromberger <fromberger@tailscale.com>

ALPINE.txt

@@ -1 +1 @@
-3.18
+3.19

Dockerfile

@@ -62,8 +62,10 @@ RUN GOARCH=$TARGETARCH go install -ldflags="\
       -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \
       -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot
 
-FROM alpine:3.18
+FROM alpine:3.19
 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables
+RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables
+RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables
 
 COPY --from=build-env /go/bin/* /usr/local/bin/
 # For compat with the previous run.sh, although ideally you should be

Dockerfile.base

@@ -1,5 +1,12 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
-FROM alpine:3.18
-RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables iputils
+FROM alpine:3.19
+RUN apk add --no-cache ca-certificates iptables iptables-legacy iproute2 ip6tables iputils
+# Alpine 3.19 replaces legacy iptables with nftables based implementation.  We
+# can't be certain that all hosts that run Tailscale containers currently
+# suppport nftables, so link back to legacy for backwards compatibility reasons.
+# TODO(irbekrm): add some way how to determine if we still run on nodes that
+# don't support nftables, so that we can eventually remove these symlinks.
+RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables
+RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables

build_docker.sh

@@ -16,7 +16,7 @@ eval "$(./build_dist.sh shellvars)"
 
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
-DEFAULT_BASE="tailscale/alpine-base:3.18"
+DEFAULT_BASE="tailscale/alpine-base:3.19"
 # Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
 # Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
 # annotations defined here will be overriden by release scripts that call this script.

clientupdate/clientupdate.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 
 	"tailscale.com/hostinfo"
+	"tailscale.com/types/lazy"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 	"tailscale.com/version"
@@ -249,9 +250,13 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 	return nil, false
 }
 
+var canAutoUpdateCache lazy.SyncValue[bool]
+
 // CanAutoUpdate reports whether auto-updating via the clientupdate package
 // is supported for the current os/distro.
-func CanAutoUpdate() bool {
+func CanAutoUpdate() bool { return canAutoUpdateCache.Get(canAutoUpdateUncached) }
+
+func canAutoUpdateUncached() bool {
 	if version.IsMacSysExt() {
 		// Macsys uses Sparkle for auto-updates, which doesn't have an update
 		// function in this package.

cmd/containerboot/certs.go

@@ -60,6 +60,9 @@ func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig)
 		if _, exists := cm.certLoops[domain]; !exists {
 			cancelCtx, cancel := context.WithCancel(ctx)
 			mak.Set(&cm.certLoops, domain, cancel)
+			// Note that most of the issuance anyway happens
+			// serially because the cert client has a shared lock
+			// that's held during any issuance.
 			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
 		}
 	}
@@ -116,7 +119,13 @@ func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
 			// issuance endpoint that explicitly only triggers
 			// issuance and stores certs in the relevant store, but
 			// does not return certs to the caller?
-			_, _, err := cm.lc.CertPair(ctx, domain)
+
+			// An issuance holds a shared lock, so we need to avoid
+			// a situation where other services cannot issue certs
+			// because a single one is holding the lock.
+			ctxT, cancel := context.WithTimeout(ctx, time.Second*300)
+			defer cancel()
+			_, _, err := cm.lc.CertPair(ctxT, domain)
 			if err != nil {
 				log.Printf("error refreshing certificate for %s: %v", domain, err)
 			}

cmd/derper/depaware.txt

@@ -155,6 +155,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
+        tailscale.com/util/eventbus                                  from tailscale.com/net/netmon
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineiter                                  from tailscale.com/hostinfo+
@@ -308,7 +309,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
-        html/template                                                from tailscale.com/cmd/derper
+        html/template                                                from tailscale.com/cmd/derper+
         internal/abi                                                 from crypto/x509/internal/macos+
         internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug

cmd/k8s-operator/depaware.txt

@@ -82,6 +82,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
+        github.com/coder/websocket                                   from tailscale.com/util/eventbus
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
      💣 github.com/davecgh/go-spew/spew                              from k8s.io/apimachinery/pkg/util/dump
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -903,7 +907,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
-        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
+        tailscale.com/tsweb                                          from tailscale.com/util/eventbus
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric+
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/bools                                    from tailscale.com/tsnet
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
@@ -932,6 +937,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics+
         tailscale.com/util/dnsname                                   from tailscale.com/appc+
+        tailscale.com/util/eventbus                                  from tailscale.com/tsd+
         tailscale.com/util/execqueue                                 from tailscale.com/appc+
         tailscale.com/util/goroutines                                from tailscale.com/ipn/ipnlocal
         tailscale.com/util/groupmember                               from tailscale.com/client/web+
@@ -1149,7 +1155,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         hash/fnv                                                     from google.golang.org/protobuf/internal/detrand
         hash/maphash                                                 from go4.org/mem
         html                                                         from html/template+
-        html/template                                                from github.com/gorilla/csrf
+        html/template                                                from github.com/gorilla/csrf+
         internal/abi                                                 from crypto/x509/internal/macos+
         internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2215,6 +2215,22 @@ spec:
                         https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
+                useLetsEncryptStagingEnvironment:
+                  description: |-
+                    Set UseLetsEncryptStagingEnvironment to true to issue TLS
+                    certificates for any HTTPS endpoints exposed to the tailnet from
+                    LetsEncrypt's staging environment.
+                    https://letsencrypt.org/docs/staging-environment/
+                    This setting only affects Tailscale Ingress resources.
+                    By default Ingress TLS certificates are issued from LetsEncrypt's
+                    production environment.
+                    Changing this setting true -> false, will result in any
+                    existing certs being re-issued from the production environment.
+                    Changing this setting false (default) -> true, when certs have already
+                    been provisioned from production environment will NOT result in certs
+                    being re-issued from the staging environment before they need to be
+                    renewed.
+                  type: boolean
             status:
               description: |-
                 Status of the ProxyClass. This is set and managed automatically.

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -103,7 +103,7 @@ spec:
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
                   description: |-
-                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                    Type of the ProxyGroup proxies. Currently the only supported type is egress.
                     Type is immutable once a ProxyGroup is created.
                   type: string
                   enum:

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2685,6 +2685,22 @@ spec:
                                             Defaults to false.
                                         type: boolean
                                 type: object
+                            useLetsEncryptStagingEnvironment:
+                                description: |-
+                                    Set UseLetsEncryptStagingEnvironment to true to issue TLS
+                                    certificates for any HTTPS endpoints exposed to the tailnet from
+                                    LetsEncrypt's staging environment.
+                                    https://letsencrypt.org/docs/staging-environment/
+                                    This setting only affects Tailscale Ingress resources.
+                                    By default Ingress TLS certificates are issued from LetsEncrypt's
+                                    production environment.
+                                    Changing this setting true -> false, will result in any
+                                    existing certs being re-issued from the production environment.
+                                    Changing this setting false (default) -> true, when certs have already
+                                    been provisioned from production environment will NOT result in certs
+                                    being re-issued from the staging environment before they need to be
+                                    renewed.
+                                type: boolean
                         type: object
                     status:
                         description: |-
@@ -2860,7 +2876,7 @@ spec:
                                 type: array
                             type:
                                 description: |-
-                                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                                    Type of the ProxyGroup proxies. Currently the only supported type is egress.
                                     Type is immutable once a ProxyGroup is created.
                                 enum:
                                     - egress

cmd/k8s-operator/ingress-for-pg.go

@@ -49,10 +49,11 @@ const (
 	// FinalizerNamePG is the finalizer used by the IngressPGReconciler
 	FinalizerNamePG = "tailscale.com/ingress-pg-finalizer"
 
-	indexIngressProxyGroup = ".metadata.annotations.ingress-proxy-group"
 	// annotationHTTPEndpoint can be used to configure the Ingress to expose an HTTP endpoint to tailnet (as
 	// well as the default HTTPS endpoint).
 	annotationHTTPEndpoint = "tailscale.com/http-endpoint"
+
+	labelDomain = "tailscale.com/domain"
 )
 
 var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
@@ -228,12 +229,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 			return false, fmt.Errorf("error getting VIPService %q: %w", hostname, err)
 		}
 	}
-	// Generate the VIPService comment for new or existing VIPService. This
-	// checks and ensures that VIPService's owner references are updated for
-	// this Ingress and errors if that is not possible (i.e. because it
-	// appears that the VIPService has been created by a non-operator
-	// actor).
-	svcComment, err := r.ownerRefsComment(existingVIPSvc)
+	// Generate the VIPService owner annotation for new or existing VIPService.
+	// This checks and ensures that VIPService's owner references are updated
+	// for this Ingress and errors if that is not possible (i.e. because it
+	// appears that the VIPService has been created by a non-operator actor).
+	updatedAnnotations, err := r.ownerAnnotations(existingVIPSvc)
 	if err != nil {
 		const instr = "To proceed, you can either manually delete the existing VIPService or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition"
 		msg := fmt.Sprintf("error ensuring ownership of VIPService %s: %v. %s", hostname, err, instr)
@@ -242,7 +242,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		return false, nil
 	}
 	// 3. Ensure that TLS Secret and RBAC exists
-	if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil {
+	if err := r.ensureCertResources(ctx, pgName, dnsName, ing); err != nil {
 		return false, fmt.Errorf("error ensuring cert resources: %w", err)
 	}
 
@@ -313,11 +313,13 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		vipPorts = append(vipPorts, "80")
 	}
 
+	const managedVIPServiceComment = "This VIPService is managed by the Tailscale Kubernetes Operator, do not modify"
 	vipSvc := &tailscale.VIPService{
-		Name:    serviceName,
-		Tags:    tags,
-		Ports:   vipPorts,
-		Comment: svcComment,
+		Name:        serviceName,
+		Tags:        tags,
+		Ports:       vipPorts,
+		Comment:     managedVIPServiceComment,
+		Annotations: updatedAnnotations,
 	}
 	if existingVIPSvc != nil {
 		vipSvc.Addrs = existingVIPSvc.Addrs
@@ -328,7 +330,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 	if existingVIPSvc == nil ||
 		!reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) ||
 		!reflect.DeepEqual(vipSvc.Ports, existingVIPSvc.Ports) ||
-		!strings.EqualFold(vipSvc.Comment, existingVIPSvc.Comment) {
+		!ownersAreSetAndEqual(vipSvc, existingVIPSvc) {
 		logger.Infof("Ensuring VIPService exists and is up to date")
 		if err := r.tsClient.CreateOrUpdateVIPService(ctx, vipSvc); err != nil {
 			return false, fmt.Errorf("error creating VIPService: %w", err)
@@ -337,7 +339,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 
 	// 5. Update tailscaled's AdvertiseServices config, which should add the VIPService
 	// IPs to the ProxyGroup Pods' AllowedIPs in the next netmap update if approved.
-	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, true, logger); err != nil {
+	mode := serviceAdvertisementHTTPS
+	if isHTTPEndpointEnabled(ing) {
+		mode = serviceAdvertisementHTTPAndHTTPS
+	}
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, mode, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config: %w", err)
 	}
 
@@ -353,11 +359,17 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 	case 0:
 		ing.Status.LoadBalancer.Ingress = nil
 	default:
-		ports := []networkingv1.IngressPortStatus{
-			{
+		var ports []networkingv1.IngressPortStatus
+		hasCerts, err := r.hasCerts(ctx, serviceName)
+		if err != nil {
+			return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err)
+		}
+		// If TLS certs have not been issued (yet), do not set port 443.
+		if hasCerts {
+			ports = append(ports, networkingv1.IngressPortStatus{
 				Protocol: "TCP",
 				Port:     443,
-			},
+			})
 		}
 		if isHTTPEndpointEnabled(ing) {
 			ports = append(ports, networkingv1.IngressPortStatus{
@@ -365,9 +377,14 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 				Port:     80,
 			})
 		}
+		// Set Ingress status hostname only if either port 443 or 80 is advertised.
+		var hostname string
+		if len(ports) != 0 {
+			hostname = dnsName
+		}
 		ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
 			{
-				Hostname: dnsName,
+				Hostname: hostname,
 				Ports:    ports,
 			},
 		}
@@ -428,7 +445,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
 			}
 
 			// Make sure the VIPService is not advertised in tailscaled or serve config.
-			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
+			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, serviceAdvertisementOff, logger); err != nil {
 				return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 			}
 			_, ok := cfg.Services[vipServiceName]
@@ -511,7 +528,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
 	}
 
 	// 4. Unadvertise the VIPService in tailscaled config.
-	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, serviceAdvertisementOff, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 	}
 
@@ -669,34 +686,34 @@ func (r *HAIngressReconciler) cleanupVIPService(ctx context.Context, name tailcf
 	if svc == nil {
 		return false, nil
 	}
-	c, err := parseComment(svc)
+	o, err := parseOwnerAnnotation(svc)
 	if err != nil {
-		return false, fmt.Errorf("error parsing VIPService comment")
+		return false, fmt.Errorf("error parsing VIPService owner annotation")
 	}
-	if c == nil || len(c.OwnerRefs) == 0 {
+	if o == nil || len(o.OwnerRefs) == 0 {
 		return false, nil
 	}
 	// Comparing with the operatorID only means that we will not be able to
 	// clean up VIPServices in cases where the operator was deleted from the
 	// cluster before deleting the Ingress. Perhaps the comparison could be
 	// 'if or.OperatorID === r.operatorID || or.ingressUID == r.ingressUID'.
-	ix := slices.IndexFunc(c.OwnerRefs, func(or OwnerRef) bool {
+	ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool {
 		return or.OperatorID == r.operatorID
 	})
 	if ix == -1 {
 		return false, nil
 	}
-	if len(c.OwnerRefs) == 1 {
+	if len(o.OwnerRefs) == 1 {
 		logger.Infof("Deleting VIPService %q", name)
 		return false, r.tsClient.DeleteVIPService(ctx, name)
 	}
-	c.OwnerRefs = slices.Delete(c.OwnerRefs, ix, ix+1)
+	o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1)
 	logger.Infof("Deleting VIPService %q", name)
-	json, err := json.Marshal(c)
+	json, err := json.Marshal(o)
 	if err != nil {
 		return false, fmt.Errorf("error marshalling updated VIPService owner reference: %w", err)
 	}
-	svc.Comment = string(json)
+	svc.Annotations[ownerAnnotation] = string(json)
 	return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc)
 }
 
@@ -708,8 +725,16 @@ func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool {
 	return ing.Annotations[annotationHTTPEndpoint] == "enabled"
 }
 
-func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, shouldBeAdvertised bool, logger *zap.SugaredLogger) (err error) {
-	logger.Debugf("Updating ProxyGroup tailscaled configs to advertise service %q: %v", serviceName, shouldBeAdvertised)
+// serviceAdvertisementMode describes the desired state of a VIPService.
+type serviceAdvertisementMode int
+
+const (
+	serviceAdvertisementOff          serviceAdvertisementMode = iota // Should not be advertised
+	serviceAdvertisementHTTPS                                        // Port 443 should be advertised
+	serviceAdvertisementHTTPAndHTTPS                                 // Both ports 80 and 443 should be advertised
+)
+
+func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) {
 
 	// Get all config Secrets for this ProxyGroup.
 	secrets := &corev1.SecretList{}
@@ -717,6 +742,21 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
 		return fmt.Errorf("failed to list config Secrets: %w", err)
 	}
 
+	// Verify that TLS cert for the VIPService has been successfully issued
+	// before attempting to advertise the service.
+	// This is so that in multi-cluster setups where some Ingresses succeed
+	// to issue certs and some do not (rate limits), clients are not pinned
+	// to a backend that is not able to serve HTTPS.
+	// The only exception is Ingresses with an HTTP endpoint enabled - if an
+	// Ingress has an HTTP endpoint enabled, it will be advertised even if the
+	// TLS cert is not yet provisioned.
+	hasCert, err := a.hasCerts(ctx, serviceName)
+	if err != nil {
+		return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err)
+	}
+	shouldBeAdvertised := (mode == serviceAdvertisementHTTPAndHTTPS) ||
+		(mode == serviceAdvertisementHTTPS && hasCert) // if we only expose port 443 and don't have certs (yet), do not advertise
+
 	for _, secret := range secrets.Items {
 		var updated bool
 		for fileName, confB := range secret.Data {
@@ -783,6 +823,15 @@ func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName
 	return count, nil
 }
 
+const ownerAnnotation = "tailscale.com/owner-references"
+
+// ownerAnnotationValue is the content of the VIPService.Annotation[ownerAnnotation] field.
+type ownerAnnotationValue struct {
+	// OwnerRefs is a list of owner references that identify all operator
+	// instances that manage this VIPService.
+	OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
+}
+
 // OwnerRef is an owner reference that uniquely identifies a Tailscale
 // Kubernetes operator instance.
 type OwnerRef struct {
@@ -790,48 +839,67 @@ type OwnerRef struct {
 	OperatorID string `json:"operatorID,omitempty"`
 }
 
-// comment is the content of the VIPService.Comment field.
-type comment struct {
-	// OwnerRefs is a list of owner references that identify all operator
-	// instances that manage this VIPService.
-	OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
-}
-
-// ownerRefsComment return VIPService Comment that includes owner reference for this
-// operator instance for the provided VIPService. If the VIPService is nil, a
-// new comment with owner ref is returned. If the VIPService is not nil, the
-// existing comment is returned with the owner reference added, if not already
-// present. If the VIPService is not nil, but does not contain a comment we
-// return an error as this likely means that the VIPService was created by
-// somthing other than a Tailscale Kubernetes operator.
-func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (string, error) {
+// ownerAnnotations returns the updated annotations required to ensure this
+// instance of the operator is included as an owner. If the VIPService is not
+// nil, but does not contain an owner we return an error as this likely means
+// that the VIPService was created by somthing other than a Tailscale
+// Kubernetes operator.
+func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) {
 	ref := OwnerRef{
 		OperatorID: r.operatorID,
 	}
 	if svc == nil {
-		c := &comment{OwnerRefs: []OwnerRef{ref}}
+		c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
 		json, err := json.Marshal(c)
 		if err != nil {
-			return "", fmt.Errorf("[unexpected] unable to marshal VIPService comment contents: %w, please report this", err)
+			return nil, fmt.Errorf("[unexpected] unable to marshal VIPService owner annotation contents: %w, please report this", err)
 		}
-		return string(json), nil
+		return map[string]string{
+			ownerAnnotation: string(json),
+		}, nil
 	}
-	c, err := parseComment(svc)
+	o, err := parseOwnerAnnotation(svc)
 	if err != nil {
-		return "", fmt.Errorf("error parsing existing VIPService comment: %w", err)
+		return nil, err
 	}
-	if c == nil || len(c.OwnerRefs) == 0 {
-		return "", fmt.Errorf("VIPService %s exists, but does not contain Comment field with owner references- not proceeding as this is likely a resource created by something other than a Tailscale Kubernetes Operator", svc.Name)
+	if o == nil || len(o.OwnerRefs) == 0 {
+		return nil, fmt.Errorf("VIPService %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
 	}
-	if slices.Contains(c.OwnerRefs, ref) { // up to date
-		return svc.Comment, nil
+	if slices.Contains(o.OwnerRefs, ref) { // up to date
+		return svc.Annotations, nil
 	}
-	c.OwnerRefs = append(c.OwnerRefs, ref)
-	json, err := json.Marshal(c)
+	o.OwnerRefs = append(o.OwnerRefs, ref)
+	json, err := json.Marshal(o)
 	if err != nil {
-		return "", fmt.Errorf("error marshalling updated owner references: %w", err)
+		return nil, fmt.Errorf("error marshalling updated owner references: %w", err)
 	}
-	return string(json), nil
+
+	newAnnots := make(map[string]string, len(svc.Annotations)+1)
+	for k, v := range svc.Annotations {
+		newAnnots[k] = v
+	}
+	newAnnots[ownerAnnotation] = string(json)
+	return newAnnots, nil
+}
+
+// parseOwnerAnnotation returns nil if no valid owner found.
+func parseOwnerAnnotation(vipSvc *tailscale.VIPService) (*ownerAnnotationValue, error) {
+	if vipSvc.Annotations == nil || vipSvc.Annotations[ownerAnnotation] == "" {
+		return nil, nil
+	}
+	o := &ownerAnnotationValue{}
+	if err := json.Unmarshal([]byte(vipSvc.Annotations[ownerAnnotation]), o); err != nil {
+		return nil, fmt.Errorf("error parsing VIPService %s annotation %q: %w", ownerAnnotation, vipSvc.Annotations[ownerAnnotation], err)
+	}
+	return o, nil
+}
+
+func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool {
+	return a != nil && b != nil &&
+		a.Annotations != nil && b.Annotations != nil &&
+		a.Annotations[ownerAnnotation] != "" &&
+		b.Annotations[ownerAnnotation] != "" &&
+		strings.EqualFold(a.Annotations[ownerAnnotation], b.Annotations[ownerAnnotation])
 }
 
 // ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC
@@ -841,8 +909,8 @@ func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (strin
 // (domain) is a valid Kubernetes resource name.
 // https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
 // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
-func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error {
-	secret := certSecret(pgName, r.tsNamespace, domain)
+func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string, ing *networkingv1.Ingress) error {
+	secret := certSecret(pgName, r.tsNamespace, domain, ing)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
 		return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
 	}
@@ -877,18 +945,6 @@ func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName s
 	return nil
 }
 
-// parseComment returns VIPService comment or nil if none found or not matching the expected format.
-func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
-	if vipSvc.Comment == "" {
-		return nil, nil
-	}
-	c := &comment{}
-	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
-		return nil, fmt.Errorf("error parsing VIPService Comment field %q: %w", vipSvc.Comment, err)
-	}
-	return c, nil
-}
-
 // requeueInterval returns a time duration between 5 and 10 minutes, which is
 // the period of time after which an HA Ingress, whose VIPService has been newly
 // created or changed, needs to be requeued. This is to protect against
@@ -949,9 +1005,14 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding
 
 // certSecret creates a Secret that will store the TLS certificate and private
 // key for the given domain. Domain must be a valid Kubernetes resource name.
-func certSecret(pgName, namespace, domain string) *corev1.Secret {
+func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret {
 	labels := certResourceLabels(pgName, domain)
 	labels[kubetypes.LabelSecretType] = "certs"
+	// Labels that let us identify the Ingress resource lets us reconcile
+	// the Ingress when the TLS Secret is updated (for example, when TLS
+	// certs have been provisioned).
+	labels[LabelParentName] = ing.Name
+	labels[LabelParentNamespace] = ing.Namespace
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -972,9 +1033,9 @@ func certSecret(pgName, namespace, domain string) *corev1.Secret {
 
 func certResourceLabels(pgName, domain string) map[string]string {
 	return map[string]string{
-		kubetypes.LabelManaged:      "true",
-		"tailscale.com/proxy-group": pgName,
-		"tailscale.com/domain":      domain,
+		kubetypes.LabelManaged: "true",
+		labelProxyGroup:        pgName,
+		labelDomain:            domain,
 	}
 }
 
@@ -987,3 +1048,28 @@ func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg
 	}
 	return s + "." + tcd, nil
 }
+
+// hasCerts checks if the TLS Secret for the given service has non-zero cert and key data.
+func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) {
+	domain, err := r.dnsNameForService(ctx, svc)
+	if err != nil {
+		return false, fmt.Errorf("failed to get DNS name for service: %w", err)
+	}
+	secret := &corev1.Secret{}
+	err = r.Get(ctx, client.ObjectKey{
+		Namespace: r.tsNamespace,
+		Name:      domain,
+	}, secret)
+
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to get TLS Secret: %w", err)
+	}
+
+	cert := secret.Data[corev1.TLSCertKey]
+	key := secret.Data[corev1.TLSPrivateKeyKey]
+
+	return len(cert) > 0 && len(key) > 0, nil
+}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 )
@@ -59,7 +60,7 @@ func TestIngressPGReconciler(t *testing.T) {
 				},
 			},
 			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
+				{Hosts: []string{"my-svc"}},
 			},
 		},
 	}
@@ -67,12 +68,14 @@ func TestIngressPGReconciler(t *testing.T) {
 
 	// Verify initial reconciliation
 	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
-	// Verify cert resources were created for the first Ingress
-	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-svc.ts.net"))
+	// Verify that Role and RoleBinding have been created for the first Ingress.
+	// Do not verify the cert Secret as that was already verified implicitly above.
 	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
 	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
 
@@ -127,11 +130,13 @@ func TestIngressPGReconciler(t *testing.T) {
 
 	// Verify second Ingress reconciliation
 	expectReconciled(t, ingPGR, "default", "my-other-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-other-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "my-other-ingress")
 	verifyServeConfig(t, fc, "svc:my-other-svc", false)
 	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
 
-	// Verify cert resources were created for the second Ingress
-	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	// Verify that Role and RoleBinding have been created for the first Ingress.
+	// Do not verify the cert Secret as that was already verified implicitly above.
 	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
 	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
 
@@ -231,7 +236,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
 				},
 			},
 			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
+				{Hosts: []string{"my-svc"}},
 			},
 		},
 	}
@@ -239,15 +244,19 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
 
 	// Verify initial reconciliation
 	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
 	// Update the Ingress hostname and make sure the original VIPService is deleted.
 	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
-		ing.Spec.TLS[0].Hosts[0] = "updated-svc.tailnetxyz.ts.net"
+		ing.Spec.TLS[0].Hosts[0] = "updated-svc"
 	})
 	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "updated-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyServeConfig(t, fc, "svc:updated-svc", false)
 	verifyVIPService(t, ft, "svc:updated-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})
@@ -468,6 +477,8 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
 
 	// Verify initial reconciliation with HTTP enabled
 	expectReconciled(t, ingPGR, "default", "test-ingress")
+	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
+	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyVIPService(t, ft, "svc:my-svc", []string{"80", "443"})
 	verifyServeConfig(t, fc, "svc:my-svc", true)
 
@@ -611,6 +622,7 @@ func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantH
 }
 
 func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) {
+	t.Helper()
 	var expected string
 	if expectedServices != nil {
 		expectedServicesJSON, err := json.Marshal(expectedServices)
@@ -745,8 +757,10 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 
 	// Simulate existing VIPService from another cluster
 	existingVIPSvc := &tailscale.VIPService{
-		Name:    "svc:my-svc",
-		Comment: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
+		Name: "svc:my-svc",
+		Annotations: map[string]string{
+			ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
+		},
 	}
 	ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{
 		"svc:my-svc": existingVIPSvc,
@@ -763,17 +777,17 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 		t.Fatal("VIPService not found")
 	}
 
-	c := &comment{}
-	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
-		t.Fatalf("parsing comment: %v", err)
+	o, err := parseOwnerAnnotation(vipSvc)
+	if err != nil {
+		t.Fatalf("parsing owner annotation: %v", err)
 	}
 
 	wantOwnerRefs := []OwnerRef{
 		{OperatorID: "operator-2"},
 		{OperatorID: "operator-1"},
 	}
-	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
-		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
+	if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
 	}
 
 	// Delete the Ingress and verify VIPService still exists with one owner ref
@@ -790,15 +804,40 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 		t.Fatal("VIPService was incorrectly deleted")
 	}
 
-	c = &comment{}
-	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
-		t.Fatalf("parsing comment after deletion: %v", err)
+	o, err = parseOwnerAnnotation(vipSvc)
+	if err != nil {
+		t.Fatalf("parsing owner annotation: %v", err)
 	}
 
 	wantOwnerRefs = []OwnerRef{
 		{OperatorID: "operator-2"},
 	}
-	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
-		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
+	if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) {
+		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
 	}
 }
+
+func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: "operator-ns",
+			Labels: map[string]string{
+				kubetypes.LabelManaged:    "true",
+				labelProxyGroup:           pgName,
+				labelDomain:               domain,
+				kubetypes.LabelSecretType: "certs",
+			},
+		},
+		Type: corev1.SecretTypeTLS,
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       []byte("fake-cert"),
+			corev1.TLSPrivateKeyKey: []byte("fake-key"),
+		},
+	}
+
+	_, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) {
+		s.Data = secret.Data
+	})
+	return err
+}

cmd/k8s-operator/ingress_test.go

@@ -6,6 +6,7 @@
 package main
 
 import (
+	"context"
 	"testing"
 
 	"go.uber.org/zap"
@@ -15,17 +16,18 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"tailscale.com/ipn"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
+	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
 )
 
 func TestTailscaleIngress(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewFakeClient(tsIngressClass)
+	fc := fake.NewFakeClient(ingressClass())
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -46,45 +48,8 @@ func TestTailscaleIngress(t *testing.T) {
 	}
 
 	// 1. Resources get created for regular Ingress
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -114,6 +79,9 @@ func TestTailscaleIngress(t *testing.T) {
 		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
 	})
 	expectReconciled(t, ingR, "default", "test")
+
+	// Get the ingress and update it with expected changes
+	ing := ingress()
 	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 	ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
 		Ingress: []networkingv1.IngressLoadBalancerIngress{
@@ -143,8 +111,7 @@ func TestTailscaleIngress(t *testing.T) {
 }
 
 func TestTailscaleIngressHostname(t *testing.T) {
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewFakeClient(tsIngressClass)
+	fc := fake.NewFakeClient(ingressClass())
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -165,45 +132,8 @@ func TestTailscaleIngressHostname(t *testing.T) {
 	}
 
 	// 1. Resources get created for regular Ingress
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -241,8 +171,10 @@ func TestTailscaleIngressHostname(t *testing.T) {
 		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
 	})
 	expectReconciled(t, ingR, "default", "test")
-	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 
+	// Get the ingress and update it with expected changes
+	ing := ingress()
+	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
 	expectEqual(t, fc, ing)
 
 	// 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint
@@ -299,10 +231,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass).
+		WithObjects(pc, ingressClass()).
 		WithStatusSubresource(pc).
 		Build()
 	ft := &fakeTSClient{}
@@ -326,45 +257,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 
 	// 1. Ingress is created with no ProxyClass specified, default proxy
 	// resources get configured.
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	})
+	mustCreate(t, fc, ingress())
+	mustCreate(t, fc, service())
 
 	expectReconciled(t, ingR, "default", "test")
 
@@ -432,54 +326,19 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				ObservedGeneration: 1,
 			}}},
 	}
-	ing := &networkingv1.Ingress{
-		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-			// The apiserver is supposed to set the UID, but the fake client
-			// doesn't. So, set it explicitly because other code later depends
-			// on it being set.
-			UID: types.UID("1234-UID"),
-			Labels: map[string]string{
-				"tailscale.com/proxy-class": "metrics",
-			},
-		},
-		Spec: networkingv1.IngressSpec{
-			IngressClassName: ptr.To("tailscale"),
-			DefaultBackend: &networkingv1.IngressBackend{
-				Service: &networkingv1.IngressServiceBackend{
-					Name: "test",
-					Port: networkingv1.ServiceBackendPort{
-						Number: 8080,
-					},
-				},
-			},
-			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"default-test"}},
-			},
-		},
-	}
-	svc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: "default",
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "1.2.3.4",
-			Ports: []corev1.ServicePort{{
-				Port: 8080,
-				Name: "http"},
-			},
-		},
-	}
 	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+
+	// Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service
+	ing := ingress()
+	ing.Labels = map[string]string{
+		LabelProxyClass: "metrics",
+	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass, ing, svc).
+		WithObjects(pc, ingressClass(), ing, service()).
 		WithStatusSubresource(pc).
 		Build()
+
 	ft := &fakeTSClient{}
 	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
 	zl, err := zap.NewDevelopment()
@@ -560,3 +419,118 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
 	// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
 }
+
+func TestIngressLetsEncryptStaging(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest()
+
+	testCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther)
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther)
+
+			fc := builder.Build()
+
+			if tt.proxyClassPerResource != "" || tt.defaultProxyClass != "" {
+				name := tt.proxyClassPerResource
+				if name == "" {
+					name = tt.defaultProxyClass
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			mustCreate(t, fc, ingressClass())
+			mustCreate(t, fc, service())
+			ing := ingress()
+			if tt.proxyClassPerResource != "" {
+				ing.Labels = map[string]string{
+					LabelProxyClass: tt.proxyClassPerResource,
+				}
+			}
+			mustCreate(t, fc, ing)
+
+			ingR := &IngressReconciler{
+				Client: fc,
+				ssr: &tailscaleSTSReconciler{
+					Client:            fc,
+					tsClient:          &fakeTSClient{},
+					tsnetServer:       &fakeTSNetServer{certDomains: []string{"test-host"}},
+					defaultTags:       []string{"tag:test"},
+					operatorNamespace: "operator-ns",
+					proxyImage:        "tailscale/tailscale:test",
+				},
+				logger:            zl.Sugar(),
+				defaultProxyClass: tt.defaultProxyClass,
+			}
+
+			expectReconciled(t, ingR, "default", "test")
+
+			_, shortName := findGenName(t, fc, "default", "test", "ingress")
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			if tt.useLEStagingEndpoint {
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			} else {
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			}
+		})
+	}
+}
+
+func ingressClass() *networkingv1.IngressClass {
+	return &networkingv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
+		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
+	}
+}
+
+func service() *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "1.2.3.4",
+			Ports: []corev1.ServicePort{{
+				Port: 8080,
+				Name: "http"},
+			},
+		},
+	}
+}
+
+func ingress() *networkingv1.Ingress {
+	return &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"default-test"}},
+			},
+		},
+	}
+}

cmd/k8s-operator/operator.go

@@ -9,7 +9,6 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"os"
 	"regexp"
@@ -40,7 +39,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -333,40 +331,6 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
 	}
-	lc, err := opts.tsServer.LocalClient()
-	if err != nil {
-		startlog.Fatalf("could not get local client: %v", err)
-	}
-	id, err := id(context.Background(), lc)
-	if err != nil {
-		startlog.Fatalf("error determining stable ID of the operator's Tailscale device: %v", err)
-	}
-	ingressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(ingressesFromIngressProxyGroup(mgr.GetClient(), opts.log))
-	err = builder.
-		ControllerManagedBy(mgr).
-		For(&networkingv1.Ingress{}).
-		Named("ingress-pg-reconciler").
-		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
-		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(ingressesFromPGStateSecret(mgr.GetClient(), startlog))).
-		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
-		Complete(&HAIngressReconciler{
-			recorder:    eventRecorder,
-			tsClient:    opts.tsClient,
-			tsnetServer: opts.tsServer,
-			defaultTags: strings.Split(opts.proxyTags, ","),
-			Client:      mgr.GetClient(),
-			logger:      opts.log.Named("ingress-pg-reconciler"),
-			lc:          lc,
-			operatorID:  id,
-			tsNamespace: opts.tailscaleNamespace,
-		})
-	if err != nil {
-		startlog.Fatalf("could not create ingress-pg-reconciler: %v", err)
-	}
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyGroup, indexPGIngresses); err != nil {
-		startlog.Fatalf("failed setting up indexer for HA Ingresses: %v", err)
-	}
-
 	connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector"))
 	// If a ProxyClassChanges, enqueue all Connectors that have
 	// .spec.proxyClass set to the name of this ProxyClass.
@@ -1039,45 +1003,6 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
 	return reqs
 }
 
-func ingressesFromPGStateSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(ctx context.Context, o client.Object) []reconcile.Request {
-		secret, ok := o.(*corev1.Secret)
-		if !ok {
-			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
-			return nil
-		}
-		if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" {
-			return nil
-		}
-		if secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
-			return nil
-		}
-		if secret.ObjectMeta.Labels[kubetypes.LabelSecretType] != "state" {
-			return nil
-		}
-		pgName, ok := secret.ObjectMeta.Labels[LabelParentName]
-		if !ok {
-			return nil
-		}
-
-		ingList := &networkingv1.IngressList{}
-		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pgName}); err != nil {
-			logger.Infof("error listing Ingresses, skipping a reconcile for event on Secret %s: %v", secret.Name, err)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, ing := range ingList.Items {
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: ing.Namespace,
-					Name:      ing.Name,
-				},
-			})
-		}
-		return reqs
-	}
-}
-
 // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
 // user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1108,36 +1033,6 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-// ingressesFromIngressProxyGroup is an event handler for ingress ProxyGroups. It returns reconcile requests for all
-// user-created Ingresses that should be exposed on this ProxyGroup.
-func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(ctx context.Context, o client.Object) []reconcile.Request {
-		pg, ok := o.(*tsapi.ProxyGroup)
-		if !ok {
-			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
-			return nil
-		}
-		if pg.Spec.Type != tsapi.ProxyGroupTypeIngress {
-			return nil
-		}
-		ingList := &networkingv1.IngressList{}
-		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pg.Name}); err != nil {
-			logger.Infof("error listing Ingresses: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, svc := range ingList.Items {
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: svc.Namespace,
-					Name:      svc.Name,
-				},
-			})
-		}
-		return reqs
-	}
-}
-
 // epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
 // should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
 func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
@@ -1258,63 +1153,7 @@ func indexEgressServices(o client.Object) []string {
 	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
 }
 
-// indexPGIngresses adds a local index to a cached Tailscale Ingresses meant to be exposed on a ProxyGroup. The index is
-// used a list filter.
-func indexPGIngresses(o client.Object) []string {
-	if !hasProxyGroupAnnotation(o) {
-		return nil
-	}
-	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
-}
-
-// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service
-// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation,
-// the associated Ingress gets reconciled.
-func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
-	return func(ctx context.Context, o client.Object) []reconcile.Request {
-		ingList := networkingv1.IngressList{}
-		if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil {
-			logger.Debugf("error listing Ingresses: %v", err)
-			return nil
-		}
-		reqs := make([]reconcile.Request, 0)
-		for _, ing := range ingList.Items {
-			if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
-				continue
-			}
-			if !hasProxyGroupAnnotation(&ing) {
-				continue
-			}
-			if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
-				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
-			}
-			for _, rule := range ing.Spec.Rules {
-				if rule.HTTP == nil {
-					continue
-				}
-				for _, path := range rule.HTTP.Paths {
-					if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() {
-						reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
-					}
-				}
-			}
-		}
-		return reqs
-	}
-}
-
 func hasProxyGroupAnnotation(obj client.Object) bool {
 	ing := obj.(*networkingv1.Ingress)
 	return ing.Annotations[AnnotationProxyGroup] != ""
 }
-
-func id(ctx context.Context, lc *local.Client) (string, error) {
-	st, err := lc.StatusWithoutPeers(ctx)
-	if err != nil {
-		return "", fmt.Errorf("error getting tailscale status: %w", err)
-	}
-	if st.Self == nil {
-		return "", fmt.Errorf("unexpected: device's status does not contain node's metadata")
-	}
-	return string(st.Self.ID), nil
-}

cmd/k8s-operator/proxygroup.go

@@ -302,7 +302,10 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	if err != nil {
 		return fmt.Errorf("error generating StatefulSet spec: %w", err)
 	}
-	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
+	cfg := &tailscaleSTSConfig{
+		proxyType: string(pg.Spec.Type),
+	}
+	ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger)
 	capver, err := r.capVerForPG(ctx, pg, logger)
 	if err != nil {
 		return fmt.Errorf("error getting device info: %w", err)

cmd/k8s-operator/proxygroup_test.go

@@ -518,6 +518,60 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
 	})
 }
 
+func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) {
+	pcLEStaging := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "le-staging",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: true,
+		},
+	}
+
+	pcLEStagingFalse := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "le-staging-false",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: false,
+		},
+	}
+
+	pcOther := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "other",
+			Generation: 1,
+		},
+		Spec: tsapi.ProxyClassSpec{},
+	}
+
+	return pcLEStaging, pcLEStagingFalse, pcOther
+}
+
+func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name string) *tsapi.ProxyClass {
+	t.Helper()
+	pc := &tsapi.ProxyClass{}
+	if err := fc.Get(context.Background(), client.ObjectKey{Name: name}, pc); err != nil {
+		t.Fatal(err)
+	}
+	pc.Status = tsapi.ProxyClassStatus{
+		Conditions: []metav1.Condition{{
+			Type:               string(tsapi.ProxyClassReady),
+			Status:             metav1.ConditionTrue,
+			Reason:             reasonProxyClassValid,
+			Message:            reasonProxyClassValid,
+			LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+			ObservedGeneration: pc.Generation,
+		}},
+	}
+	if err := fc.Status().Update(context.Background(), pc); err != nil {
+		t.Fatal(err)
+	}
+	return pc
+}
+
 func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
 	t.Helper()
 	if r.ingressProxyGroups.Len() != wantIngress {
@@ -541,6 +595,16 @@ func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue str
 	t.Errorf("%s environment variable not found", name)
 }
 
+func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) {
+	t.Helper()
+	for _, env := range sts.Spec.Template.Spec.Containers[0].Env {
+		if env.Name == name {
+			t.Errorf("environment variable %s should not be present", name)
+			return
+		}
+	}
+}
+
 func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) {
 	t.Helper()
 
@@ -618,3 +682,146 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
 		})
 	}
 }
+
+func TestProxyGroupLetsEncryptStaging(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	// Set up test cases- most are shared with non-HA Ingress.
+	type proxyGroupLETestCase struct {
+		leStagingTestCase
+		pgType tsapi.ProxyGroupType
+	}
+	pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest()
+	sharedTestCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther)
+	var tests []proxyGroupLETestCase
+	for _, tt := range sharedTestCases {
+		tests = append(tests, proxyGroupLETestCase{
+			leStagingTestCase: tt,
+			pgType:            tsapi.ProxyGroupTypeIngress,
+		})
+	}
+	tests = append(tests, proxyGroupLETestCase{
+		leStagingTestCase: leStagingTestCase{
+			name:                  "egress_pg_with_staging_proxyclass",
+			proxyClassPerResource: "le-staging",
+			useLEStagingEndpoint:  false,
+		},
+		pgType: tsapi.ProxyGroupTypeEgress,
+	})
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			// Pre-populate the fake client with ProxyClasses.
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther)
+
+			fc := builder.Build()
+
+			// If the test case needs a ProxyClass to exist, ensure it is set to Ready.
+			if tt.proxyClassPerResource != "" || tt.defaultProxyClass != "" {
+				name := tt.proxyClassPerResource
+				if name == "" {
+					name = tt.defaultProxyClass
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			// Create ProxyGroup
+			pg := &tsapi.ProxyGroup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+				},
+				Spec: tsapi.ProxyGroupSpec{
+					Type:       tt.pgType,
+					Replicas:   ptr.To[int32](1),
+					ProxyClass: tt.proxyClassPerResource,
+				},
+			}
+			mustCreate(t, fc, pg)
+
+			reconciler := &ProxyGroupReconciler{
+				tsNamespace:       tsNamespace,
+				proxyImage:        testProxyImage,
+				defaultTags:       []string{"tag:test"},
+				defaultProxyClass: tt.defaultProxyClass,
+				Client:            fc,
+				tsClient:          &fakeTSClient{},
+				l:                 zl.Sugar(),
+				clock:             cl,
+			}
+
+			expectReconciled(t, reconciler, "", pg.Name)
+
+			// Verify that the StatefulSet created for ProxyGrup has
+			// the expected setting for the staging endpoint.
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			if tt.useLEStagingEndpoint {
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			} else {
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			}
+		})
+	}
+}
+
+type leStagingTestCase struct {
+	name string
+	// ProxyClass set on ProxyGroup or Ingress resource.
+	proxyClassPerResource string
+	// Default ProxyClass.
+	defaultProxyClass    string
+	useLEStagingEndpoint bool
+}
+
+// Shared test cases for LE staging endpoint configuration for ProxyGroup and
+// non-HA Ingress.
+func testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther *tsapi.ProxyClass) []leStagingTestCase {
+	return []leStagingTestCase{
+		{
+			name:                  "with_staging_proxyclass",
+			proxyClassPerResource: "le-staging",
+			useLEStagingEndpoint:  true,
+		},
+		{
+			name:                  "with_staging_proxyclass_false",
+			proxyClassPerResource: "le-staging-false",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_other_proxyclass",
+			proxyClassPerResource: "other",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "no_proxyclass",
+			proxyClassPerResource: "",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_default_staging_proxyclass",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "le-staging",
+			useLEStagingEndpoint:  true,
+		},
+		{
+			name:                  "with_default_other_proxyclass",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "other",
+			useLEStagingEndpoint:  false,
+		},
+		{
+			name:                  "with_default_staging_proxyclass_false",
+			proxyClassPerResource: "",
+			defaultProxyClass:     "le-staging-false",
+			useLEStagingEndpoint:  false,
+		},
+	}
+}

cmd/k8s-operator/sts.go

@@ -102,6 +102,8 @@ const (
 
 	envVarTSLocalAddrPort = "TS_LOCAL_ADDR_PORT"
 	defaultLocalAddrPort  = 9002 // metrics and health check port
+
+	letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory"
 )
 
 var (
@@ -783,6 +785,17 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 			enableEndpoints(ss, metricsEnabled, debugEnabled)
 		}
 	}
+	if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) {
+		for i, c := range ss.Spec.Template.Spec.Containers {
+			if c.Name == "tailscale" {
+				ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{
+					Name:  "TS_DEBUG_ACME_DIRECTORY_URL",
+					Value: letsEncryptStagingEndpoint,
+				})
+				break
+			}
+		}
+	}
 
 	if pc.Spec.StatefulSet == nil {
 		return ss

cmd/natc/natc.go

@@ -94,18 +94,24 @@ func main() {
 		}
 		ignoreDstTable.Insert(pfx, true)
 	}
-	var v4Prefixes []netip.Prefix
+	var (
+		v4Prefixes    []netip.Prefix
+		numV4DNSAddrs int
+	)
 	for _, s := range strings.Split(*v4PfxStr, ",") {
 		p := netip.MustParsePrefix(strings.TrimSpace(s))
 		if p.Masked() != p {
 			log.Fatalf("v4 prefix %v is not a masked prefix", p)
 		}
 		v4Prefixes = append(v4Prefixes, p)
+		numIPs := 1 << (32 - p.Bits())
+		numV4DNSAddrs += numIPs
 	}
 	if len(v4Prefixes) == 0 {
 		log.Fatalf("no v4 prefixes specified")
 	}
 	dnsAddr := v4Prefixes[0].Addr()
+	numV4DNSAddrs -= 1 // Subtract the dnsAddr allocated above.
 	ts := &tsnet.Server{
 		Hostname: *hostname,
 	}
@@ -153,12 +159,13 @@ func main() {
 	}
 
 	c := &connector{
-		ts:         ts,
-		lc:         lc,
-		dnsAddr:    dnsAddr,
-		v4Ranges:   v4Prefixes,
-		v6ULA:      ula(uint16(*siteID)),
-		ignoreDsts: ignoreDstTable,
+		ts:            ts,
+		lc:            lc,
+		dnsAddr:       dnsAddr,
+		v4Ranges:      v4Prefixes,
+		numV4DNSAddrs: numV4DNSAddrs,
+		v6ULA:         ula(uint16(*siteID)),
+		ignoreDsts:    ignoreDstTable,
 	}
 	c.run(ctx)
 }
@@ -177,6 +184,11 @@ type connector struct {
 	// v4Ranges is the list of IPv4 ranges to advertise and assign addresses from.
 	// These are masked prefixes.
 	v4Ranges []netip.Prefix
+
+	// numV4DNSAddrs is the total size of the IPv4 ranges in addresses, minus the
+	// dnsAddr allocation.
+	numV4DNSAddrs int
+
 	// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
 	v6ULA netip.Prefix
 
@@ -502,6 +514,7 @@ type perPeerState struct {
 	mu           sync.Mutex
 	domainToAddr map[string][]netip.Addr
 	addrToDomain *bart.Table[string]
+	numV4Allocs  int
 }
 
 // domainForIP returns the domain name assigned to the given IP address and
@@ -547,17 +560,25 @@ func (ps *perPeerState) isIPUsedLocked(ip netip.Addr) bool {
 
 // unusedIPv4Locked returns an unused IPv4 address from the available ranges.
 func (ps *perPeerState) unusedIPv4Locked() netip.Addr {
+	// All addresses have been allocated.
+	if ps.numV4Allocs >= ps.c.numV4DNSAddrs {
+		return netip.Addr{}
+	}
+
 	// TODO: skip ranges that have been exhausted
-	for _, r := range ps.c.v4Ranges {
-		ip := randV4(r)
-		for r.Contains(ip) {
+	// TODO: implement a much more efficient algorithm for finding unused IPs,
+	// this is fairly crazy.
+	for {
+		for _, r := range ps.c.v4Ranges {
+			ip := randV4(r)
+			if !r.Contains(ip) {
+				panic("error: randV4 returned invalid address")
+			}
 			if !ps.isIPUsedLocked(ip) && ip != ps.c.dnsAddr {
 				return ip
 			}
-			ip = ip.Next()
 		}
 	}
-	return netip.Addr{}
 }
 
 // randV4 returns a random IPv4 address within the given prefix.
@@ -583,6 +604,7 @@ func (ps *perPeerState) assignAddrsLocked(domain string) []netip.Addr {
 	if !v4.IsValid() {
 		return nil
 	}
+	ps.numV4Allocs++
 	as16 := ps.c.v6ULA.Addr().As16()
 	as4 := v4.As4()
 	copy(as16[12:], as4[:])

cmd/natc/natc_test.go

@@ -0,0 +1,429 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"slices"
+	"testing"
+
+	"github.com/gaissmai/bart"
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/tailcfg"
+)
+
+func prefixEqual(a, b netip.Prefix) bool {
+	return a.Bits() == b.Bits() && a.Addr() == b.Addr()
+}
+
+func TestULA(t *testing.T) {
+	tests := []struct {
+		name     string
+		siteID   uint16
+		expected string
+	}{
+		{"zero", 0, "fd7a:115c:a1e0:a99c:0000::/80"},
+		{"one", 1, "fd7a:115c:a1e0:a99c:0001::/80"},
+		{"max", 65535, "fd7a:115c:a1e0:a99c:ffff::/80"},
+		{"random", 12345, "fd7a:115c:a1e0:a99c:3039::/80"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ula(tc.siteID)
+			expected := netip.MustParsePrefix(tc.expected)
+			if !prefixEqual(got, expected) {
+				t.Errorf("ula(%d) = %s; want %s", tc.siteID, got, expected)
+			}
+		})
+	}
+}
+
+func TestRandV4(t *testing.T) {
+	pfx := netip.MustParsePrefix("100.64.1.0/24")
+
+	for i := 0; i < 512; i++ {
+		ip := randV4(pfx)
+		if !pfx.Contains(ip) {
+			t.Errorf("randV4(%s) = %s; not contained in prefix", pfx, ip)
+		}
+	}
+}
+
+func TestDNSResponse(t *testing.T) {
+	tests := []struct {
+		name        string
+		questions   []dnsmessage.Question
+		addrs       []netip.Addr
+		wantEmpty   bool
+		wantAnswers []struct {
+			name  string
+			qType dnsmessage.Type
+			addr  netip.Addr
+		}
+	}{
+		{
+			name:        "empty_request",
+			questions:   []dnsmessage.Question{},
+			addrs:       []netip.Addr{},
+			wantEmpty:   false,
+			wantAnswers: nil,
+		},
+		{
+			name: "a_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs: []netip.Addr{netip.MustParseAddr("100.64.1.5")},
+			wantAnswers: []struct {
+				name  string
+				qType dnsmessage.Type
+				addr  netip.Addr
+			}{
+				{
+					name:  "example.com.",
+					qType: dnsmessage.TypeA,
+					addr:  netip.MustParseAddr("100.64.1.5"),
+				},
+			},
+		},
+		{
+			name: "aaaa_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeAAAA,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs: []netip.Addr{netip.MustParseAddr("fd7a:115c:a1e0:a99c:0001:0505:0505:0505")},
+			wantAnswers: []struct {
+				name  string
+				qType dnsmessage.Type
+				addr  netip.Addr
+			}{
+				{
+					name:  "example.com.",
+					qType: dnsmessage.TypeAAAA,
+					addr:  netip.MustParseAddr("fd7a:115c:a1e0:a99c:0001:0505:0505:0505"),
+				},
+			},
+		},
+		{
+			name: "soa_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeSOA,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs:       []netip.Addr{},
+			wantAnswers: nil,
+		},
+		{
+			name: "ns_record",
+			questions: []dnsmessage.Question{
+				{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeNS,
+					Class: dnsmessage.ClassINET,
+				},
+			},
+			addrs:       []netip.Addr{},
+			wantAnswers: nil,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			req := &dnsmessage.Message{
+				Header: dnsmessage.Header{
+					ID: 1234,
+				},
+				Questions: tc.questions,
+			}
+
+			resp, err := dnsResponse(req, tc.addrs)
+			if err != nil {
+				t.Fatalf("dnsResponse() error = %v", err)
+			}
+
+			if tc.wantEmpty && len(resp) != 0 {
+				t.Errorf("dnsResponse() returned non-empty response when expected empty")
+			}
+
+			if !tc.wantEmpty && len(resp) == 0 {
+				t.Errorf("dnsResponse() returned empty response when expected non-empty")
+			}
+
+			if len(resp) > 0 {
+				var msg dnsmessage.Message
+				err = msg.Unpack(resp)
+				if err != nil {
+					t.Fatalf("Failed to unpack response: %v", err)
+				}
+
+				if !msg.Header.Response {
+					t.Errorf("Response header is not set")
+				}
+
+				if msg.Header.ID != req.Header.ID {
+					t.Errorf("Response ID = %d, want %d", msg.Header.ID, req.Header.ID)
+				}
+
+				if len(tc.wantAnswers) > 0 {
+					if len(msg.Answers) != len(tc.wantAnswers) {
+						t.Errorf("got %d answers, want %d", len(msg.Answers), len(tc.wantAnswers))
+					} else {
+						for i, want := range tc.wantAnswers {
+							ans := msg.Answers[i]
+
+							gotName := ans.Header.Name.String()
+							if gotName != want.name {
+								t.Errorf("answer[%d] name = %s, want %s", i, gotName, want.name)
+							}
+
+							if ans.Header.Type != want.qType {
+								t.Errorf("answer[%d] type = %v, want %v", i, ans.Header.Type, want.qType)
+							}
+
+							var gotIP netip.Addr
+							switch want.qType {
+							case dnsmessage.TypeA:
+								if ans.Body.(*dnsmessage.AResource) == nil {
+									t.Errorf("answer[%d] not an A record", i)
+									continue
+								}
+								resource := ans.Body.(*dnsmessage.AResource)
+								gotIP = netip.AddrFrom4([4]byte(resource.A))
+							case dnsmessage.TypeAAAA:
+								if ans.Body.(*dnsmessage.AAAAResource) == nil {
+									t.Errorf("answer[%d] not an AAAA record", i)
+									continue
+								}
+								resource := ans.Body.(*dnsmessage.AAAAResource)
+								gotIP = netip.AddrFrom16([16]byte(resource.AAAA))
+							}
+
+							if gotIP != want.addr {
+								t.Errorf("answer[%d] IP = %s, want %s", i, gotIP, want.addr)
+							}
+						}
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestPerPeerState(t *testing.T) {
+	c := &connector{
+		v4Ranges:      []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
+		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
+		numV4DNSAddrs: (1<<(32-24) - 1),
+	}
+
+	ps := &perPeerState{c: c}
+
+	addrs, err := ps.ipForDomain("example.com")
+	if err != nil {
+		t.Fatalf("ipForDomain() error = %v", err)
+	}
+
+	if len(addrs) != 2 {
+		t.Fatalf("ipForDomain() returned %d addresses, want 2", len(addrs))
+	}
+
+	v4 := addrs[0]
+	v6 := addrs[1]
+
+	if !v4.Is4() {
+		t.Errorf("First address is not IPv4: %s", v4)
+	}
+
+	if !v6.Is6() {
+		t.Errorf("Second address is not IPv6: %s", v6)
+	}
+
+	if !c.v4Ranges[0].Contains(v4) {
+		t.Errorf("IPv4 address %s not in range %s", v4, c.v4Ranges[0])
+	}
+
+	domain, ok := ps.domainForIP(v4)
+	if !ok {
+		t.Errorf("domainForIP(%s) not found", v4)
+	} else if domain != "example.com" {
+		t.Errorf("domainForIP(%s) = %s, want %s", v4, domain, "example.com")
+	}
+
+	domain, ok = ps.domainForIP(v6)
+	if !ok {
+		t.Errorf("domainForIP(%s) not found", v6)
+	} else if domain != "example.com" {
+		t.Errorf("domainForIP(%s) = %s, want %s", v6, domain, "example.com")
+	}
+
+	addrs2, err := ps.ipForDomain("example.com")
+	if err != nil {
+		t.Fatalf("ipForDomain() second call error = %v", err)
+	}
+
+	if !slices.Equal(addrs, addrs2) {
+		t.Errorf("ipForDomain() second call = %v, want %v", addrs2, addrs)
+	}
+}
+
+func TestIgnoreDestination(t *testing.T) {
+	ignoreDstTable := &bart.Table[bool]{}
+	ignoreDstTable.Insert(netip.MustParsePrefix("192.168.1.0/24"), true)
+	ignoreDstTable.Insert(netip.MustParsePrefix("10.0.0.0/8"), true)
+
+	c := &connector{
+		ignoreDsts: ignoreDstTable,
+	}
+
+	tests := []struct {
+		name     string
+		addrs    []netip.Addr
+		expected bool
+	}{
+		{
+			name:     "no_match",
+			addrs:    []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("1.1.1.1")},
+			expected: false,
+		},
+		{
+			name:     "one_match",
+			addrs:    []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("192.168.1.5")},
+			expected: true,
+		},
+		{
+			name:     "all_match",
+			addrs:    []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("192.168.1.5")},
+			expected: true,
+		},
+		{
+			name:     "empty_addrs",
+			addrs:    []netip.Addr{},
+			expected: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := c.ignoreDestination(tc.addrs)
+			if got != tc.expected {
+				t.Errorf("ignoreDestination(%v) = %v, want %v", tc.addrs, got, tc.expected)
+			}
+		})
+	}
+}
+
+func TestConnectorGenerateDNSResponse(t *testing.T) {
+	c := &connector{
+		v4Ranges:      []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
+		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
+		numV4DNSAddrs: (1<<(32-24) - 1),
+	}
+
+	req := &dnsmessage.Message{
+		Header: dnsmessage.Header{ID: 1234},
+		Questions: []dnsmessage.Question{
+			{
+				Name:  dnsmessage.MustNewName("example.com."),
+				Type:  dnsmessage.TypeA,
+				Class: dnsmessage.ClassINET,
+			},
+		},
+	}
+
+	nodeID := tailcfg.NodeID(12345)
+
+	resp1, err := c.generateDNSResponse(req, nodeID)
+	if err != nil {
+		t.Fatalf("generateDNSResponse() error = %v", err)
+	}
+	if len(resp1) == 0 {
+		t.Fatalf("generateDNSResponse() returned empty response")
+	}
+
+	resp2, err := c.generateDNSResponse(req, nodeID)
+	if err != nil {
+		t.Fatalf("generateDNSResponse() second call error = %v", err)
+	}
+
+	if !cmp.Equal(resp1, resp2) {
+		t.Errorf("generateDNSResponse() responses differ between calls")
+	}
+}
+
+func TestIPPoolExhaustion(t *testing.T) {
+	smallPrefix := netip.MustParsePrefix("100.64.1.0/30") // Only 4 IPs: .0, .1, .2, .3
+	c := &connector{
+		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		v4Ranges:      []netip.Prefix{smallPrefix},
+		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
+		numV4DNSAddrs: 3,
+	}
+
+	ps := &perPeerState{c: c}
+
+	assignedIPs := make(map[netip.Addr]string)
+
+	domains := []string{"a.example.com", "b.example.com", "c.example.com", "d.example.com"}
+
+	var errs []error
+
+	for i := 0; i < 5; i++ {
+		for _, domain := range domains {
+			addrs, err := ps.ipForDomain(domain)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to get IP for domain %q: %w", domain, err))
+				continue
+			}
+
+			for _, addr := range addrs {
+				if d, ok := assignedIPs[addr]; ok {
+					if d != domain {
+						t.Errorf("IP %s reused for domain %q, previously assigned to %q", addr, domain, d)
+					}
+				} else {
+					assignedIPs[addr] = domain
+				}
+			}
+		}
+	}
+
+	for addr, domain := range assignedIPs {
+		if addr.Is4() && !smallPrefix.Contains(addr) {
+			t.Errorf("IP %s for domain %q not in expected range %s", addr, domain, smallPrefix)
+		}
+		if addr.Is6() && !c.v6ULA.Contains(addr) {
+			t.Errorf("IP %s for domain %q not in expected range %s", addr, domain, c.v6ULA)
+		}
+		if addr == c.dnsAddr {
+			t.Errorf("IP %s for domain %q is the reserved DNS address", addr, domain)
+		}
+	}
+
+	// expect one error for each iteration with the 4th domain
+	if len(errs) != 5 {
+		t.Errorf("Expected 5 errors, got %d: %v", len(errs), errs)
+	}
+	for _, err := range errs {
+		if !errors.Is(err, ErrNoIPsAvailable) {
+			t.Errorf("generateDNSResponse() error = %v, want ErrNoIPsAvailable", err)
+		}
+	}
+}

cmd/tailscale/cli/debug.go

@@ -43,6 +43,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/must"
 )
 
@@ -136,6 +137,17 @@ func debugCmd() *ffcli.Command {
 				Exec:       runLocalCreds,
 				ShortHelp:  "Print how to access Tailscale LocalAPI",
 			},
+			{
+				Name:       "localapi",
+				ShortUsage: "tailscale debug localapi [<method>] <path> [<body| \"-\">]",
+				Exec:       runLocalAPI,
+				ShortHelp:  "Call a LocalAPI method directly",
+				FlagSet: (func() *flag.FlagSet {
+					fs := newFlagSet("localapi")
+					fs.BoolVar(&localAPIFlags.verbose, "v", false, "verbose; dump HTTP headers")
+					return fs
+				})(),
+			},
 			{
 				Name:       "restun",
 				ShortUsage: "tailscale debug restun",
@@ -451,6 +463,81 @@ func runLocalCreds(ctx context.Context, args []string) error {
 	return nil
 }
 
+func looksLikeHTTPMethod(s string) bool {
+	if len(s) > len("OPTIONS") {
+		return false
+	}
+	for _, r := range s {
+		if r < 'A' || r > 'Z' {
+			return false
+		}
+	}
+	return true
+}
+
+var localAPIFlags struct {
+	verbose bool
+}
+
+func runLocalAPI(ctx context.Context, args []string) error {
+	if len(args) == 0 {
+		return errors.New("expected at least one argument")
+	}
+	method := "GET"
+	if looksLikeHTTPMethod(args[0]) {
+		method = args[0]
+		args = args[1:]
+		if len(args) == 0 {
+			return errors.New("expected at least one argument after method")
+		}
+	}
+	path := args[0]
+	if !strings.HasPrefix(path, "/localapi/") {
+		if !strings.Contains(path, "/") {
+			path = "/localapi/v0/" + path
+		} else {
+			path = "/localapi/" + path
+		}
+	}
+
+	var body io.Reader
+	if len(args) > 1 {
+		if args[1] == "-" {
+			fmt.Fprintf(Stderr, "# reading request body from stdin...\n")
+			all, err := io.ReadAll(os.Stdin)
+			if err != nil {
+				return fmt.Errorf("reading Stdin: %q", err)
+			}
+			body = bytes.NewReader(all)
+		} else {
+			body = strings.NewReader(args[1])
+		}
+	}
+	req, err := http.NewRequest(method, "http://local-tailscaled.sock"+path, body)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(Stderr, "# doing request %s %s\n", method, path)
+
+	res, err := localClient.DoLocalRequest(req)
+	if err != nil {
+		return err
+	}
+	is2xx := res.StatusCode >= 200 && res.StatusCode <= 299
+	if localAPIFlags.verbose {
+		res.Write(Stdout)
+	} else {
+		if !is2xx {
+			fmt.Fprintf(Stderr, "# Response status %s\n", res.Status)
+		}
+		io.Copy(Stdout, res.Body)
+	}
+	if is2xx {
+		return nil
+	}
+	return errors.New(res.Status)
+}
+
 type localClientRoundTripper struct{}
 
 func (localClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -870,7 +957,10 @@ func runTS2021(ctx context.Context, args []string) error {
 		logf = log.Printf
 	}
 
-	netMon, err := netmon.New(logger.WithPrefix(logf, "netmon: "))
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logger.WithPrefix(logf, "netmon: "))
 	if err != nil {
 		return fmt.Errorf("creating netmon: %w", err)
 	}

cmd/tailscale/cli/netcheck.go

@@ -24,6 +24,7 @@ import (
 	"tailscale.com/net/tlsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 var netcheckCmd = &ffcli.Command{
@@ -48,14 +49,19 @@ var netcheckArgs struct {
 
 func runNetcheck(ctx context.Context, args []string) error {
 	logf := logger.WithPrefix(log.Printf, "portmap: ")
-	netMon, err := netmon.New(logf)
+	bus := eventbus.New()
+	defer bus.Close()
+	netMon, err := netmon.New(bus, logf)
 	if err != nil {
 		return err
 	}
 
 	// Ensure that we close the portmapper after running a netcheck; this
 	// will release any port mappings created.
-	pm := portmapper.NewClient(logf, netMon, nil, nil, nil)
+	pm := portmapper.NewClient(portmapper.Config{
+		Logf:   logf,
+		NetMon: netMon,
+	})
 	defer pm.Close()
 
 	c := &netcheck.Client{

cmd/tailscale/depaware.txt

@@ -5,6 +5,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/coder/websocket                                   from tailscale.com/util/eventbus
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
@@ -89,6 +93,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/drive                                          from tailscale.com/client/local+
         tailscale.com/envknob                                        from tailscale.com/client/local+
         tailscale.com/envknob/featureknob                            from tailscale.com/client/web
+        tailscale.com/feature                                        from tailscale.com/tsweb
         tailscale.com/feature/capture/dissector                      from tailscale.com/cmd/tailscale/cli
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/health/healthmsg                               from tailscale.com/cmd/tailscale/cli
@@ -131,7 +136,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tstime                                         from tailscale.com/control/controlhttp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric
+        tailscale.com/tsweb                                          from tailscale.com/util/eventbus
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetric+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg+
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/ipn+
@@ -156,6 +162,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
      💣 tailscale.com/util/deephash                                  from tailscale.com/util/syspolicy/setting
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/util/eventbus                                  from tailscale.com/net/portmapper+
         tailscale.com/util/groupmember                               from tailscale.com/client/web
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
@@ -166,6 +173,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/must                                      from tailscale.com/clientupdate/distsign+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
         tailscale.com/util/quarantine                                from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/set                                       from tailscale.com/derp+
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache+
         tailscale.com/util/slicesx                                   from tailscale.com/net/dns/recursive+
@@ -328,7 +336,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         hash/crc32                                                   from compress/gzip+
         hash/maphash                                                 from go4.org/mem
         html                                                         from html/template+
-        html/template                                                from github.com/gorilla/csrf
+        html/template                                                from github.com/gorilla/csrf+
         image                                                        from github.com/skip2/go-qrcode+
         image/color                                                  from github.com/skip2/go-qrcode+
         image/png                                                    from github.com/skip2/go-qrcode
@@ -352,7 +360,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         internal/nettrace                                            from net+
         internal/oserror                                             from io/fs+
         internal/poll                                                from net+
-        internal/profilerecord                                       from runtime
+        internal/profile                                             from net/http/pprof
+        internal/profilerecord                                       from runtime+
         internal/race                                                from internal/poll+
         internal/reflectlite                                         from context+
         internal/runtime/atomic                                      from internal/runtime/exithook+
@@ -394,6 +403,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         net/http/httputil                                            from tailscale.com/client/web+
         net/http/internal                                            from net/http+
         net/http/internal/ascii                                      from net/http+
+        net/http/pprof                                               from tailscale.com/tsweb
         net/netip                                                    from go4.org/netipx+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
@@ -408,6 +418,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         regexp/syntax                                                from regexp
         runtime                                                      from archive/tar+
         runtime/debug                                                from tailscale.com+
+        runtime/pprof                                                from net/http/pprof
+        runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/client/web+
         sort                                                         from compress/flate+
         strconv                                                      from archive/tar+

cmd/tailscaled/debug.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/util/eventbus"
 )
 
 var debugArgs struct {
@@ -72,11 +73,14 @@ func debugMode(args []string) error {
 }
 
 func runMonitor(ctx context.Context, loop bool) error {
+	b := eventbus.New()
+	defer b.Close()
+
 	dump := func(st *netmon.State) {
 		j, _ := json.MarshalIndent(st, "", "    ")
 		os.Stderr.Write(j)
 	}
-	mon, err := netmon.New(log.Printf)
+	mon, err := netmon.New(b, log.Printf)
 	if err != nil {
 		return err
 	}

cmd/tailscaled/depaware.txt

@@ -81,6 +81,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http                      from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
+        github.com/coder/websocket                                   from tailscale.com/util/eventbus
+        github.com/coder/websocket/internal/errd                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/util                     from github.com/coder/websocket
+        github.com/coder/websocket/internal/xsync                    from github.com/coder/websocket
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
@@ -353,6 +357,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tstime                                         from tailscale.com/control/controlclient+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tsweb                                          from tailscale.com/util/eventbus
         tailscale.com/tsweb/varz                                     from tailscale.com/cmd/tailscaled+
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
@@ -382,6 +387,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics+
         tailscale.com/util/dnsname                                   from tailscale.com/appc+
+        tailscale.com/util/eventbus                                  from tailscale.com/tsd+
         tailscale.com/util/execqueue                                 from tailscale.com/control/controlclient+
         tailscale.com/util/goroutines                                from tailscale.com/ipn/ipnlocal
         tailscale.com/util/groupmember                               from tailscale.com/client/web+
@@ -587,7 +593,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         hash/crc32                                                   from compress/gzip+
         hash/maphash                                                 from go4.org/mem
         html                                                         from html/template+
-        html/template                                                from github.com/gorilla/csrf
+        html/template                                                from github.com/gorilla/csrf+
         internal/abi                                                 from crypto/x509/internal/macos+
         internal/asan                                                from internal/runtime/maps+
         internal/bisect                                              from internal/godebug

cmd/tailscaled/tailscaled.go

@@ -339,7 +339,9 @@ var debugMux *http.ServeMux
 func run() (err error) {
 	var logf logger.Logf = log.Printf
 
-	sys := new(tsd.System)
+	// Install an event bus as early as possible, so that it's
+	// available universally when setting up everything else.
+	sys := tsd.NewSystem()
 
 	// Parse config, if specified, to fail early if it's invalid.
 	var conf *conffile.Config
@@ -354,9 +356,7 @@ func run() (err error) {
 	var netMon *netmon.Monitor
 	isWinSvc := isWindowsService()
 	if !isWinSvc {
-		netMon, err = netmon.New(func(format string, args ...any) {
-			logf(format, args...)
-		})
+		netMon, err = netmon.New(sys.Bus.Get(), logf)
 		if err != nil {
 			return fmt.Errorf("netmon.New: %w", err)
 		}

cmd/tailscaled/tailscaled_windows.go

@@ -327,8 +327,8 @@ func beWindowsSubprocess() bool {
 		log.Printf("Error pre-loading \"%s\": %v", fqWintunPath, err)
 	}
 
-	sys := new(tsd.System)
-	netMon, err := netmon.New(log.Printf)
+	sys := tsd.NewSystem()
+	netMon, err := netmon.New(sys.Bus.Get(), log.Printf)
 	if err != nil {
 		log.Fatalf("Could not create netMon: %v", err)
 	}

cmd/tsconnect/wasm/wasm_js.go

@@ -100,7 +100,7 @@ func newIPN(jsConfig js.Value) map[string]any {
 	logtail := logtail.NewLogger(c, log.Printf)
 	logf := logtail.Logf
 
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	sys.Set(store)
 	dialer := &tsdial.Dialer{Logf: logf}
 	eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{

control/controlhttp/client.go

@@ -96,6 +96,9 @@ func (a *Dialer) httpsFallbackDelay() time.Duration {
 var _ = envknob.RegisterBool("TS_USE_CONTROL_DIAL_PLAN") // to record at init time whether it's in use
 
 func (a *Dialer) dial(ctx context.Context) (*ClientConn, error) {
+
+	a.logPort80Failure.Store(true)
+
 	// If we don't have a dial plan, just fall back to dialing the single
 	// host we know about.
 	useDialPlan := envknob.BoolDefaultTrue("TS_USE_CONTROL_DIAL_PLAN")
@@ -278,7 +281,9 @@ func (d *Dialer) forceNoise443() bool {
 		// This heuristic works around networks where port 80 is MITMed and
 		// appears to work for a bit post-Upgrade but then gets closed,
 		// such as seen in https://github.com/tailscale/tailscale/issues/13597.
-		d.logf("controlhttp: forcing port 443 dial due to recent noise dial")
+		if d.logPort80Failure.CompareAndSwap(true, false) {
+			d.logf("controlhttp: forcing port 443 dial due to recent noise dial")
+		}
 		return true
 	}
 

control/controlhttp/constants.go

@@ -6,6 +6,7 @@ package controlhttp
 import (
 	"net/http"
 	"net/url"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/health"
@@ -90,6 +91,11 @@ type Dialer struct {
 
 	proxyFunc func(*http.Request) (*url.URL, error) // or nil
 
+	// logPort80Failure is whether we should log about port 80 interceptions
+	// and forcing a port 443 dial. We do this only once per "dial" method
+	// which can result in many concurrent racing dialHost calls.
+	logPort80Failure atomic.Bool
+
 	// For tests only
 	drainFinished        chan struct{}
 	omitCertErrorLogging bool

control/controlhttp/http_test.go

@@ -32,7 +32,6 @@ import (
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
-	"tailscale.com/tstest/deptest"
 	"tailscale.com/tstime"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -822,14 +821,3 @@ func (c *closeTrackConn) Close() error {
 	c.d.noteClose(c)
 	return c.Conn.Close()
 }
-
-func TestDeps(t *testing.T) {
-	deptest.DepChecker{
-		GOOS:   "darwin",
-		GOARCH: "arm64",
-		BadDeps: map[string]string{
-			// Only the controlhttpserver needs WebSockets...
-			"github.com/coder/websocket": "controlhttp client shouldn't need websockets",
-		},
-	}.Check(t)
-}

derp/derphttp/derphttp_test.go

@@ -17,9 +17,7 @@ import (
 
 	"tailscale.com/derp"
 	"tailscale.com/net/netmon"
-	"tailscale.com/tstest/deptest"
 	"tailscale.com/types/key"
-	"tailscale.com/util/set"
 )
 
 func TestSendRecv(t *testing.T) {
@@ -487,23 +485,3 @@ func TestProbe(t *testing.T) {
 		}
 	}
 }
-
-func TestDeps(t *testing.T) {
-	deptest.DepChecker{
-		GOOS:   "darwin",
-		GOARCH: "arm64",
-		BadDeps: map[string]string{
-			"github.com/coder/websocket": "shouldn't link websockets except on js/wasm",
-		},
-	}.Check(t)
-
-	deptest.DepChecker{
-		GOOS:   "darwin",
-		GOARCH: "arm64",
-		Tags:   "ts_debug_websockets",
-		WantDeps: set.Of(
-			"github.com/coder/websocket",
-		),
-	}.Check(t)
-
-}

internal/client/tailscale/vip_service.go

@@ -27,6 +27,8 @@ type VIPService struct {
 	Addrs []string `json:"addrs,omitempty"`
 	// Comment is an optional text string for display in the admin panel.
 	Comment string `json:"comment,omitempty"`
+	// Annotations are optional key-value pairs that can be used to store arbitrary metadata.
+	Annotations map[string]string `json:"annotations,omitempty"`
 	// Ports are the ports of a VIPService that will be configured via Tailscale serve config.
 	// If set, any node wishing to advertise this VIPService must have this port configured via Tailscale serve.
 	Ports []string `json:"ports,omitempty"`

ipn/ipnlocal/local.go

@@ -958,7 +958,9 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) {
 
 	if peerAPIListenAsync && b.netMap != nil && b.state == ipn.Running {
 		want := b.netMap.GetAddresses().Len()
-		if len(b.peerAPIListeners) < want {
+		have := len(b.peerAPIListeners)
+		b.logf("[v1] linkChange: have %d peerAPIListeners, want %d", have, want)
+		if have < want {
 			b.logf("linkChange: peerAPIListeners too low; trying again")
 			b.goTracker.Go(b.initPeerAPIListener)
 		}
@@ -2380,12 +2382,10 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 	b.applyPrefsToHostinfoLocked(hostinfo, prefs)
 
-	b.setNetMapLocked(nil)
 	persistv := prefs.Persist().AsStruct()
 	if persistv == nil {
 		persistv = new(persist.Persist)
 	}
-	b.updateFilterLocked(nil, ipn.PrefsView{})
 
 	if b.portpoll != nil {
 		b.portpollOnce.Do(func() {
@@ -2404,11 +2404,9 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	var auditLogShutdown func()
-	// Audit logging is only available if the client has set up a proper persistent
-	// store for the logs in sys.
 	store, ok := b.sys.AuditLogStore.GetOK()
 	if !ok {
-		b.logf("auditlog: [unexpected] no persistent audit log storage configured.  using memory store.")
+		// Use memory store by default if no explicit store is provided.
 		store = auditlog.NewLogStore(&memstore.Store{})
 	}
 
@@ -3481,18 +3479,20 @@ func (b *LocalBackend) onTailnetDefaultAutoUpdate(au bool) {
 		// can still manually enable auto-updates on this node.
 		return
 	}
-	b.logf("using tailnet default auto-update setting: %v", au)
-	prefsClone := prefs.AsStruct()
-	prefsClone.AutoUpdate.Apply = opt.NewBool(au)
-	_, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{
-		Prefs: *prefsClone,
-		AutoUpdateSet: ipn.AutoUpdatePrefsMask{
-			ApplySet: true,
-		},
-	}, unlock)
-	if err != nil {
-		b.logf("failed to apply tailnet-wide default for auto-updates (%v): %v", au, err)
-		return
+	if clientupdate.CanAutoUpdate() {
+		b.logf("using tailnet default auto-update setting: %v", au)
+		prefsClone := prefs.AsStruct()
+		prefsClone.AutoUpdate.Apply = opt.NewBool(au)
+		_, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{
+			Prefs: *prefsClone,
+			AutoUpdateSet: ipn.AutoUpdatePrefsMask{
+				ApplySet: true,
+			},
+		}, unlock)
+		if err != nil {
+			b.logf("failed to apply tailnet-wide default for auto-updates (%v): %v", au, err)
+			return
+		}
 	}
 }
 
@@ -4968,7 +4968,7 @@ func (b *LocalBackend) authReconfig() {
 		return
 	}
 
-	oneCGNATRoute := shouldUseOneCGNATRoute(b.logf, b.sys.ControlKnobs(), version.OS())
+	oneCGNATRoute := shouldUseOneCGNATRoute(b.logf, b.sys.NetMon.Get(), b.sys.ControlKnobs(), version.OS())
 	rcfg := b.routerConfig(cfg, prefs, oneCGNATRoute)
 
 	err = b.e.Reconfig(cfg, rcfg, dcfg)
@@ -4992,7 +4992,7 @@ func (b *LocalBackend) authReconfig() {
 //
 // The versionOS is a Tailscale-style version ("iOS", "macOS") and not
 // a runtime.GOOS.
-func shouldUseOneCGNATRoute(logf logger.Logf, controlKnobs *controlknobs.Knobs, versionOS string) bool {
+func shouldUseOneCGNATRoute(logf logger.Logf, mon *netmon.Monitor, controlKnobs *controlknobs.Knobs, versionOS string) bool {
 	if controlKnobs != nil {
 		// Explicit enabling or disabling always take precedence.
 		if v, ok := controlKnobs.OneCGNAT.Load().Get(); ok {
@@ -5007,7 +5007,7 @@ func shouldUseOneCGNATRoute(logf logger.Logf, controlKnobs *controlknobs.Knobs,
 	// use fine-grained routes if another interfaces is also using the CGNAT
 	// IP range.
 	if versionOS == "macOS" {
-		hasCGNATInterface, err := netmon.HasCGNATInterface()
+		hasCGNATInterface, err := mon.HasCGNATInterface()
 		if err != nil {
 			logf("shouldUseOneCGNATRoute: Could not determine if any interfaces use CGNAT: %v", err)
 			return false
@@ -5369,6 +5369,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 			ln, err = ps.listen(a.Addr(), b.prevIfState)
 			if err != nil {
 				if peerAPIListenAsync {
+					b.logf("possibly transient peerapi listen(%q) error, will try again on linkChange: %v", a.Addr(), err)
 					// Expected. But we fix it later in linkChange
 					// ("peerAPIListeners too low").
 					continue
@@ -5920,6 +5921,9 @@ func (b *LocalBackend) requestEngineStatusAndWait() {
 	b.logf("requestEngineStatusAndWait: got status update.")
 }
 
+// [controlclient.Auto] implements [auditlog.Transport].
+var _ auditlog.Transport = (*controlclient.Auto)(nil)
+
 // setControlClientLocked sets the control client to cc,
 // which may be nil.
 //
@@ -5927,12 +5931,12 @@ func (b *LocalBackend) requestEngineStatusAndWait() {
 func (b *LocalBackend) setControlClientLocked(cc controlclient.Client) {
 	b.cc = cc
 	b.ccAuto, _ = cc.(*controlclient.Auto)
-	if b.auditLogger != nil {
+	if t, ok := b.cc.(auditlog.Transport); ok && b.auditLogger != nil {
 		if err := b.auditLogger.SetProfileID(b.pm.CurrentProfile().ID()); err != nil {
 			b.logf("audit logger set profile ID failure: %v", err)
 		}
 
-		if err := b.auditLogger.Start(b.ccAuto); err != nil {
+		if err := b.auditLogger.Start(t); err != nil {
 			b.logf("audit logger start failure: %v", err)
 		}
 	}
@@ -7531,6 +7535,7 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err
 		return nil
 	}
 	b.setNetMapLocked(nil) // Reset netmap.
+	b.updateFilterLocked(nil, ipn.PrefsView{})
 	// Reset the NetworkMap in the engine
 	b.e.SetNetworkMap(new(netmap.NetworkMap))
 	if prevCC := b.resetControlClientLocked(); prevCC != nil {

ipn/ipnlocal/local_test.go

@@ -436,7 +436,7 @@ func (panicOnUseTransport) RoundTrip(*http.Request) (*http.Response, error) {
 }
 
 func newTestLocalBackend(t testing.TB) *LocalBackend {
-	return newTestLocalBackendWithSys(t, new(tsd.System))
+	return newTestLocalBackendWithSys(t, tsd.NewSystem())
 }
 
 // newTestLocalBackendWithSys creates a new LocalBackend with the given tsd.System.
@@ -448,7 +448,7 @@ func newTestLocalBackendWithSys(t testing.TB, sys *tsd.System) *LocalBackend {
 		sys.Set(new(mem.Store))
 	}
 	if _, ok := sys.Engine.GetOK(); !ok {
-		eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+		eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 		if err != nil {
 			t.Fatalf("NewFakeUserspaceEngine: %v", err)
 		}
@@ -1510,6 +1510,15 @@ func TestReconfigureAppConnector(t *testing.T) {
 func TestBackfillAppConnectorRoutes(t *testing.T) {
 	// Create backend with an empty app connector.
 	b := newTestBackend(t)
+	// newTestBackend creates a backend with a non-nil netmap,
+	// but this test requires a nil netmap.
+	// Otherwise, instead of backfilling, [LocalBackend.reconfigAppConnectorLocked]
+	// uses the domains and routes from netmap's [appctype.AppConnectorAttr].
+	// Additionally, a non-nil netmap makes reconfigAppConnectorLocked
+	// asynchronous, resulting in a flaky test.
+	// Therefore, we set the netmap to nil to simulate a fresh backend start
+	// or a profile switch where the netmap is not yet available.
+	b.setNetMapLocked(nil)
 	if err := b.Start(ipn.Options{}); err != nil {
 		t.Fatal(err)
 	}
@@ -4402,10 +4411,10 @@ func newLocalBackendWithTestControl(t *testing.T, enableLogging bool, newControl
 	if enableLogging {
 		logf = tstest.WhileTestRunningLogger(t)
 	}
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	store := new(mem.Store)
 	sys.Set(store)
-	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}
@@ -4850,9 +4859,8 @@ func TestConfigFileReload(t *testing.T) {
 			// Create backend with initial config
 			tc.initial.Path = path
 			tc.initial.Raw = initialJSON
-			sys := &tsd.System{
-				InitialConfig: tc.initial,
-			}
+			sys := tsd.NewSystem()
+			sys.InitialConfig = tc.initial
 			b := newTestLocalBackendWithSys(t, sys)
 
 			// Update config file

ipn/ipnlocal/loglines_test.go

@@ -47,10 +47,10 @@ func TestLocalLogLines(t *testing.T) {
 	idA := logid(0xaa)
 
 	// set up a LocalBackend, super bare bones. No functional data.
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	store := new(mem.Store)
 	sys.Set(store)
-	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		t.Fatal(err)
 	}

ipn/ipnlocal/peerapi.go

@@ -481,7 +481,7 @@ func (h *peerAPIHandler) handleServeInterfaces(w http.ResponseWriter, r *http.Re
 		fmt.Fprintf(w, "<h3>Could not get the default route: %s</h3>\n", html.EscapeString(err.Error()))
 	}
 
-	if hasCGNATInterface, err := netmon.HasCGNATInterface(); hasCGNATInterface {
+	if hasCGNATInterface, err := h.ps.b.sys.NetMon.Get().HasCGNATInterface(); hasCGNATInterface {
 		fmt.Fprintln(w, "<p>There is another interface using the CGNAT range.</p>")
 	} else if err != nil {
 		fmt.Fprintf(w, "<p>Could not check for CGNAT interfaces: %s</p>\n", html.EscapeString(err.Error()))

ipn/ipnlocal/peerapi_test.go

@@ -34,6 +34,7 @@ import (
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/must"
 	"tailscale.com/util/usermetric"
 	"tailscale.com/wgengine"
@@ -643,9 +644,12 @@ func TestPeerAPIReplyToDNSQueries(t *testing.T) {
 	h.isSelf = false
 	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
+	bus := eventbus.New()
+	defer bus.Close()
+
 	ht := new(health.Tracker)
 	reg := new(usermetric.Registry)
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg)
+	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg, bus)
 	pm := must.Get(newProfileManager(new(mem.Store), t.Logf, ht))
 	h.ps = &peerAPIServer{
 		b: &LocalBackend{
@@ -695,9 +699,12 @@ func TestPeerAPIPrettyReplyCNAME(t *testing.T) {
 		var h peerAPIHandler
 		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
+		bus := eventbus.New()
+		defer bus.Close()
+
 		ht := new(health.Tracker)
 		reg := new(usermetric.Registry)
-		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg)
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg, bus)
 		pm := must.Get(newProfileManager(new(mem.Store), t.Logf, ht))
 		var a *appc.AppConnector
 		if shouldStore {
@@ -768,10 +775,12 @@ func TestPeerAPIReplyToDNSQueriesAreObserved(t *testing.T) {
 		var h peerAPIHandler
 		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
+		bus := eventbus.New()
+		defer bus.Close()
 		rc := &appctest.RouteCollector{}
 		ht := new(health.Tracker)
 		reg := new(usermetric.Registry)
-		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg)
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg, bus)
 		pm := must.Get(newProfileManager(new(mem.Store), t.Logf, ht))
 		var a *appc.AppConnector
 		if shouldStore {
@@ -833,10 +842,12 @@ func TestPeerAPIReplyToDNSQueriesAreObservedWithCNAMEFlattening(t *testing.T) {
 		var h peerAPIHandler
 		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
+		bus := eventbus.New()
+		defer bus.Close()
 		ht := new(health.Tracker)
 		reg := new(usermetric.Registry)
 		rc := &appctest.RouteCollector{}
-		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg)
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0, ht, reg, bus)
 		pm := must.Get(newProfileManager(new(mem.Store), t.Logf, ht))
 		var a *appc.AppConnector
 		if shouldStore {

ipn/ipnlocal/serve_test.go

@@ -877,11 +877,12 @@ func newTestBackend(t *testing.T) *LocalBackend {
 		logf = logger.WithPrefix(tstest.WhileTestRunningLogger(t), "... ")
 	}
 
-	sys := &tsd.System{}
+	sys := tsd.NewSystem()
 	e, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
 		SetSubsystem:  sys.Set,
 		HealthTracker: sys.HealthTracker(),
 		Metrics:       sys.UserMetricsRegistry(),
+		EventBus:      sys.Bus.Get(),
 	})
 	if err != nil {
 		t.Fatal(err)

ipn/ipnlocal/state_test.go

@@ -295,10 +295,10 @@ func TestStateMachine(t *testing.T) {
 	c := qt.New(t)
 
 	logf := tstest.WhileTestRunningLogger(t)
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	store := new(testStateStorage)
 	sys.Set(store)
-	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}
@@ -735,12 +735,10 @@ func TestStateMachine(t *testing.T) {
 		// b.Shutdown() explicitly ourselves.
 		previousCC.assertShutdown(false)
 
-		// Note: unpause happens because ipn needs to get at least one netmap
-		//  on startup, otherwise UIs can't show the node list, login
-		//  name, etc when in state ipn.Stopped.
-		//  Arguably they shouldn't try. But they currently do.
 		nn := notifies.drain(2)
-		cc.assertCalls("New", "Login")
+		// We already have a netmap for this node,
+		// and WantRunning is false, so cc should be paused.
+		cc.assertCalls("New", "Login", "pause")
 		c.Assert(nn[0].Prefs, qt.IsNotNil)
 		c.Assert(nn[1].State, qt.IsNotNil)
 		c.Assert(nn[0].Prefs.WantRunning(), qt.IsFalse)
@@ -751,7 +749,11 @@ func TestStateMachine(t *testing.T) {
 	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
 	// the first netmap. Simulate that netmap being received, after which
 	// it should pause us, to avoid wasting CPU retrieving unnecessarily
-	// additional netmap updates.
+	// additional netmap updates. Since our LocalBackend instance already
+	// has a netmap, we will reset it to nil to simulate the first netmap
+	// retrieval.
+	b.setNetMapLocked(nil)
+	cc.assertCalls("unpause")
 	//
 	// TODO: really the various GUIs and prefs should be refactored to
 	//  not require the netmap structure at all when starting while
@@ -853,7 +855,7 @@ func TestStateMachine(t *testing.T) {
 	// The last test case is the most common one: restarting when both
 	// logged in and WantRunning.
 	t.Logf("\n\nStart5")
-	notifies.expect(1)
+	notifies.expect(2)
 	c.Assert(b.Start(ipn.Options{}), qt.IsNil)
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
@@ -861,30 +863,32 @@ func TestStateMachine(t *testing.T) {
 		previousCC.assertShutdown(false)
 		cc.assertCalls("New", "Login")
 
-		nn := notifies.drain(1)
+		nn := notifies.drain(2)
 		cc.assertCalls()
 		c.Assert(nn[0].Prefs, qt.IsNotNil)
 		c.Assert(nn[0].Prefs.LoggedOut(), qt.IsFalse)
 		c.Assert(nn[0].Prefs.WantRunning(), qt.IsTrue)
-		c.Assert(b.State(), qt.Equals, ipn.NoState)
+		// We're logged in and have a valid netmap, so we should
+		// be in the Starting state.
+		c.Assert(nn[1].State, qt.IsNotNil)
+		c.Assert(*nn[1].State, qt.Equals, ipn.Starting)
+		c.Assert(b.State(), qt.Equals, ipn.Starting)
 	}
 
 	// Control server accepts our valid key from before.
 	t.Logf("\n\nLoginFinished5")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		SelfNode: (&tailcfg.Node{MachineAuthorized: true}).View(),
 	})
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		cc.assertCalls()
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
-		c.Assert(nn[0].State, qt.IsNotNil)
-		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 		// NOTE: No prefs change this time. WantRunning stays true.
 		// We were in Starting in the first place, so that doesn't
-		// change either.
+		// change either, so we don't expect any notifications.
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
 	t.Logf("\n\nExpireKey")
@@ -930,9 +934,9 @@ func TestStateMachine(t *testing.T) {
 
 func TestEditPrefsHasNoKeys(t *testing.T) {
 	logf := tstest.WhileTestRunningLogger(t)
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	sys.Set(new(mem.Store))
-	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}
@@ -1010,10 +1014,10 @@ func TestWGEngineStatusRace(t *testing.T) {
 	t.Skip("test fails")
 	c := qt.New(t)
 	logf := tstest.WhileTestRunningLogger(t)
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	sys.Set(new(mem.Store))
 
-	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set)
+	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.Bus.Get())
 	c.Assert(err, qt.IsNil)
 	t.Cleanup(eng.Close)
 	sys.Set(eng)

ipn/ipnserver/server_test.go

@@ -517,12 +517,12 @@ type newControlClientFn func(tb testing.TB, opts controlclient.Options) controlc
 func newLocalBackendWithTestControl(tb testing.TB, newControl newControlClientFn, enableLogging bool) *ipnlocal.LocalBackend {
 	tb.Helper()
 
-	sys := &tsd.System{}
+	sys := tsd.NewSystem()
 	store := &mem.Store{}
 	sys.Set(store)
 
 	logf := testLogger(tb, enableLogging)
-	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	e, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		tb.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}

ipn/localapi/localapi.go

@@ -56,6 +56,7 @@ import (
 	"tailscale.com/types/ptr"
 	"tailscale.com/types/tkatype"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/httphdr"
 	"tailscale.com/util/httpm"
 	"tailscale.com/util/mak"
@@ -818,23 +819,31 @@ func (h *Handler) serveDebugPortmap(w http.ResponseWriter, r *http.Request) {
 	done := make(chan bool, 1)
 
 	var c *portmapper.Client
-	c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), h.b.NetMon(), debugKnobs, h.b.ControlKnobs(), func() {
-		logf("portmapping changed.")
-		logf("have mapping: %v", c.HaveMapping())
+	c = portmapper.NewClient(portmapper.Config{
+		Logf:         logger.WithPrefix(logf, "portmapper: "),
+		NetMon:       h.b.NetMon(),
+		DebugKnobs:   debugKnobs,
+		ControlKnobs: h.b.ControlKnobs(),
+		OnChange: func() {
+			logf("portmapping changed.")
+			logf("have mapping: %v", c.HaveMapping())
 
-		if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
-			logf("cb: mapping: %v", ext)
-			select {
-			case done <- true:
-			default:
+			if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+				logf("cb: mapping: %v", ext)
+				select {
+				case done <- true:
+				default:
+				}
+				return
 			}
-			return
-		}
-		logf("cb: no mapping")
+			logf("cb: no mapping")
+		},
 	})
 	defer c.Close()
 
-	netMon, err := netmon.New(logger.WithPrefix(logf, "monitor: "))
+	bus := eventbus.New()
+	defer bus.Close()
+	netMon, err := netmon.New(bus, logger.WithPrefix(logf, "monitor: "))
 	if err != nil {
 		logf("error creating monitor: %v", err)
 		return

ipn/localapi/localapi_test.go

@@ -336,10 +336,10 @@ func TestServeWatchIPNBus(t *testing.T) {
 
 func newTestLocalBackend(t testing.TB) *ipnlocal.LocalBackend {
 	var logf logger.Logf = logger.Discard
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	store := new(mem.Store)
 	sys.Set(store)
-	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}

ipn/store/kubestore/store_kube.go

@@ -143,15 +143,6 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
 	if err := dnsname.ValidHostname(domain); err != nil {
 		return fmt.Errorf("invalid domain name %q: %w", domain, err)
 	}
-	defer func() {
-		// TODO(irbekrm): a read between these two separate writes would
-		// get a mismatched cert and key.  Allow writing both cert and
-		// key to the memory store in a single, lock-protected operation.
-		if err == nil {
-			s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
-			s.memory.WriteState(ipn.StateKey(domain+".key"), key)
-		}
-	}()
 	secretName := s.secretName
 	data := map[string][]byte{
 		domain + ".crt": cert,
@@ -166,19 +157,32 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
 			keyTLSKey:  key,
 		}
 	}
-	return s.updateSecret(data, secretName)
+	if err := s.updateSecret(data, secretName); err != nil {
+		return fmt.Errorf("error writing TLS cert and key to Secret: %w", err)
+	}
+	// TODO(irbekrm): certs for write replicas are currently not
+	// written to memory to avoid out of sync memory state after
+	// Ingress resources have been recreated.  This means that TLS
+	// certs for write replicas are retrieved from the Secret on
+	// each HTTPS request.  This is a temporary solution till we
+	// implement a Secret watch.
+	if s.certShareMode != "rw" {
+		s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
+		s.memory.WriteState(ipn.StateKey(domain+".key"), key)
+	}
+	return nil
 }
 
 // ReadTLSCertAndKey reads a TLS cert and key from memory or from a
 // domain-specific Secret. It first checks the in-memory store, if not found in
 // memory and running cert store in read-only mode, looks up a Secret.
+// Note that write replicas of HA Ingress always retrieve TLS certs from Secrets.
 func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
 	if err := dnsname.ValidHostname(domain); err != nil {
 		return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
 	}
 	certKey := domain + ".crt"
 	keyKey := domain + ".key"
-
 	cert, err = s.memory.ReadState(ipn.StateKey(certKey))
 	if err == nil {
 		key, err = s.memory.ReadState(ipn.StateKey(keyKey))
@@ -186,16 +190,12 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
 			return cert, key, nil
 		}
 	}
-	if s.certShareMode != "ro" {
+	if s.certShareMode == "" {
 		return nil, nil, ipn.ErrStateNotExist
 	}
-	// If we are in cert share read only mode, it is possible that a write
-	// replica just issued the TLS cert for this DNS name and it has not
-	// been loaded to store yet, so check the Secret.
 
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
-
 	secret, err := s.client.GetSecret(ctx, domain)
 	if err != nil {
 		if kubeclient.IsNotFoundErr(err) {
@@ -212,9 +212,18 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
 	}
 	// TODO(irbekrm): a read between these two separate writes would
 	// get a mismatched cert and key.  Allow writing both cert and
-	// key to the memory store in a single lock-protected operation.
-	s.memory.WriteState(ipn.StateKey(certKey), cert)
-	s.memory.WriteState(ipn.StateKey(keyKey), key)
+	// key to the memory store in a single, lock-protected operation.
+	//
+	// TODO(irbekrm): currently certs for write replicas of HA Ingress get
+	// retrieved from the cluster Secret on each HTTPS request to avoid a
+	// situation when after Ingress recreation stale certs are read from
+	// memory.
+	// Fix this by watching Secrets to ensure that memory store gets updated
+	// when Secrets are deleted.
+	if s.certShareMode == "ro" {
+		s.memory.WriteState(ipn.StateKey(certKey), cert)
+		s.memory.WriteState(ipn.StateKey(keyKey), key)
+	}
 	return cert, key, nil
 }
 

ipn/store/kubestore/store_kube_test.go

@@ -201,10 +201,6 @@ func TestWriteTLSCertAndKey(t *testing.T) {
 				"tls.crt": []byte(testCert),
 				"tls.key": []byte(testKey),
 			},
-			wantMemoryStore: map[ipn.StateKey][]byte{
-				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
-				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
-			},
 		},
 		{
 			name: "cert_share_mode_write_update_existing",
@@ -219,10 +215,6 @@ func TestWriteTLSCertAndKey(t *testing.T) {
 				"tls.crt": []byte(testCert),
 				"tls.key": []byte(testKey),
 			},
-			wantMemoryStore: map[ipn.StateKey][]byte{
-				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
-				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
-			},
 		},
 		{
 			name: "update_existing",
@@ -367,7 +359,7 @@ func TestReadTLSCertAndKey(t *testing.T) {
 		wantMemoryStore map[ipn.StateKey][]byte
 	}{
 		{
-			name: "found",
+			name: "found_in_memory",
 			memoryStore: map[ipn.StateKey][]byte{
 				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
 				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
@@ -381,7 +373,7 @@ func TestReadTLSCertAndKey(t *testing.T) {
 			},
 		},
 		{
-			name:    "not_found",
+			name:    "not_found_in_memory",
 			domain:  testDomain,
 			wantErr: ipn.ErrStateNotExist,
 		},
@@ -400,6 +392,17 @@ func TestReadTLSCertAndKey(t *testing.T) {
 				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
 			},
 		},
+		{
+			name:          "cert_share_rw_mode_found_in_secret",
+			certShareMode: "rw",
+			domain:        testDomain,
+			secretData: map[string][]byte{
+				"tls.crt": []byte(testCert),
+				"tls.key": []byte(testKey),
+			},
+			wantCert: []byte(testCert),
+			wantKey:  []byte(testKey),
+		},
 		{
 			name:          "cert_share_ro_mode_found_in_memory",
 			certShareMode: "ro",

k8s-operator/api.md

@@ -517,6 +517,7 @@ _Appears in:_
 | `statefulSet` _[StatefulSet](#statefulset)_ | Configuration parameters for the proxy's StatefulSet. Tailscale<br />Kubernetes operator deploys a StatefulSet for each of the user<br />configured proxies (Tailscale Ingress, Tailscale Service, Connector). |  |  |
 | `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported<br />for egress proxies and for Ingress proxies that have been configured<br />with tailscale.com/experimental-forward-cluster-traffic-via-ingress<br />annotation. Note that the metrics are currently considered unstable<br />and will likely change in breaking ways in the future - we only<br />recommend that you use those for debugging purposes. |  |  |
 | `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific<br />parameters of proxies. |  |  |
+| `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS<br />certificates for any HTTPS endpoints exposed to the tailnet from<br />LetsEncrypt's staging environment.<br />https://letsencrypt.org/docs/staging-environment/<br />This setting only affects Tailscale Ingress resources.<br />By default Ingress TLS certificates are issued from LetsEncrypt's<br />production environment.<br />Changing this setting true -> false, will result in any<br />existing certs being re-issued from the production environment.<br />Changing this setting false (default) -> true, when certs have already<br />been provisioned from production environment will NOT result in certs<br />being re-issued from the staging environment before they need to be<br />renewed. |  |  |
 
 
 #### ProxyClassStatus
@@ -599,7 +600,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
 | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  | Minimum: 0 <br /> |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -66,6 +66,21 @@ type ProxyClassSpec struct {
 	// parameters of proxies.
 	// +optional
 	TailscaleConfig *TailscaleConfig `json:"tailscale,omitempty"`
+	// Set UseLetsEncryptStagingEnvironment to true to issue TLS
+	// certificates for any HTTPS endpoints exposed to the tailnet from
+	// LetsEncrypt's staging environment.
+	// https://letsencrypt.org/docs/staging-environment/
+	// This setting only affects Tailscale Ingress resources.
+	// By default Ingress TLS certificates are issued from LetsEncrypt's
+	// production environment.
+	// Changing this setting true -> false, will result in any
+	// existing certs being re-issued from the production environment.
+	// Changing this setting false (default) -> true, when certs have already
+	// been provisioned from production environment will NOT result in certs
+	// being re-issued from the staging environment before they need to be
+	// renewed.
+	// +optional
+	UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"`
 }
 
 type TailscaleConfig struct {

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -48,7 +48,7 @@ type ProxyGroupList struct {
 }
 
 type ProxyGroupSpec struct {
-	// Type of the ProxyGroup proxies. Supported types are egress and ingress.
+	// Type of the ProxyGroup proxies. Currently the only supported type is egress.
 	// Type is immutable once a ProxyGroup is created.
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable"
 	Type ProxyGroupType `json:"type"`

net/dns/resolver/forwarder_test.go

@@ -29,6 +29,7 @@ import (
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tstest"
 	"tailscale.com/types/dnstype"
+	"tailscale.com/util/eventbus"
 )
 
 func (rr resolverAndDelay) String() string {
@@ -454,7 +455,9 @@ func makeLargeResponse(tb testing.TB, domain string) (request, response []byte)
 
 func runTestQuery(tb testing.TB, request []byte, modify func(*forwarder), ports ...uint16) ([]byte, error) {
 	logf := tstest.WhileTestRunningLogger(tb)
-	netMon, err := netmon.New(logf)
+	bus := eventbus.New()
+	defer bus.Close()
+	netMon, err := netmon.New(bus, logf)
 	if err != nil {
 		tb.Fatal(err)
 	}

net/dns/resolver/tsdns_test.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/eventbus"
 )
 
 var (
@@ -1059,7 +1060,10 @@ func TestForwardLinkSelection(t *testing.T) {
 	// routes differently.
 	specialIP := netaddr.IPv4(1, 2, 3, 4)
 
-	netMon, err := netmon.New(logger.WithPrefix(t.Logf, ".... netmon: "))
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logger.WithPrefix(t.Logf, ".... netmon: "))
 	if err != nil {
 		t.Fatal(err)
 	}

net/dnsfallback/dnsfallback_test.go

@@ -15,6 +15,7 @@ import (
 	"tailscale.com/net/netmon"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 func TestGetDERPMap(t *testing.T) {
@@ -185,7 +186,10 @@ func TestLookup(t *testing.T) {
 	logf, closeLogf := logger.LogfCloser(t.Logf)
 	defer closeLogf()
 
-	netMon, err := netmon.New(logf)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logf)
 	if err != nil {
 		t.Fatal(err)
 	}

net/netmon/interfaces_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestGetState(t *testing.T) {
-	st, err := GetState()
+	st, err := getState("")
 	if err != nil {
 		t.Fatal(err)
 	}

net/netmon/loghelper_test.go

@@ -7,10 +7,14 @@ import (
 	"bytes"
 	"fmt"
 	"testing"
+
+	"tailscale.com/util/eventbus"
 )
 
 func TestLinkChangeLogLimiter(t *testing.T) {
-	mon, err := New(t.Logf)
+	bus := eventbus.New()
+	defer bus.Close()
+	mon, err := New(bus, t.Logf)
 	if err != nil {
 		t.Fatal(err)
 	}

net/netmon/netmon.go

@@ -16,6 +16,7 @@ import (
 
 	"tailscale.com/types/logger"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/set"
 )
 
@@ -50,7 +51,10 @@ type osMon interface {
 
 // Monitor represents a monitoring instance.
 type Monitor struct {
-	logf   logger.Logf
+	logf    logger.Logf
+	b       *eventbus.Client
+	changed *eventbus.Publisher[*ChangeDelta]
+
 	om     osMon         // nil means not supported on this platform
 	change chan bool     // send false to wake poller, true to also force ChangeDeltas be sent
 	stop   chan struct{} // closed on Stop
@@ -114,21 +118,23 @@ type ChangeDelta struct {
 // New instantiates and starts a monitoring instance.
 // The returned monitor is inactive until it's started by the Start method.
 // Use RegisterChangeCallback to get notified of network changes.
-func New(logf logger.Logf) (*Monitor, error) {
+func New(bus *eventbus.Bus, logf logger.Logf) (*Monitor, error) {
 	logf = logger.WithPrefix(logf, "monitor: ")
 	m := &Monitor{
 		logf:     logf,
+		b:        bus.Client("netmon"),
 		change:   make(chan bool, 1),
 		stop:     make(chan struct{}),
 		lastWall: wallTime(),
 	}
+	m.changed = eventbus.Publish[*ChangeDelta](m.b)
 	st, err := m.interfaceStateUncached()
 	if err != nil {
 		return nil, err
 	}
 	m.ifState = st
 
-	m.om, err = newOSMon(logf, m)
+	m.om, err = newOSMon(bus, logf, m)
 	if err != nil {
 		return nil, err
 	}
@@ -161,7 +167,7 @@ func (m *Monitor) InterfaceState() *State {
 }
 
 func (m *Monitor) interfaceStateUncached() (*State, error) {
-	return GetState()
+	return getState(m.tsIfName)
 }
 
 // SetTailscaleInterfaceName sets the name of the Tailscale interface. For
@@ -465,6 +471,7 @@ func (m *Monitor) handlePotentialChange(newState *State, forceCallbacks bool) {
 	if delta.TimeJumped {
 		metricChangeTimeJump.Add(1)
 	}
+	m.changed.Publish(delta)
 	for _, cb := range m.cbs {
 		go cb(delta)
 	}

net/netmon/netmon_darwin.go

@@ -13,6 +13,7 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/netaddr"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 const debugRouteMessages = false
@@ -24,7 +25,7 @@ type unspecifiedMessage struct{}
 
 func (unspecifiedMessage) ignore() bool { return false }
 
-func newOSMon(logf logger.Logf, _ *Monitor) (osMon, error) {
+func newOSMon(_ *eventbus.Bus, logf logger.Logf, _ *Monitor) (osMon, error) {
 	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, 0)
 	if err != nil {
 		return nil, err

net/netmon/netmon_freebsd.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 // unspecifiedMessage is a minimal message implementation that should not
@@ -24,7 +25,7 @@ type devdConn struct {
 	conn net.Conn
 }
 
-func newOSMon(logf logger.Logf, m *Monitor) (osMon, error) {
+func newOSMon(_ *eventbus.Bus, logf logger.Logf, m *Monitor) (osMon, error) {
 	conn, err := net.Dial("unixpacket", "/var/run/devd.seqpacket.pipe")
 	if err != nil {
 		logf("devd dial error: %v, falling back to polling method", err)

net/netmon/netmon_linux.go

@@ -16,6 +16,7 @@ import (
 	"tailscale.com/envknob"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 var debugNetlinkMessages = envknob.RegisterBool("TS_DEBUG_NETLINK")
@@ -27,15 +28,26 @@ type unspecifiedMessage struct{}
 
 func (unspecifiedMessage) ignore() bool { return false }
 
+// RuleDeleted reports that one of Tailscale's policy routing rules
+// was deleted.
+type RuleDeleted struct {
+	// Table is the table number that the deleted rule referenced.
+	Table uint8
+	// Priority is the lookup priority of the deleted rule.
+	Priority uint32
+}
+
 // nlConn wraps a *netlink.Conn and returns a monitor.Message
 // instead of a netlink.Message. Currently, messages are discarded,
 // but down the line, when messages trigger different logic depending
 // on the type of event, this provides the capability of handling
 // each architecture-specific message in a generic fashion.
 type nlConn struct {
-	logf     logger.Logf
-	conn     *netlink.Conn
-	buffered []netlink.Message
+	busClient    *eventbus.Client
+	rulesDeleted *eventbus.Publisher[RuleDeleted]
+	logf         logger.Logf
+	conn         *netlink.Conn
+	buffered     []netlink.Message
 
 	// addrCache maps interface indices to a set of addresses, and is
 	// used to suppress duplicate RTM_NEWADDR messages. It is populated
@@ -44,7 +56,7 @@ type nlConn struct {
 	addrCache map[uint32]map[netip.Addr]bool
 }
 
-func newOSMon(logf logger.Logf, m *Monitor) (osMon, error) {
+func newOSMon(bus *eventbus.Bus, logf logger.Logf, m *Monitor) (osMon, error) {
 	conn, err := netlink.Dial(unix.NETLINK_ROUTE, &netlink.Config{
 		// Routes get us most of the events of interest, but we need
 		// address as well to cover things like DHCP deciding to give
@@ -59,12 +71,22 @@ func newOSMon(logf logger.Logf, m *Monitor) (osMon, error) {
 		logf("monitor_linux: AF_NETLINK RTMGRP failed, falling back to polling")
 		return newPollingMon(logf, m)
 	}
-	return &nlConn{logf: logf, conn: conn, addrCache: make(map[uint32]map[netip.Addr]bool)}, nil
+	client := bus.Client("netmon-iprules")
+	return &nlConn{
+		busClient:    client,
+		rulesDeleted: eventbus.Publish[RuleDeleted](client),
+		logf:         logf,
+		conn:         conn,
+		addrCache:    make(map[uint32]map[netip.Addr]bool),
+	}, nil
 }
 
 func (c *nlConn) IsInterestingInterface(iface string) bool { return true }
 
-func (c *nlConn) Close() error { return c.conn.Close() }
+func (c *nlConn) Close() error {
+	c.busClient.Close()
+	return c.conn.Close()
+}
 
 func (c *nlConn) Receive() (message, error) {
 	if len(c.buffered) == 0 {
@@ -219,6 +241,10 @@ func (c *nlConn) Receive() (message, error) {
 			// On `ip -4 rule del pref 5210 table main`, logs:
 			// monitor: ip rule deleted: {Family:2 DstLength:0 SrcLength:0 Tos:0 Table:254 Protocol:0 Scope:0 Type:1 Flags:0 Attributes:{Dst:<nil> Src:<nil> Gateway:<nil> OutIface:0 Priority:5210 Table:254 Mark:4294967295 Expires:<nil> Metrics:<nil> Multipath:[]}}
 		}
+		c.rulesDeleted.Publish(RuleDeleted{
+			Table:    rmsg.Table,
+			Priority: rmsg.Attributes.Priority,
+		})
 		rdm := ipRuleDeletedMessage{
 			table:    rmsg.Table,
 			priority: rmsg.Attributes.Priority,

net/netmon/netmon_polling.go

@@ -7,9 +7,10 @@ package netmon
 
 import (
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
-func newOSMon(logf logger.Logf, m *Monitor) (osMon, error) {
+func newOSMon(_ *eventbus.Bus, logf logger.Logf, m *Monitor) (osMon, error) {
 	return newPollingMon(logf, m)
 }
 

net/netmon/netmon_test.go

@@ -11,11 +11,15 @@ import (
 	"testing"
 	"time"
 
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/mak"
 )
 
 func TestMonitorStartClose(t *testing.T) {
-	mon, err := New(t.Logf)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	mon, err := New(bus, t.Logf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -26,7 +30,10 @@ func TestMonitorStartClose(t *testing.T) {
 }
 
 func TestMonitorJustClose(t *testing.T) {
-	mon, err := New(t.Logf)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	mon, err := New(bus, t.Logf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -36,7 +43,10 @@ func TestMonitorJustClose(t *testing.T) {
 }
 
 func TestMonitorInjectEvent(t *testing.T) {
-	mon, err := New(t.Logf)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	mon, err := New(bus, t.Logf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -71,7 +81,11 @@ func TestMonitorMode(t *testing.T) {
 	default:
 		t.Skipf(`invalid --monitor value: must be "raw" or "callback"`)
 	}
-	mon, err := New(t.Logf)
+
+	bus := eventbus.New()
+	defer bus.Close()
+
+	mon, err := New(bus, t.Logf)
 	if err != nil {
 		t.Fatal(err)
 	}

net/netmon/netmon_windows.go

@@ -13,6 +13,7 @@ import (
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 var (
@@ -45,7 +46,7 @@ type winMon struct {
 	noDeadlockTicker *time.Ticker
 }
 
-func newOSMon(logf logger.Logf, pm *Monitor) (osMon, error) {
+func newOSMon(_ *eventbus.Bus, logf logger.Logf, pm *Monitor) (osMon, error) {
 	m := &winMon{
 		logf:             logf,
 		isActive:         pm.isActive,

net/netmon/state.go

@@ -461,21 +461,22 @@ func isTailscaleInterface(name string, ips []netip.Prefix) bool {
 // getPAC, if non-nil, returns the current PAC file URL.
 var getPAC func() string
 
-// GetState returns the state of all the current machine's network interfaces.
+// getState returns the state of all the current machine's network interfaces.
 //
 // It does not set the returned State.IsExpensive. The caller can populate that.
 //
-// Deprecated: use netmon.Monitor.InterfaceState instead.
-func GetState() (*State, error) {
+// optTSInterfaceName is the name of the Tailscale interface, if known.
+func getState(optTSInterfaceName string) (*State, error) {
 	s := &State{
 		InterfaceIPs: make(map[string][]netip.Prefix),
 		Interface:    make(map[string]Interface),
 	}
 	if err := ForeachInterface(func(ni Interface, pfxs []netip.Prefix) {
+		isTSInterfaceName := optTSInterfaceName != "" && ni.Name == optTSInterfaceName
 		ifUp := ni.IsUp()
 		s.Interface[ni.Name] = ni
 		s.InterfaceIPs[ni.Name] = append(s.InterfaceIPs[ni.Name], pfxs...)
-		if !ifUp || isTailscaleInterface(ni.Name, pfxs) {
+		if !ifUp || isTSInterfaceName || isTailscaleInterface(ni.Name, pfxs) {
 			return
 		}
 		for _, pfx := range pfxs {
@@ -755,11 +756,12 @@ func DefaultRoute() (DefaultRouteDetails, error) {
 
 // HasCGNATInterface reports whether there are any non-Tailscale interfaces that
 // use a CGNAT IP range.
-func HasCGNATInterface() (bool, error) {
+func (m *Monitor) HasCGNATInterface() (bool, error) {
 	hasCGNATInterface := false
 	cgnatRange := tsaddr.CGNATRange()
 	err := ForeachInterface(func(i Interface, pfxs []netip.Prefix) {
-		if hasCGNATInterface || !i.IsUp() || isTailscaleInterface(i.Name, pfxs) {
+		isTSInterfaceName := m.tsIfName != "" && i.Name == m.tsIfName
+		if hasCGNATInterface || !i.IsUp() || isTSInterfaceName || isTailscaleInterface(i.Name, pfxs) {
 			return
 		}
 		for _, pfx := range pfxs {

net/netutil/netutil_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"tailscale.com/net/netmon"
+	"tailscale.com/util/eventbus"
 )
 
 type conn struct {
@@ -72,7 +73,10 @@ func TestCheckReversePathFiltering(t *testing.T) {
 	if runtime.GOOS != "linux" {
 		t.Skipf("skipping on %s", runtime.GOOS)
 	}
-	netMon, err := netmon.New(t.Logf)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, t.Logf)
 	if err != nil {
 		t.Fatal(err)
 	}

net/portmapper/igd_test.go

@@ -19,6 +19,7 @@ import (
 	"tailscale.com/net/netmon"
 	"tailscale.com/syncs"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 )
 
 // TestIGD is an IGD (Internet Gateway Device) for testing. It supports fake
@@ -258,15 +259,25 @@ func (d *TestIGD) handlePCPQuery(pkt []byte, src netip.AddrPort) {
 	}
 }
 
-func newTestClient(t *testing.T, igd *TestIGD) *Client {
+// newTestClient configures a new test client connected to igd for mapping updates.
+// If bus != nil, update events are published to it.
+// A cleanup for the resulting client is added to t.
+func newTestClient(t *testing.T, igd *TestIGD, bus *eventbus.Bus) *Client {
 	var c *Client
-	c = NewClient(t.Logf, netmon.NewStatic(), nil, new(controlknobs.Knobs), func() {
-		t.Logf("port map changed")
-		t.Logf("have mapping: %v", c.HaveMapping())
+	c = NewClient(Config{
+		Logf:         t.Logf,
+		NetMon:       netmon.NewStatic(),
+		ControlKnobs: new(controlknobs.Knobs),
+		EventBus:     bus,
+		OnChange: func() {
+			t.Logf("port map changed")
+			t.Logf("have mapping: %v", c.HaveMapping())
+		},
 	})
 	c.testPxPPort = igd.TestPxPPort()
 	c.testUPnPPort = igd.TestUPnPPort()
 	c.netMon = netmon.NewStatic()
 	c.SetGatewayLookupFunc(testIPAndGateway)
+	t.Cleanup(func() { c.Close() })
 	return c
 }

net/portmapper/portmapper.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/nettype"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/eventbus"
 )
 
 var disablePortMapperEnv = envknob.RegisterBool("TS_DISABLE_PORTMAPPER")
@@ -84,6 +85,11 @@ const trustServiceStillAvailableDuration = 10 * time.Minute
 
 // Client is a port mapping client.
 type Client struct {
+	// The following two fields must either both be nil, or both non-nil.
+	// Both are immutable after construction.
+	pubClient *eventbus.Client
+	updates   *eventbus.Publisher[Mapping]
+
 	logf         logger.Logf
 	netMon       *netmon.Monitor // optional; nil means interfaces will be looked up on-demand
 	controlKnobs *controlknobs.Knobs
@@ -201,32 +207,56 @@ func (m *pmpMapping) Release(ctx context.Context) {
 	uc.WriteToUDPAddrPort(pkt, m.gw)
 }
 
-// NewClient returns a new portmapping client.
-//
-// The netMon parameter is required.
-//
-// The debug argument allows configuring the behaviour of the portmapper for
-// debugging; if nil, a sensible set of defaults will be used.
-//
-// The controlKnobs, if non-nil, specifies the control knobs from the control
-// plane that might disable portmapping.
-//
-// The optional onChange argument specifies a func to run in a new goroutine
-// whenever the port mapping status has changed. If nil, it doesn't make a
-// callback.
-func NewClient(logf logger.Logf, netMon *netmon.Monitor, debug *DebugKnobs, controlKnobs *controlknobs.Knobs, onChange func()) *Client {
-	if netMon == nil {
+// Config carries the settings for a [Client].
+type Config struct {
+	// EventBus, if non-nil, is used for event publication and subscription by
+	// portmapper clients created from this config.
+	//
+	// TODO(creachadair): As of 2025-03-19 this is optional, but is intended to
+	// become required non-nil.
+	EventBus *eventbus.Bus
+
+	// Logf is called to generate text logs for the client. If nil, logger.Discard is used.
+	Logf logger.Logf
+
+	// NetMon is the network monitor used by the client. It must be non-nil.
+	NetMon *netmon.Monitor
+
+	// DebugKnobs, if non-nil, configure the behaviour of the portmapper for
+	// debugging.  If nil, a sensible set of defaults will be used.
+	DebugKnobs *DebugKnobs
+
+	// ControlKnobs, if non-nil, specifies knobs from the control plane that
+	// might disable port mapping.
+	ControlKnobs *controlknobs.Knobs
+
+	// OnChange is called to run in a new goroutine whenever the port mapping
+	// status has changed. If nil, no callback is issued.
+	OnChange func()
+}
+
+// NewClient constructs a new portmapping [Client] from c. It will panic if any
+// required parameters are omitted.
+func NewClient(c Config) *Client {
+	if c.NetMon == nil {
 		panic("nil netMon")
 	}
 	ret := &Client{
-		logf:         logf,
-		netMon:       netMon,
+		logf:         c.Logf,
+		netMon:       c.NetMon,
 		ipAndGateway: netmon.LikelyHomeRouterIP, // TODO(bradfitz): move this to method on netMon
-		onChange:     onChange,
-		controlKnobs: controlKnobs,
+		onChange:     c.OnChange,
+		controlKnobs: c.ControlKnobs,
 	}
-	if debug != nil {
-		ret.debug = *debug
+	if c.EventBus != nil {
+		ret.pubClient = c.EventBus.Client("portmapper")
+		ret.updates = eventbus.Publish[Mapping](ret.pubClient)
+	}
+	if ret.logf == nil {
+		ret.logf = logger.Discard
+	}
+	if c.DebugKnobs != nil {
+		ret.debug = *c.DebugKnobs
 	}
 	return ret
 }
@@ -256,6 +286,10 @@ func (c *Client) Close() error {
 	}
 	c.closed = true
 	c.invalidateMappingsLocked(true)
+	if c.updates != nil {
+		c.updates.Close()
+		c.pubClient.Close()
+	}
 	// TODO: close some future ever-listening UDP socket(s),
 	// waiting for multicast announcements from router.
 	return nil
@@ -467,11 +501,30 @@ func (c *Client) createMapping() {
 		c.runningCreate = false
 	}()
 
-	if _, err := c.createOrGetMapping(ctx); err == nil && c.onChange != nil {
-		go c.onChange()
-	} else if err != nil && !IsNoMappingError(err) {
-		c.logf("createOrGetMapping: %v", err)
+	mapping, _, err := c.createOrGetMapping(ctx)
+	if err != nil {
+		if !IsNoMappingError(err) {
+			c.logf("createOrGetMapping: %v", err)
+		}
+		return
 	}
+	c.updates.Publish(Mapping{
+		External:  mapping.External(),
+		Type:      mapping.MappingType(),
+		GoodUntil: mapping.GoodUntil(),
+	})
+	if c.onChange != nil {
+		go c.onChange()
+	}
+}
+
+// Mapping is an event recording the allocation of a port mapping.
+type Mapping struct {
+	External  netip.AddrPort
+	Type      string
+	GoodUntil time.Time
+
+	// TODO(creachadair): Record whether we reused an existing mapping?
 }
 
 // wildcardIP is used when the previous external IP is not known for PCP port mapping.
@@ -482,19 +535,19 @@ var wildcardIP = netip.MustParseAddr("0.0.0.0")
 //
 // If no mapping is available, the error will be of type
 // NoMappingError; see IsNoMappingError.
-func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPort, err error) {
+func (c *Client) createOrGetMapping(ctx context.Context) (mapping mapping, external netip.AddrPort, err error) {
 	if c.debug.disableAll() {
-		return netip.AddrPort{}, NoMappingError{ErrPortMappingDisabled}
+		return nil, netip.AddrPort{}, NoMappingError{ErrPortMappingDisabled}
 	}
 	if c.debug.DisableUPnP && c.debug.DisablePCP && c.debug.DisablePMP {
-		return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
+		return nil, netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
 	}
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return netip.AddrPort{}, NoMappingError{ErrGatewayRange}
+		return nil, netip.AddrPort{}, NoMappingError{ErrGatewayRange}
 	}
 	if gw.Is6() {
-		return netip.AddrPort{}, NoMappingError{ErrGatewayIPv6}
+		return nil, netip.AddrPort{}, NoMappingError{ErrGatewayIPv6}
 	}
 
 	now := time.Now()
@@ -523,6 +576,17 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 			return
 		}
 
+		// TODO(creachadair): This is more subtle than it should be. Ideally we
+		// would just return the mapping directly, but there are many different
+		// paths through the function with carefully-balanced locks, and not all
+		// the paths have a mapping to return. As a workaround, while we're here
+		// doing cleanup under the lock, grab the final mapping value and return
+		// it, so the caller does not need to grab the lock again and potentially
+		// race with a later update. The mapping itself is concurrency-safe.
+		//
+		// We should restructure this code so the locks are properly scoped.
+		mapping = c.mapping
+
 		// Print the internal details of each mapping if we're being verbose.
 		if c.debug.VerboseLogs {
 			c.logf("successfully obtained mapping: now=%d external=%v type=%s mapping=%s",
@@ -548,7 +612,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 		if now.Before(m.RenewAfter()) {
 			defer c.mu.Unlock()
 			reusedExisting = true
-			return m.External(), nil
+			return nil, m.External(), nil
 		}
 		// The mapping might still be valid, so just try to renew it.
 		prevPort = m.External().Port()
@@ -557,10 +621,10 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 	if c.debug.DisablePCP && c.debug.DisablePMP {
 		c.mu.Unlock()
 		if external, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
-			return external, nil
+			return nil, external, nil
 		}
 		c.vlogf("fallback to UPnP due to PCP and PMP being disabled failed")
-		return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
+		return nil, netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
 	}
 
 	// If we just did a Probe (e.g. via netchecker) but didn't
@@ -587,16 +651,16 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 		c.mu.Unlock()
 		// fallback to UPnP portmapping
 		if external, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
-			return external, nil
+			return nil, external, nil
 		}
 		c.vlogf("fallback to UPnP due to no PCP and PMP failed")
-		return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
+		return nil, netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
 	}
 	c.mu.Unlock()
 
 	uc, err := c.listenPacket(ctx, "udp4", ":0")
 	if err != nil {
-		return netip.AddrPort{}, err
+		return nil, netip.AddrPort{}, err
 	}
 	defer uc.Close()
 
@@ -616,7 +680,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 			if neterror.TreatAsLostUDP(err) {
 				err = NoMappingError{ErrNoPortMappingServices}
 			}
-			return netip.AddrPort{}, err
+			return nil, netip.AddrPort{}, err
 		}
 	} else {
 		// Ask for our external address if needed.
@@ -625,7 +689,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 				if neterror.TreatAsLostUDP(err) {
 					err = NoMappingError{ErrNoPortMappingServices}
 				}
-				return netip.AddrPort{}, err
+				return nil, netip.AddrPort{}, err
 			}
 		}
 
@@ -634,7 +698,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 			if neterror.TreatAsLostUDP(err) {
 				err = NoMappingError{ErrNoPortMappingServices}
 			}
-			return netip.AddrPort{}, err
+			return nil, netip.AddrPort{}, err
 		}
 	}
 
@@ -643,13 +707,13 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 		n, src, err := uc.ReadFromUDPAddrPort(res)
 		if err != nil {
 			if ctx.Err() == context.Canceled {
-				return netip.AddrPort{}, err
+				return nil, netip.AddrPort{}, err
 			}
 			// fallback to UPnP portmapping
 			if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
-				return mapping, nil
+				return nil, mapping, nil
 			}
-			return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
+			return nil, netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
 		}
 		src = netaddr.Unmap(src)
 		if !src.IsValid() {
@@ -665,7 +729,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 					continue
 				}
 				if pres.ResultCode != 0 {
-					return netip.AddrPort{}, NoMappingError{fmt.Errorf("PMP response Op=0x%x,Res=0x%x", pres.OpCode, pres.ResultCode)}
+					return nil, netip.AddrPort{}, NoMappingError{fmt.Errorf("PMP response Op=0x%x,Res=0x%x", pres.OpCode, pres.ResultCode)}
 				}
 				if pres.OpCode == pmpOpReply|pmpOpMapPublicAddr {
 					m.external = netip.AddrPortFrom(pres.PublicAddr, m.external.Port())
@@ -683,7 +747,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 				if err != nil {
 					c.logf("failed to get PCP mapping: %v", err)
 					// PCP should only have a single packet response
-					return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
+					return nil, netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
 				}
 				pcpMapping.c = c
 				pcpMapping.internal = m.internal
@@ -691,10 +755,10 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 				c.mu.Lock()
 				defer c.mu.Unlock()
 				c.mapping = pcpMapping
-				return pcpMapping.external, nil
+				return pcpMapping, pcpMapping.external, nil
 			default:
 				c.logf("unknown PMP/PCP version number: %d %v", version, res[:n])
-				return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
+				return nil, netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
 			}
 		}
 
@@ -702,7 +766,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
 			c.mu.Lock()
 			defer c.mu.Unlock()
 			c.mapping = m
-			return m.external, nil
+			return nil, m.external, nil
 		}
 	}
 }

net/portmapper/portmapper_test.go

@@ -12,20 +12,21 @@ import (
 	"time"
 
 	"tailscale.com/control/controlknobs"
+	"tailscale.com/util/eventbus"
 )
 
 func TestCreateOrGetMapping(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf, nil, nil, new(controlknobs.Knobs), nil)
+	c := NewClient(Config{Logf: t.Logf, ControlKnobs: new(controlknobs.Knobs)})
 	defer c.Close()
 	c.SetLocalPort(1234)
 	for i := range 2 {
 		if i > 0 {
 			time.Sleep(100 * time.Millisecond)
 		}
-		ext, err := c.createOrGetMapping(context.Background())
+		_, ext, err := c.createOrGetMapping(context.Background())
 		t.Logf("Got: %v, %v", ext, err)
 	}
 }
@@ -34,7 +35,7 @@ func TestClientProbe(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf, nil, nil, new(controlknobs.Knobs), nil)
+	c := NewClient(Config{Logf: t.Logf, ControlKnobs: new(controlknobs.Knobs)})
 	defer c.Close()
 	for i := range 3 {
 		if i > 0 {
@@ -49,13 +50,13 @@ func TestClientProbeThenMap(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf, nil, nil, new(controlknobs.Knobs), nil)
+	c := NewClient(Config{Logf: t.Logf, ControlKnobs: new(controlknobs.Knobs)})
 	defer c.Close()
 	c.debug.VerboseLogs = true
 	c.SetLocalPort(1234)
 	res, err := c.Probe(context.Background())
 	t.Logf("Probe: %+v, %v", res, err)
-	ext, err := c.createOrGetMapping(context.Background())
+	_, ext, err := c.createOrGetMapping(context.Background())
 	t.Logf("createOrGetMapping: %v, %v", ext, err)
 }
 
@@ -66,9 +67,8 @@ func TestProbeIntegration(t *testing.T) {
 	}
 	defer igd.Close()
 
-	c := newTestClient(t, igd)
+	c := newTestClient(t, igd, nil)
 	t.Logf("Listening on pxp=%v, upnp=%v", c.testPxPPort, c.testUPnPPort)
-	defer c.Close()
 
 	res, err := c.Probe(context.Background())
 	if err != nil {
@@ -101,8 +101,7 @@ func TestPCPIntegration(t *testing.T) {
 	}
 	defer igd.Close()
 
-	c := newTestClient(t, igd)
-	defer c.Close()
+	c := newTestClient(t, igd, nil)
 	res, err := c.Probe(context.Background())
 	if err != nil {
 		t.Fatalf("probe failed: %v", err)
@@ -114,7 +113,7 @@ func TestPCPIntegration(t *testing.T) {
 		t.Fatalf("probe did not see pcp: %+v", res)
 	}
 
-	external, err := c.createOrGetMapping(context.Background())
+	_, external, err := c.createOrGetMapping(context.Background())
 	if err != nil {
 		t.Fatalf("failed to get mapping: %v", err)
 	}
@@ -136,3 +135,29 @@ func TestGetUPnPErrorsMetric(t *testing.T) {
 	getUPnPErrorsMetric(0)
 	getUPnPErrorsMetric(-100)
 }
+
+func TestUpdateEvent(t *testing.T) {
+	igd, err := NewTestIGD(t.Logf, TestIGDOptions{PCP: true})
+	if err != nil {
+		t.Fatalf("Create test gateway: %v", err)
+	}
+
+	bus := eventbus.New()
+	defer bus.Close()
+
+	sub := eventbus.Subscribe[Mapping](bus.Client("TestUpdateEvent"))
+	c := newTestClient(t, igd, bus)
+	if _, err := c.Probe(t.Context()); err != nil {
+		t.Fatalf("Probe failed: %v", err)
+	}
+	c.GetCachedMappingOrStartCreatingOne()
+
+	select {
+	case evt := <-sub.Events():
+		t.Logf("Received portmap update: %+v", evt)
+	case <-sub.Done():
+		t.Error("Subscriber closed prematurely")
+	case <-time.After(5 * time.Second):
+		t.Error("Timed out waiting for an update event")
+	}
+}

net/portmapper/select_test.go

@@ -163,9 +163,8 @@ func TestSelectBestService(t *testing.T) {
 				Desc:    rootDesc,
 				Control: tt.control,
 			})
-			c := newTestClient(t, igd)
+			c := newTestClient(t, igd, nil)
 			t.Logf("Listening on upnp=%v", c.testUPnPPort)
-			defer c.Close()
 
 			// Ensure that we're using the HTTP client that talks to our test IGD server
 			ctx := context.Background()

net/portmapper/upnp_test.go

@@ -586,9 +586,8 @@ func TestGetUPnPPortMapping(t *testing.T) {
 			},
 		})
 
-		c := newTestClient(t, igd)
+		c := newTestClient(t, igd, nil)
 		t.Logf("Listening on upnp=%v", c.testUPnPPort)
-		defer c.Close()
 
 		c.debug.VerboseLogs = true
 
@@ -689,10 +688,9 @@ func TestGetUPnPPortMapping_LeaseDuration(t *testing.T) {
 			})
 
 			ctx := context.Background()
-			c := newTestClient(t, igd)
+			c := newTestClient(t, igd, nil)
 			c.debug.VerboseLogs = true
 			t.Logf("Listening on upnp=%v", c.testUPnPPort)
-			defer c.Close()
 
 			// Actually test the UPnP port mapping.
 			mustProbeUPnP(t, ctx, c)
@@ -735,8 +733,7 @@ func TestGetUPnPPortMapping_NoValidServices(t *testing.T) {
 		Desc: noSupportedServicesRootDesc,
 	})
 
-	c := newTestClient(t, igd)
-	defer c.Close()
+	c := newTestClient(t, igd, nil)
 	c.debug.VerboseLogs = true
 
 	ctx := context.Background()
@@ -778,8 +775,7 @@ func TestGetUPnPPortMapping_Legacy(t *testing.T) {
 		},
 	})
 
-	c := newTestClient(t, igd)
-	defer c.Close()
+	c := newTestClient(t, igd, nil)
 	c.debug.VerboseLogs = true
 
 	ctx := context.Background()
@@ -806,9 +802,8 @@ func TestGetUPnPPortMappingNoResponses(t *testing.T) {
 	}
 	defer igd.Close()
 
-	c := newTestClient(t, igd)
+	c := newTestClient(t, igd, nil)
 	t.Logf("Listening on upnp=%v", c.testUPnPPort)
-	defer c.Close()
 
 	c.debug.VerboseLogs = true
 
@@ -939,8 +934,7 @@ func TestGetUPnPPortMapping_Invalid(t *testing.T) {
 				},
 			})
 
-			c := newTestClient(t, igd)
-			defer c.Close()
+			c := newTestClient(t, igd, nil)
 			c.debug.VerboseLogs = true
 
 			ctx := context.Background()

pkgdoc_test.go

@@ -26,6 +26,9 @@ func TestPackageDocs(t *testing.T) {
 		if err != nil {
 			return err
 		}
+		if fi.Mode().IsDir() && path == ".git" {
+			return filepath.SkipDir // No documentation lives in .git
+		}
 		if fi.Mode().IsRegular() && strings.HasSuffix(path, ".go") {
 			if strings.HasSuffix(path, "_test.go") {
 				return nil

prober/derp.go

@@ -596,11 +596,23 @@ func (d *derpProber) updateMap(ctx context.Context) error {
 }
 
 func (d *derpProber) ProbeUDP(ipaddr string, port int) ProbeClass {
+	initLabels := make(Labels)
+	ip := net.ParseIP(ipaddr)
+
+	if ip.To4() != nil {
+		initLabels["address_family"] = "ipv4"
+	} else if ip.To16() != nil { // Will return an IPv4 as 16 byte, so ensure the check for IPv4 precedes this
+		initLabels["address_family"] = "ipv6"
+	} else {
+		initLabels["address_family"] = "unknown"
+	}
+
 	return ProbeClass{
 		Probe: func(ctx context.Context) error {
 			return derpProbeUDP(ctx, ipaddr, port)
 		},
-		Class: "derp_udp",
+		Class:  "derp_udp",
+		Labels: initLabels,
 	}
 }
 

prober/prober.go

@@ -404,10 +404,14 @@ func (p *Probe) recordEndLocked(err error) {
 		p.mSeconds.WithLabelValues("ok").Add(latency.Seconds())
 		p.latencyHist.Value = latency
 		p.latencyHist = p.latencyHist.Next()
+		p.mAttempts.WithLabelValues("fail").Add(0)
+		p.mSeconds.WithLabelValues("fail").Add(0)
 	} else {
 		p.latency = 0
 		p.mAttempts.WithLabelValues("fail").Inc()
 		p.mSeconds.WithLabelValues("fail").Add(latency.Seconds())
+		p.mAttempts.WithLabelValues("ok").Add(0)
+		p.mSeconds.WithLabelValues("ok").Add(0)
 	}
 	p.successHist.Value = p.succeeded
 	p.successHist = p.successHist.Next()

safesocket/safesocket.go

@@ -61,7 +61,11 @@ func ConnectContext(ctx context.Context, path string) (net.Conn, error) {
 			if ctx.Err() != nil {
 				return nil, ctx.Err()
 			}
-			time.Sleep(250 * time.Millisecond)
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(250 * time.Millisecond):
+			}
 			continue
 		}
 		return c, err

ssh/tailssh/tailssh_test.go

@@ -1037,8 +1037,8 @@ func TestSSHAuthFlow(t *testing.T) {
 
 func TestSSH(t *testing.T) {
 	var logf logger.Logf = t.Logf
-	sys := &tsd.System{}
-	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry())
+	sys := tsd.NewSystem()
+	eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker(), sys.UserMetricsRegistry(), sys.Bus.Get())
 	if err != nil {
 		t.Fatal(err)
 	}

tsd/tsd.go

@@ -34,6 +34,7 @@ import (
 	"tailscale.com/net/tstun"
 	"tailscale.com/proxymap"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/usermetric"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/magicsock"
@@ -41,7 +42,12 @@ import (
 )
 
 // System contains all the subsystems of a Tailscale node (tailscaled, etc.)
+//
+// A valid System value must always have a non-nil Bus populated.  Callers must
+// ensure this before using the value further. Call [NewSystem] to obtain a
+// value ready to use.
 type System struct {
+	Bus            SubSystem[*eventbus.Bus]
 	Dialer         SubSystem[*tsdial.Dialer]
 	DNSManager     SubSystem[*dns.Manager] // can get its *resolver.Resolver from DNSManager.Resolver
 	Engine         SubSystem[wgengine.Engine]
@@ -74,6 +80,14 @@ type System struct {
 	userMetricsRegistry usermetric.Registry
 }
 
+// NewSystem constructs a new otherwise-empty [System] with a
+// freshly-constructed event bus populated.
+func NewSystem() *System {
+	sys := new(System)
+	sys.Set(eventbus.New())
+	return sys
+}
+
 // NetstackImpl is the interface that *netstack.Impl implements.
 // It's an interface for circular dependency reasons: netstack.Impl
 // references LocalBackend, and LocalBackend has a tsd.System.
@@ -86,6 +100,8 @@ type NetstackImpl interface {
 // has already been set.
 func (s *System) Set(v any) {
 	switch v := v.(type) {
+	case *eventbus.Bus:
+		s.Bus.Set(v)
 	case *netmon.Monitor:
 		s.NetMon.Set(v)
 	case *dns.Manager:

tsnet/tsnet.go

@@ -435,8 +435,11 @@ func (s *Server) Close() error {
 	for _, ln := range s.listeners {
 		ln.closeLocked()
 	}
-
 	wg.Wait()
+
+	if bus := s.sys.Bus.Get(); bus != nil {
+		bus.Close()
+	}
 	s.closed = true
 	return nil
 }
@@ -505,6 +508,11 @@ func (s *Server) start() (reterr error) {
 			// directory and hostname when they're not supplied. But we can fall
 			// back to "tsnet" as well.
 			exe = "tsnet"
+		case "ios":
+			// When compiled as a framework (via TailscaleKit in libtailscale),
+			// os.Executable() returns an error, so fall back to "tsnet" there
+			// too.
+			exe = "tsnet"
 		default:
 			return err
 		}
@@ -553,13 +561,13 @@ func (s *Server) start() (reterr error) {
 		s.Logf(format, a...)
 	}
 
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	s.sys = sys
 	if err := s.startLogger(&closePool, sys.HealthTracker(), tsLogf); err != nil {
 		return err
 	}
 
-	s.netMon, err = netmon.New(tsLogf)
+	s.netMon, err = netmon.New(sys.Bus.Get(), tsLogf)
 	if err != nil {
 		return err
 	}
@@ -567,6 +575,7 @@ func (s *Server) start() (reterr error) {
 
 	s.dialer = &tsdial.Dialer{Logf: tsLogf} // mutated below (before used)
 	eng, err := wgengine.NewUserspaceEngine(tsLogf, wgengine.Config{
+		EventBus:      sys.Bus.Get(),
 		ListenPort:    s.Port,
 		NetMon:        s.netMon,
 		Dialer:        s.dialer,

tsnet/tsnet_test.go

@@ -120,6 +120,7 @@ func startControl(t *testing.T) (controlURL string, control *testcontrol.Server)
 			Proxied: true,
 		},
 		MagicDNSDomain: "tail-scale.ts.net",
+		Logf:           t.Logf,
 	}
 	control.HTTPTestServer = httptest.NewUnstartedServer(control)
 	control.HTTPTestServer.Start()
@@ -221,7 +222,7 @@ func startServer(t *testing.T, ctx context.Context, controlURL, hostname string)
 		getCertForTesting: testCertRoot.getCert,
 	}
 	if *verboseNodes {
-		s.Logf = log.Printf
+		s.Logf = t.Logf
 	}
 	t.Cleanup(func() { s.Close() })
 

tstest/integration/integration_test.go

@@ -1942,6 +1942,8 @@ func (n *testNode) AwaitIP6() netip.Addr {
 
 // AwaitRunning waits for n to reach the IPN state "Running".
 func (n *testNode) AwaitRunning() {
+	t := n.env.t
+	t.Helper()
 	n.AwaitBackendState("Running")
 }
 
@@ -2015,7 +2017,7 @@ func (n *testNode) Status() (*ipnstate.Status, error) {
 	}
 	st := new(ipnstate.Status)
 	if err := json.Unmarshal(out, st); err != nil {
-		return nil, fmt.Errorf("decoding tailscale status JSON: %w", err)
+		return nil, fmt.Errorf("decoding tailscale status JSON: %w\njson:\n%s", err, out)
 	}
 	return st, nil
 }

tstest/integration/tailscaled_deps_test_darwin.go

@@ -48,6 +48,7 @@ import (
 	_ "tailscale.com/types/logger"
 	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/eventbus"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -48,6 +48,7 @@ import (
 	_ "tailscale.com/types/logger"
 	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/eventbus"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"

tstest/integration/tailscaled_deps_test_linux.go

@@ -48,6 +48,7 @@ import (
 	_ "tailscale.com/types/logger"
 	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/eventbus"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -48,6 +48,7 @@ import (
 	_ "tailscale.com/types/logger"
 	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/eventbus"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"

tstest/integration/tailscaled_deps_test_windows.go

@@ -56,6 +56,7 @@ import (
 	_ "tailscale.com/types/logger"
 	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/eventbus"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osdiag"
 	_ "tailscale.com/util/osshare"

tstest/integration/testcontrol/testcontrol.go

@@ -839,15 +839,17 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey key.Machi
 
 	w.WriteHeader(200)
 	for {
-		if resBytes, ok := s.takeRawMapMessage(req.NodeKey); ok {
-			if err := s.sendMapMsg(w, compress, resBytes); err != nil {
-				s.logf("sendMapMsg of raw message: %v", err)
-				return
-			}
-			if streaming {
+		// Only send raw map responses to the streaming poll, to avoid a
+		// non-streaming map request beating the streaming poll in a race and
+		// potentially dropping the map response.
+		if streaming {
+			if resBytes, ok := s.takeRawMapMessage(req.NodeKey); ok {
+				if err := s.sendMapMsg(w, compress, resBytes); err != nil {
+					s.logf("sendMapMsg of raw message: %v", err)
+					return
+				}
 				continue
 			}
-			return
 		}
 
 		if s.canGenerateAutomaticMapResponseFor(req.NodeKey) {

tstest/mts/mts.go

@@ -0,0 +1,599 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux || darwin
+
+// The mts ("Multiple Tailscale") command runs multiple tailscaled instances for
+// development, managing their directories and sockets, and lets you easily direct
+// tailscale CLI commands to them.
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"maps"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"tailscale.com/client/local"
+	"tailscale.com/types/bools"
+	"tailscale.com/types/lazy"
+	"tailscale.com/util/mak"
+)
+
+func usage(args ...any) {
+	var format string
+	if len(args) > 0 {
+		format, args = args[0].(string), args[1:]
+	}
+	if format != "" {
+		format = strings.TrimSpace(format) + "\n\n"
+		fmt.Fprintf(os.Stderr, format, args...)
+	}
+	io.WriteString(os.Stderr, strings.TrimSpace(`
+usage:
+
+   mts server <subcommand>      # manage tailscaled instances
+   mts server run               # run the mts server (parent process of all tailscaled)
+   mts server list              # list all tailscaled and their state
+   mts server list <name>       # show details of named instance
+   mts server add <name>        # add+start new named tailscaled
+   mts server start <name>      # start a previously added tailscaled
+   mts server stop <name>       # stop & remove a named tailscaled
+   mts server rm <name>         # stop & remove a named tailscaled
+   mts server logs [-f] <name>  # get/follow tailscaled logs
+
+  mts <inst-name> [tailscale CLI args] # run Tailscale CLI against a named instance
+    e.g.
+      mts gmail1 up
+      mts github2 status --json
+	`)+"\n")
+	os.Exit(1)
+}
+
+func main() {
+	// Don't use flag.Parse here; we mostly just delegate through
+	// to the Tailscale CLI.
+
+	if len(os.Args) < 2 {
+		usage()
+	}
+	firstArg, args := os.Args[1], os.Args[2:]
+	if firstArg == "server" || firstArg == "s" {
+		if err := runMTSServer(args); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		var c Client
+		inst := firstArg
+		c.RunCommand(inst, args)
+	}
+}
+
+func runMTSServer(args []string) error {
+	if len(args) == 0 {
+		usage()
+	}
+	cmd, args := args[0], args[1:]
+	if cmd == "run" {
+		var s Server
+		return s.Run()
+	}
+
+	// Commands other than "run" all use the HTTP client to
+	// hit the mts server over its unix socket.
+	var c Client
+
+	switch cmd {
+	default:
+		usage("unknown mts server subcommand %q", cmd)
+	case "list", "ls":
+		list, err := c.List()
+		if err != nil {
+			return err
+		}
+		if len(args) == 0 {
+			names := slices.Sorted(maps.Keys(list.Instances))
+			for _, name := range names {
+				running := list.Instances[name].Running
+				fmt.Printf("%10s %s\n", bools.IfElse(running, "RUNNING", "stopped"), name)
+			}
+		} else {
+			for _, name := range args {
+				inst, ok := list.Instances[name]
+				if !ok {
+					return fmt.Errorf("no instance named %q", name)
+				}
+				je := json.NewEncoder(os.Stdout)
+				je.SetIndent("", "  ")
+				if err := je.Encode(inst); err != nil {
+					return err
+				}
+			}
+		}
+
+	case "rm":
+		if len(args) == 0 {
+			return fmt.Errorf("missing instance name(s) to remove")
+		}
+		log.SetFlags(0)
+		for _, name := range args {
+			ok, err := c.Remove(name)
+			if err != nil {
+				return err
+			}
+			if ok {
+				log.Printf("%s deleted.", name)
+			} else {
+				log.Printf("%s didn't exist.", name)
+			}
+		}
+	case "stop":
+		if len(args) == 0 {
+			return fmt.Errorf("missing instance name(s) to stop")
+		}
+		log.SetFlags(0)
+		for _, name := range args {
+			ok, err := c.Stop(name)
+			if err != nil {
+				return err
+			}
+			if ok {
+				log.Printf("%s stopped.", name)
+			} else {
+				log.Printf("%s didn't exist.", name)
+			}
+		}
+	case "start", "restart":
+		list, err := c.List()
+		if err != nil {
+			return err
+		}
+		shouldStop := cmd == "restart"
+		for _, arg := range args {
+			is, ok := list.Instances[arg]
+			if !ok {
+				return fmt.Errorf("no instance named %q", arg)
+			}
+			if is.Running {
+				if shouldStop {
+					if _, err := c.Stop(arg); err != nil {
+						return fmt.Errorf("stopping %q: %w", arg, err)
+					}
+				} else {
+					log.SetFlags(0)
+					log.Printf("%s already running.", arg)
+					continue
+				}
+			}
+			// Creating an existing one starts it up.
+			if err := c.Create(arg); err != nil {
+				return fmt.Errorf("starting %q: %w", arg, err)
+			}
+		}
+	case "add":
+		if len(args) == 0 {
+			return fmt.Errorf("missing instance name(s) to add")
+		}
+		for _, name := range args {
+			if err := c.Create(name); err != nil {
+				return fmt.Errorf("creating %q: %w", name, err)
+			}
+		}
+	case "logs":
+		fs := flag.NewFlagSet("logs", flag.ExitOnError)
+		fs.Usage = func() { usage() }
+		follow := fs.Bool("f", false, "follow logs")
+		fs.Parse(args)
+		log.Printf("Parsed; following=%v, args=%q", *follow, fs.Args())
+		if fs.NArg() != 1 {
+			usage()
+		}
+		cmd := bools.IfElse(*follow, "tail", "cat")
+		args := []string{cmd}
+		if *follow {
+			args = append(args, "-f")
+		}
+		path, err := exec.LookPath(cmd)
+		if err != nil {
+			return fmt.Errorf("looking up %q: %w", cmd, err)
+		}
+		args = append(args, instLogsFile(fs.Arg(0)))
+		log.Fatal(syscall.Exec(path, args, os.Environ()))
+	}
+	return nil
+}
+
+type Client struct {
+}
+
+func (c *Client) client() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return net.Dial("unix", mtsSock())
+			},
+		},
+	}
+}
+
+func getJSON[T any](res *http.Response, err error) (T, error) {
+	var ret T
+	if err != nil {
+		return ret, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return ret, fmt.Errorf("unexpected status: %v: %s", res.Status, body)
+	}
+	if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
+		return ret, err
+	}
+	return ret, nil
+}
+
+func (c *Client) List() (listResponse, error) {
+	return getJSON[listResponse](c.client().Get("http://mts/list"))
+}
+
+func (c *Client) Remove(name string) (found bool, err error) {
+	return getJSON[bool](c.client().PostForm("http://mts/rm", url.Values{
+		"name": []string{name},
+	}))
+}
+
+func (c *Client) Stop(name string) (found bool, err error) {
+	return getJSON[bool](c.client().PostForm("http://mts/stop", url.Values{
+		"name": []string{name},
+	}))
+}
+
+func (c *Client) Create(name string) error {
+	req, err := http.NewRequest("POST", "http://mts/create/"+name, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.client().Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("unexpected status: %v: %s", resp.Status, body)
+	}
+	return nil
+}
+
+func (c *Client) RunCommand(name string, args []string) {
+	sock := instSock(name)
+	lc := &local.Client{
+		Socket:        sock,
+		UseSocketOnly: true,
+	}
+	probeCtx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
+	defer cancel()
+	if _, err := lc.StatusWithoutPeers(probeCtx); err != nil {
+		log.Fatalf("instance %q not running? start with 'mts server start %q'; got error: %v", name, name, err)
+	}
+	args = append([]string{"run", "tailscale.com/cmd/tailscale", "--socket=" + sock}, args...)
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+	err := cmd.Run()
+	if err == nil {
+		os.Exit(0)
+	}
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		os.Exit(exitErr.ExitCode())
+	}
+	panic(err)
+}
+
+type Server struct {
+	lazyTailscaled lazy.GValue[string]
+
+	mu   sync.Mutex
+	cmds map[string]*exec.Cmd // running tailscaled instances
+}
+
+func (s *Server) tailscaled() string {
+	v, err := s.lazyTailscaled.GetErr(func() (string, error) {
+		out, err := exec.Command("go", "list", "-f", "{{.Target}}", "tailscale.com/cmd/tailscaled").CombinedOutput()
+		if err != nil {
+			return "", err
+		}
+		return strings.TrimSpace(string(out)), nil
+	})
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+func (s *Server) Run() error {
+	if err := os.MkdirAll(mtsRoot(), 0700); err != nil {
+		return err
+	}
+	sock := mtsSock()
+	os.Remove(sock)
+	log.Printf("Multi-Tailscaled Server running; listening on %q ...", sock)
+	ln, err := net.Listen("unix", sock)
+	if err != nil {
+		return err
+	}
+	return http.Serve(ln, s)
+}
+
+var validNameRx = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
+
+func validInstanceName(name string) bool {
+	return validNameRx.MatchString(name)
+}
+
+func (s *Server) InstanceRunning(name string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	_, ok := s.cmds[name]
+	return ok
+}
+
+func (s *Server) Stop(name string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if cmd, ok := s.cmds[name]; ok {
+		if err := cmd.Process.Kill(); err != nil {
+			log.Printf("error killing %q: %v", name, err)
+		}
+		delete(s.cmds, name)
+	}
+}
+
+func (s *Server) RunInstance(name string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.cmds[name]; ok {
+		return fmt.Errorf("instance %q already running", name)
+	}
+
+	if !validInstanceName(name) {
+		return fmt.Errorf("invalid instance name %q", name)
+	}
+	dir := filepath.Join(mtsRoot(), name)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+
+	env := os.Environ()
+	env = append(env, "TS_DEBUG_LOG_RATE=all")
+	if ef, err := os.Open(instEnvFile(name)); err == nil {
+		defer ef.Close()
+		sc := bufio.NewScanner(ef)
+		for sc.Scan() {
+			t := strings.TrimSpace(sc.Text())
+			if strings.HasPrefix(t, "#") || !strings.Contains(t, "=") {
+				continue
+			}
+			env = append(env, t)
+		}
+	} else if os.IsNotExist(err) {
+		// Write an example one.
+		os.WriteFile(instEnvFile(name), fmt.Appendf(nil, "# Example mts env.txt file; uncomment/add stuff you want for %q\n\n#TS_DEBUG_MAP=1\n#TS_DEBUG_REGISTER=1\n#TS_NO_LOGS_NO_SUPPORT=1\n", name), 0600)
+	}
+
+	extraArgs := []string{"--verbose=1"}
+	if af, err := os.Open(instArgsFile(name)); err == nil {
+		extraArgs = nil // clear default args
+		defer af.Close()
+		sc := bufio.NewScanner(af)
+		for sc.Scan() {
+			t := strings.TrimSpace(sc.Text())
+			if strings.HasPrefix(t, "#") || t == "" {
+				continue
+			}
+			extraArgs = append(extraArgs, t)
+		}
+	} else if os.IsNotExist(err) {
+		// Write an example one.
+		os.WriteFile(instArgsFile(name), fmt.Appendf(nil, "# Example mts args.txt file for instance %q.\n# One line per extra arg to tailscaled; no magic string quoting\n\n--verbose=1\n#--socks5-server=127.0.0.1:5000\n", name), 0600)
+	}
+
+	log.Printf("Running Tailscale daemon %q in %q", name, dir)
+
+	args := []string{
+		"--tun=userspace-networking",
+		"--statedir=" + filepath.Join(dir),
+		"--socket=" + filepath.Join(dir, "tailscaled.sock"),
+	}
+	args = append(args, extraArgs...)
+
+	cmd := exec.Command(s.tailscaled(), args...)
+	cmd.Dir = dir
+	cmd.Env = env
+
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	cmd.Stderr = cmd.Stdout
+
+	logs := instLogsFile(name)
+	logFile, err := os.OpenFile(logs, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_TRUNC, 0644)
+	if err != nil {
+		return fmt.Errorf("opening logs file: %w", err)
+	}
+
+	go func() {
+		bs := bufio.NewScanner(out)
+		for bs.Scan() {
+			// TODO(bradfitz): record in memory too, serve via HTTP
+			line := strings.TrimSpace(bs.Text())
+			fmt.Fprintf(logFile, "%s\n", line)
+			fmt.Printf("tailscaled[%s]: %s\n", name, line)
+		}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	go func() {
+		err := cmd.Wait()
+		logFile.Close()
+		log.Printf("Tailscale daemon %q exited: %v", name, err)
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		delete(s.cmds, name)
+	}()
+
+	mak.Set(&s.cmds, name, cmd)
+	return nil
+}
+
+type listResponse struct {
+	// Instances maps instance name to its details.
+	Instances map[string]listResponseInstance `json:"instances"`
+}
+
+type listResponseInstance struct {
+	Name    string `json:"name"`
+	Dir     string `json:"dir"`
+	Sock    string `json:"sock"`
+	Running bool   `json:"running"`
+	Env     string `json:"env"`
+	Args    string `json:"args"`
+	Logs    string `json:"logs"`
+}
+
+func writeJSON(w http.ResponseWriter, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "  ")
+	e.Encode(v)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path == "/list" {
+		var res listResponse
+		for _, name := range s.InstanceNames() {
+			mak.Set(&res.Instances, name, listResponseInstance{
+				Name:    name,
+				Dir:     instDir(name),
+				Sock:    instSock(name),
+				Running: s.InstanceRunning(name),
+				Env:     instEnvFile(name),
+				Args:    instArgsFile(name),
+				Logs:    instLogsFile(name),
+			})
+		}
+		writeJSON(w, res)
+		return
+	}
+	if r.URL.Path == "/rm" || r.URL.Path == "/stop" {
+		shouldRemove := r.URL.Path == "/rm"
+		if r.Method != "POST" {
+			http.Error(w, "POST required", http.StatusMethodNotAllowed)
+			return
+		}
+		target := r.FormValue("name")
+		var ok bool
+		for _, name := range s.InstanceNames() {
+			if name != target {
+				continue
+			}
+			ok = true
+			s.Stop(name)
+			if shouldRemove {
+				if err := os.RemoveAll(instDir(name)); err != nil {
+					http.Error(w, err.Error(), http.StatusInternalServerError)
+					return
+				}
+			}
+			break
+		}
+		writeJSON(w, ok)
+		return
+	}
+	if inst, ok := strings.CutPrefix(r.URL.Path, "/create/"); ok {
+		if !s.InstanceRunning(inst) {
+			if err := s.RunInstance(inst); err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+		fmt.Fprintf(w, "OK\n")
+		return
+	}
+	if r.URL.Path == "/" {
+		fmt.Fprintf(w, "This is mts, the multi-tailscaled server.\n")
+		return
+	}
+	http.NotFound(w, r)
+}
+
+func (s *Server) InstanceNames() []string {
+	var ret []string
+	des, err := os.ReadDir(mtsRoot())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		panic(err)
+	}
+	for _, de := range des {
+		if !de.IsDir() {
+			continue
+		}
+		ret = append(ret, de.Name())
+	}
+	return ret
+}
+
+func mtsRoot() string {
+	dir, err := os.UserConfigDir()
+	if err != nil {
+		panic(err)
+	}
+	return filepath.Join(dir, "multi-tailscale-dev")
+}
+
+func instDir(name string) string {
+	return filepath.Join(mtsRoot(), name)
+}
+
+func instSock(name string) string {
+	return filepath.Join(instDir(name), "tailscaled.sock")
+}
+
+func instEnvFile(name string) string {
+	return filepath.Join(mtsRoot(), name, "env.txt")
+}
+
+func instArgsFile(name string) string {
+	return filepath.Join(mtsRoot(), name, "args.txt")
+}
+
+func instLogsFile(name string) string {
+	return filepath.Join(mtsRoot(), name, "logs.txt")
+}
+
+func mtsSock() string {
+	return filepath.Join(mtsRoot(), "mts.sock")
+}

tsweb/debug.go

@@ -9,7 +9,6 @@ import (
 	"html"
 	"io"
 	"net/http"
-	"net/http/pprof"
 	"net/url"
 	"os"
 	"runtime"
@@ -64,16 +63,7 @@ func Debugger(mux *http.ServeMux) *DebugHandler {
 		ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(varz.Handler))
 	}
 
-	// pprof.Index serves everything that runtime/pprof.Lookup finds:
-	// goroutine, threadcreate, heap, allocs, block, mutex
-	ret.Handle("pprof/", "pprof (index)", http.HandlerFunc(pprof.Index))
-	// But register the other ones from net/http/pprof directly:
-	ret.HandleSilent("pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
-	ret.HandleSilent("pprof/profile", http.HandlerFunc(pprof.Profile))
-	ret.HandleSilent("pprof/symbol", http.HandlerFunc(pprof.Symbol))
-	ret.HandleSilent("pprof/trace", http.HandlerFunc(pprof.Trace))
-	ret.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)")
-	ret.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)")
+	addProfilingHandlers(ret)
 	ret.Handle("gc", "force GC", http.HandlerFunc(gcHandler))
 	hostname, err := os.Hostname()
 	if err == nil {

tsweb/pprof_default.go

@@ -0,0 +1,24 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !js && !wasm
+
+package tsweb
+
+import (
+	"net/http"
+	"net/http/pprof"
+)
+
+func addProfilingHandlers(d *DebugHandler) {
+	// pprof.Index serves everything that runtime/pprof.Lookup finds:
+	// goroutine, threadcreate, heap, allocs, block, mutex
+	d.Handle("pprof/", "pprof (index)", http.HandlerFunc(pprof.Index))
+	// But register the other ones from net/http/pprof directly:
+	d.HandleSilent("pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+	d.HandleSilent("pprof/profile", http.HandlerFunc(pprof.Profile))
+	d.HandleSilent("pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	d.HandleSilent("pprof/trace", http.HandlerFunc(pprof.Trace))
+	d.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)")
+	d.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)")
+}

tsweb/pprof_js.go

@@ -0,0 +1,10 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build js && wasm
+
+package tsweb
+
+func addProfilingHandlers(d *DebugHandler) {
+	// No pprof in js builds, pprof doesn't work and bloats the build.
+}

tsweb/tsweb.go

@@ -15,7 +15,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	_ "net/http/pprof"
 	"net/netip"
 	"net/url"
 	"os"

wgengine/bench/wg.go

@@ -46,7 +46,7 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netip.
 		logf: logger.WithPrefix(logf, "tun1: "),
 		traf: traf,
 	}
-	s1 := new(tsd.System)
+	s1 := tsd.NewSystem()
 	e1, err := wgengine.NewUserspaceEngine(l1, wgengine.Config{
 		Router:        router.NewFake(l1),
 		NetMon:        nil,
@@ -73,7 +73,7 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netip.
 		logf: logger.WithPrefix(logf, "tun2: "),
 		traf: traf,
 	}
-	s2 := new(tsd.System)
+	s2 := tsd.NewSystem()
 	e2, err := wgengine.NewUserspaceEngine(l2, wgengine.Config{
 		Router:        router.NewFake(l2),
 		NetMon:        nil,

wgengine/magicsock/magicsock.go

@@ -56,6 +56,7 @@ import (
 	"tailscale.com/types/nettype"
 	"tailscale.com/types/views"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/ringbuffer"
 	"tailscale.com/util/set"
@@ -136,6 +137,8 @@ type Conn struct {
 	// This block mirrors the contents and field order of the Options
 	// struct. Initialized once at construction, then constant.
 
+	eventBus               *eventbus.Bus
+	eventClient            *eventbus.Client
 	logf                   logger.Logf
 	epFunc                 func([]tailcfg.Endpoint)
 	derpActiveFunc         func()
@@ -401,8 +404,15 @@ func (c *Conn) dlogf(format string, a ...any) {
 
 // Options contains options for Listen.
 type Options struct {
-	// Logf optionally provides a log function to use.
-	// Must not be nil.
+	// EventBus, if non-nil, is used for event publication and subscription by
+	// each Conn created from these Options.
+	//
+	// TODO(creachadair): As of 2025-03-19 this is optional, but is intended to
+	// become required non-nil.
+	EventBus *eventbus.Bus
+
+	// Logf provides a log function to use. It must not be nil.
+	// Use [logger.Discard] to disrcard logs.
 	Logf logger.Logf
 
 	// Port is the port to listen on.
@@ -529,6 +539,7 @@ func NewConn(opts Options) (*Conn, error) {
 	}
 
 	c := newConn(opts.logf())
+	c.eventBus = opts.EventBus
 	c.port.Store(uint32(opts.Port))
 	c.controlKnobs = opts.ControlKnobs
 	c.epFunc = opts.endpointsFunc()
@@ -537,6 +548,31 @@ func NewConn(opts Options) (*Conn, error) {
 	c.testOnlyPacketListener = opts.TestOnlyPacketListener
 	c.noteRecvActivity = opts.NoteRecvActivity
 
+	// If an event bus is enabled, subscribe to portmapping changes; otherwise
+	// use the callback mechanism of portmapper.Client.
+	//
+	// TODO(creachadair): Remove the switch once the event bus is mandatory.
+	onPortMapChanged := c.onPortMapChanged
+	if c.eventBus != nil {
+		c.eventClient = c.eventBus.Client("magicsock.Conn")
+
+		pmSub := eventbus.Subscribe[portmapper.Mapping](c.eventClient)
+		go func() {
+			defer pmSub.Close()
+			for {
+				select {
+				case <-pmSub.Events():
+					c.onPortMapChanged()
+				case <-pmSub.Done():
+					return
+				}
+			}
+		}()
+
+		// Disable the explicit callback from the portmapper, the subscriber handles it.
+		onPortMapChanged = nil
+	}
+
 	// Don't log the same log messages possibly every few seconds in our
 	// portmapper.
 	portmapperLogf := logger.WithPrefix(c.logf, "portmapper: ")
@@ -544,7 +580,14 @@ func NewConn(opts Options) (*Conn, error) {
 	portMapOpts := &portmapper.DebugKnobs{
 		DisableAll: func() bool { return opts.DisablePortMapper || c.onlyTCP443.Load() },
 	}
-	c.portMapper = portmapper.NewClient(portmapperLogf, opts.NetMon, portMapOpts, opts.ControlKnobs, c.onPortMapChanged)
+	c.portMapper = portmapper.NewClient(portmapper.Config{
+		EventBus:     c.eventBus,
+		Logf:         portmapperLogf,
+		NetMon:       opts.NetMon,
+		DebugKnobs:   portMapOpts,
+		ControlKnobs: opts.ControlKnobs,
+		OnChange:     onPortMapChanged,
+	})
 	c.portMapper.SetGatewayLookupFunc(opts.NetMon.GatewayAndSelfIP)
 	c.netMon = opts.NetMon
 	c.health = opts.HealthTracker
@@ -2461,6 +2504,9 @@ func (c *connBind) Close() error {
 	if c.closeDisco6 != nil {
 		c.closeDisco6.Close()
 	}
+	if c.eventClient != nil {
+		c.eventClient.Close()
+	}
 	// Send an empty read result to unblock receiveDERP,
 	// which will then check connBind.Closed.
 	// connBind.Closed takes c.mu, but c.derpRecvCh is buffered.

wgengine/magicsock/magicsock_test.go

@@ -62,6 +62,7 @@ import (
 	"tailscale.com/types/nettype"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/cibuild"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/must"
 	"tailscale.com/util/racebuild"
 	"tailscale.com/util/set"
@@ -173,7 +174,10 @@ func newMagicStack(t testing.TB, logf logger.Logf, l nettype.PacketListener, der
 func newMagicStackWithKey(t testing.TB, logf logger.Logf, l nettype.PacketListener, derpMap *tailcfg.DERPMap, privateKey key.NodePrivate) *magicStack {
 	t.Helper()
 
-	netMon, err := netmon.New(logf)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logf)
 	if err != nil {
 		t.Fatalf("netmon.New: %v", err)
 	}
@@ -390,7 +394,10 @@ func TestNewConn(t *testing.T) {
 		}
 	}
 
-	netMon, err := netmon.New(logger.WithPrefix(t.Logf, "... netmon: "))
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logger.WithPrefix(t.Logf, "... netmon: "))
 	if err != nil {
 		t.Fatalf("netmon.New: %v", err)
 	}
@@ -523,7 +530,10 @@ func TestDeviceStartStop(t *testing.T) {
 	tstest.PanicOnLog()
 	tstest.ResourceCheck(t)
 
-	netMon, err := netmon.New(logger.WithPrefix(t.Logf, "... netmon: "))
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logger.WithPrefix(t.Logf, "... netmon: "))
 	if err != nil {
 		t.Fatalf("netmon.New: %v", err)
 	}
@@ -1362,7 +1372,10 @@ func newTestConn(t testing.TB) *Conn {
 	t.Helper()
 	port := pickPort(t)
 
-	netMon, err := netmon.New(logger.WithPrefix(t.Logf, "... netmon: "))
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon, err := netmon.New(bus, logger.WithPrefix(t.Logf, "... netmon: "))
 	if err != nil {
 		t.Fatalf("netmon.New: %v", err)
 	}
@@ -3117,7 +3130,10 @@ func TestMaybeRebindOnError(t *testing.T) {
 }
 
 func TestNetworkDownSendErrors(t *testing.T) {
-	netMon := must.Get(netmon.New(t.Logf))
+	bus := eventbus.New()
+	defer bus.Close()
+
+	netMon := must.Get(netmon.New(bus, t.Logf))
 	defer netMon.Close()
 
 	reg := new(usermetric.Registry)

wgengine/netstack/netstack_test.go

@@ -44,13 +44,14 @@ func TestInjectInboundLeak(t *testing.T) {
 			t.Logf(format, args...)
 		}
 	}
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
 		Tun:           tunDev,
 		Dialer:        dialer,
 		SetSubsystem:  sys.Set,
 		HealthTracker: sys.HealthTracker(),
 		Metrics:       sys.UserMetricsRegistry(),
+		EventBus:      sys.Bus.Get(),
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -100,7 +101,7 @@ func getMemStats() (ms runtime.MemStats) {
 
 func makeNetstack(tb testing.TB, config func(*Impl)) *Impl {
 	tunDev := tstun.NewFake()
-	sys := &tsd.System{}
+	sys := tsd.NewSystem()
 	sys.Set(new(mem.Store))
 	dialer := new(tsdial.Dialer)
 	logf := tstest.WhileTestRunningLogger(tb)
@@ -110,6 +111,7 @@ func makeNetstack(tb testing.TB, config func(*Impl)) *Impl {
 		SetSubsystem:  sys.Set,
 		HealthTracker: sys.HealthTracker(),
 		Metrics:       sys.UserMetricsRegistry(),
+		EventBus:      sys.Bus.Get(),
 	})
 	if err != nil {
 		tb.Fatal(err)

wgengine/router/router_linux_test.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/linuxfw"
 	"tailscale.com/version/distro"
 )
@@ -363,7 +364,9 @@ ip route add throw 192.168.0.0/24 table 52` + basic,
 		},
 	}
 
-	mon, err := netmon.New(logger.Discard)
+	bus := eventbus.New()
+	defer bus.Close()
+	mon, err := netmon.New(bus, logger.Discard)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -973,7 +976,10 @@ func newLinuxRootTest(t *testing.T) *linuxTest {
 
 	logf := lt.logOutput.Logf
 
-	mon, err := netmon.New(logger.Discard)
+	bus := eventbus.New()
+	defer bus.Close()
+
+	mon, err := netmon.New(bus, logger.Discard)
 	if err != nil {
 		lt.Close()
 		t.Fatal(err)

wgengine/userspace.go

@@ -46,6 +46,7 @@ import (
 	"tailscale.com/types/views"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/deephash"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
 	"tailscale.com/util/testenv"
@@ -89,8 +90,12 @@ const statusPollInterval = 1 * time.Minute
 const networkLoggerUploadTimeout = 5 * time.Second
 
 type userspaceEngine struct {
+	// eventBus will eventually become required, but for now may be nil.
+	// TODO(creachadair): Enforce that this is non-nil at construction.
+	eventBus *eventbus.Bus
+
 	logf             logger.Logf
-	wgLogger         *wglog.Logger //a wireguard-go logging wrapper
+	wgLogger         *wglog.Logger // a wireguard-go logging wrapper
 	reqCh            chan struct{}
 	waitCh           chan struct{} // chan is closed when first Close call completes; contrast with closing bool
 	timeNow          func() mono.Time
@@ -227,6 +232,13 @@ type Config struct {
 	// DriveForLocal, if populated, will cause the engine to expose a Taildrive
 	// listener at 100.100.100.100:8080.
 	DriveForLocal drive.FileSystemForLocal
+
+	// EventBus, if non-nil, is used for event publication and subscription by
+	// the Engine and its subsystems.
+	//
+	// TODO(creachadair): As of 2025-03-19 this is optional, but is intended to
+	// become required non-nil.
+	EventBus *eventbus.Bus
 }
 
 // NewFakeUserspaceEngine returns a new userspace engine for testing.
@@ -255,6 +267,8 @@ func NewFakeUserspaceEngine(logf logger.Logf, opts ...any) (Engine, error) {
 			conf.HealthTracker = v
 		case *usermetric.Registry:
 			conf.Metrics = v
+		case *eventbus.Bus:
+			conf.EventBus = v
 		default:
 			return nil, fmt.Errorf("unknown option type %T", v)
 		}
@@ -323,6 +337,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	}
 
 	e := &userspaceEngine{
+		eventBus:       conf.EventBus,
 		timeNow:        mono.Now,
 		logf:           logf,
 		reqCh:          make(chan struct{}, 1),
@@ -348,7 +363,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	if conf.NetMon != nil {
 		e.netMon = conf.NetMon
 	} else {
-		mon, err := netmon.New(logf)
+		mon, err := netmon.New(conf.EventBus, logf)
 		if err != nil {
 			return nil, err
 		}
@@ -389,6 +404,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		}
 	}
 	magicsockOpts := magicsock.Options{
+		EventBus:         e.eventBus,
 		Logf:             logf,
 		Port:             conf.ListenPort,
 		EndpointsFunc:    endpointsFn,

wgengine/userspace_ext_test.go

@@ -16,13 +16,14 @@ import (
 )
 
 func TestIsNetstack(t *testing.T) {
-	sys := new(tsd.System)
+	sys := tsd.NewSystem()
 	e, err := wgengine.NewUserspaceEngine(
 		tstest.WhileTestRunningLogger(t),
 		wgengine.Config{
 			SetSubsystem:  sys.Set,
 			HealthTracker: sys.HealthTracker(),
 			Metrics:       sys.UserMetricsRegistry(),
+			EventBus:      sys.Bus.Get(),
 		},
 	)
 	if err != nil {
@@ -66,7 +67,7 @@ func TestIsNetstackRouter(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			sys := &tsd.System{}
+			sys := tsd.NewSystem()
 			if tt.setNetstackRouter {
 				sys.NetstackRouter.Set(true)
 			}
@@ -74,6 +75,7 @@ func TestIsNetstackRouter(t *testing.T) {
 			conf.SetSubsystem = sys.Set
 			conf.HealthTracker = sys.HealthTracker()
 			conf.Metrics = sys.UserMetricsRegistry()
+			conf.EventBus = sys.Bus.Get()
 			e, err := wgengine.NewUserspaceEngine(logger.Discard, conf)
 			if err != nil {
 				t.Fatal(err)

wgengine/userspace_test.go

@@ -25,6 +25,7 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/opt"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/usermetric"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
@@ -100,9 +101,12 @@ func nodeViews(v []*tailcfg.Node) []tailcfg.NodeView {
 }
 
 func TestUserspaceEngineReconfig(t *testing.T) {
+	bus := eventbus.New()
+	defer bus.Close()
+
 	ht := new(health.Tracker)
 	reg := new(usermetric.Registry)
-	e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg)
+	e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg, bus)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -166,13 +170,16 @@ func TestUserspaceEnginePortReconfig(t *testing.T) {
 
 	var knobs controlknobs.Knobs
 
+	bus := eventbus.New()
+	defer bus.Close()
+
 	// Keep making a wgengine until we find an unused port
 	var ue *userspaceEngine
 	ht := new(health.Tracker)
 	reg := new(usermetric.Registry)
 	for i := range 100 {
 		attempt := uint16(defaultPort + i)
-		e, err := NewFakeUserspaceEngine(t.Logf, attempt, &knobs, ht, reg)
+		e, err := NewFakeUserspaceEngine(t.Logf, attempt, &knobs, ht, reg, bus)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -251,9 +258,11 @@ func TestUserspaceEnginePeerMTUReconfig(t *testing.T) {
 
 	var knobs controlknobs.Knobs
 
+	bus := eventbus.New()
+	defer bus.Close()
 	ht := new(health.Tracker)
 	reg := new(usermetric.Registry)
-	e, err := NewFakeUserspaceEngine(t.Logf, 0, &knobs, ht, reg)
+	e, err := NewFakeUserspaceEngine(t.Logf, 0, &knobs, ht, reg, bus)
 	if err != nil {
 		t.Fatal(err)
 	}

wgengine/watchdog_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"tailscale.com/health"
+	"tailscale.com/util/eventbus"
 	"tailscale.com/util/usermetric"
 )
 
@@ -24,9 +25,11 @@ func TestWatchdog(t *testing.T) {
 
 	t.Run("default watchdog does not fire", func(t *testing.T) {
 		t.Parallel()
+		bus := eventbus.New()
+		defer bus.Close()
 		ht := new(health.Tracker)
 		reg := new(usermetric.Registry)
-		e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg)
+		e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg, bus)
 		if err != nil {
 			t.Fatal(err)
 		}