ipn/ipnlocal: empty allowed exit nodes syspolicy should be treated as allow all

Updates tailscale/corp#19681

If the syspolicy returns an empty list of allowed exit nodes,
this should be treated as "allow all" rather than "allow none"

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

.github/workflows/installer.yml

@@ -32,7 +32,6 @@ jobs:
           - "ubuntu:18.04"
           - "ubuntu:20.04"
           - "ubuntu:22.04"
-          - "ubuntu:22.10"
           - "ubuntu:23.04"
           - "elementary/docker:stable"
           - "elementary/docker:unstable"
@@ -91,7 +90,10 @@ jobs:
         || contains(matrix.image, 'parrotsec')
         || contains(matrix.image, 'kalilinux')
     - name: checkout
-      uses: actions/checkout@v4
+      # We cannot use v4, as it requires a newer glibc version than some of the
+      # tested images provide. See
+      # https://github.com/actions/checkout/issues/1487
+      uses: actions/checkout@v3
     - name: run installer
       run: scripts/installer.sh
       # Package installation can fail in docker because systemd is not running

.github/workflows/test.yml

@@ -356,7 +356,7 @@ jobs:
       # some Android breakages early.
       # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482
     - name: build some
-      run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/interfaces ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version
+      run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/netmon ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version
       env:
         GOOS: android
         GOARCH: arm64

.gitignore

@@ -9,6 +9,7 @@
 
 cmd/tailscale/tailscale
 cmd/tailscaled/tailscaled
+ssh/tailssh/testcontainers/tailscaled
 
 # Test binary, built with `go test -c`
 *.test

Makefile

@@ -100,6 +100,23 @@ publishdevoperator: ## Build and publish k8s-operator image to location specifie
 	@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
 	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=operator ./build_docker.sh
 
+publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO}
+	@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
+	@test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1)
+	@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
+	@test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1)
+	@test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1)
+	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh
+
+.PHONY: sshintegrationtest
+sshintegrationtest: ## Run the SSH integration tests in various Docker containers
+	@GOOS=linux GOARCH=amd64 go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \
+	GOOS=linux GOARCH=amd64 go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
+	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
+
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"
 	@grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[0;36m%-20s\033[m %s\n", $$1, $$2}'

VERSION.txt

@@ -1 +1 @@
-1.65.0
+1.67.0

appc/appconnector.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/execqueue"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/slicesx"
 )
 
 // RouteAdvertiser is an interface that allows the AppConnector to advertise
@@ -36,6 +37,19 @@ type RouteAdvertiser interface {
 	UnadvertiseRoute(...netip.Prefix) error
 }
 
+// RouteInfo is a data structure used to persist the in memory state of an AppConnector
+// so that we can know, even after a restart, which routes came from ACLs and which were
+// learned from domains.
+type RouteInfo struct {
+	// Control is the routes from the 'routes' section of an app connector acl.
+	Control []netip.Prefix `json:",omitempty"`
+	// Domains are the routes discovered by observing DNS lookups for configured domains.
+	Domains map[string][]netip.Addr `json:",omitempty"`
+	// Wildcards are the configured DNS lookup domains to observe. When a DNS query matches Wildcards,
+	// its result is added to Domains.
+	Wildcards []string `json:",omitempty"`
+}
+
 // AppConnector is an implementation of an AppConnector that performs
 // its function as a subsystem inside of a tailscale node. At the control plane
 // side App Connector routing is configured in terms of domains rather than IP
@@ -49,6 +63,9 @@ type AppConnector struct {
 	logf            logger.Logf
 	routeAdvertiser RouteAdvertiser
 
+	// storeRoutesFunc will be called to persist routes if it is not nil.
+	storeRoutesFunc func(*RouteInfo) error
+
 	// mu guards the fields that follow
 	mu sync.Mutex
 
@@ -67,11 +84,46 @@ type AppConnector struct {
 }
 
 // NewAppConnector creates a new AppConnector.
-func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser) *AppConnector {
-	return &AppConnector{
+func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInfo *RouteInfo, storeRoutesFunc func(*RouteInfo) error) *AppConnector {
+	ac := &AppConnector{
 		logf:            logger.WithPrefix(logf, "appc: "),
 		routeAdvertiser: routeAdvertiser,
+		storeRoutesFunc: storeRoutesFunc,
 	}
+	if routeInfo != nil {
+		ac.domains = routeInfo.Domains
+		ac.wildcards = routeInfo.Wildcards
+		ac.controlRoutes = routeInfo.Control
+	}
+	return ac
+}
+
+// ShouldStoreRoutes returns true if the appconnector was created with the controlknob on
+// and is storing its discovered routes persistently.
+func (e *AppConnector) ShouldStoreRoutes() bool {
+	return e.storeRoutesFunc != nil
+}
+
+// storeRoutesLocked takes the current state of the AppConnector and persists it
+func (e *AppConnector) storeRoutesLocked() error {
+	if !e.ShouldStoreRoutes() {
+		return nil
+	}
+	return e.storeRoutesFunc(&RouteInfo{
+		Control:   e.controlRoutes,
+		Domains:   e.domains,
+		Wildcards: e.wildcards,
+	})
+}
+
+// ClearRoutes removes all route state from the AppConnector.
+func (e *AppConnector) ClearRoutes() error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.controlRoutes = nil
+	e.domains = nil
+	e.wildcards = nil
+	return e.storeRoutesLocked()
 }
 
 // UpdateDomainsAndRoutes starts an asynchronous update of the configuration
@@ -125,10 +177,26 @@ func (e *AppConnector) updateDomains(domains []string) {
 		for _, wc := range e.wildcards {
 			if dnsname.HasSuffix(d, wc) {
 				e.domains[d] = addrs
+				delete(oldDomains, d)
 				break
 			}
 		}
 	}
+
+	// Everything left in oldDomains is a domain we're no longer tracking
+	// and if we are storing route info we can unadvertise the routes
+	if e.ShouldStoreRoutes() {
+		toRemove := []netip.Prefix{}
+		for _, addrs := range oldDomains {
+			for _, a := range addrs {
+				toRemove = append(toRemove, netip.PrefixFrom(a, a.BitLen()))
+			}
+		}
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
+		}
+	}
+
 	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
 }
 
@@ -152,6 +220,14 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 
 	var toRemove []netip.Prefix
 
+	// If we're storing routes and know e.controlRoutes is a good
+	// representation of what should be in AdvertisedRoutes we can stop
+	// advertising routes that used to be in e.controlRoutes but are not
+	// in routes.
+	if e.ShouldStoreRoutes() {
+		toRemove = routesWithout(e.controlRoutes, routes)
+	}
+
 nextRoute:
 	for _, r := range routes {
 		for _, addr := range e.domains {
@@ -170,6 +246,9 @@ nextRoute:
 	}
 
 	e.controlRoutes = routes
+	if err := e.storeRoutesLocked(); err != nil {
+		e.logf("failed to store route info: %v", err)
+	}
 }
 
 // Domains returns the currently configured domain list.
@@ -380,6 +459,9 @@ func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Pref
 				e.logf("[v2] advertised route for %v: %v", domain, addr)
 			}
 		}
+		if err := e.storeRoutesLocked(); err != nil {
+			e.logf("failed to store route info: %v", err)
+		}
 	})
 }
 
@@ -400,3 +482,15 @@ func (e *AppConnector) addDomainAddrLocked(domain string, addr netip.Addr) {
 func compareAddr(l, r netip.Addr) int {
 	return l.Compare(r)
 }
+
+// routesWithout returns a without b where a and b
+// are unsorted slices of netip.Prefix
+func routesWithout(a, b []netip.Prefix) []netip.Prefix {
+	m := make(map[netip.Prefix]bool, len(b))
+	for _, p := range b {
+		m[p] = true
+	}
+	return slicesx.Filter(make([]netip.Prefix, 0, len(a)), a, func(p netip.Prefix) bool {
+		return !m[p]
+	})
+}

appc/appconnector_test.go

@@ -17,194 +17,238 @@ import (
 	"tailscale.com/util/must"
 )
 
+func fakeStoreRoutes(*RouteInfo) error { return nil }
+
 func TestUpdateDomains(t *testing.T) {
-	ctx := context.Background()
-	a := NewAppConnector(t.Logf, nil)
-	a.UpdateDomains([]string{"example.com"})
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, &appctest.RouteCollector{}, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, &appctest.RouteCollector{}, nil, nil)
+		}
+		a.UpdateDomains([]string{"example.com"})
 
-	a.Wait(ctx)
-	if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.Wait(ctx)
+		if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	addr := netip.MustParseAddr("192.0.0.8")
-	a.domains["example.com"] = append(a.domains["example.com"], addr)
-	a.UpdateDomains([]string{"example.com"})
-	a.Wait(ctx)
+		addr := netip.MustParseAddr("192.0.0.8")
+		a.domains["example.com"] = append(a.domains["example.com"], addr)
+		a.UpdateDomains([]string{"example.com"})
+		a.Wait(ctx)
 
-	if got, want := a.domains["example.com"], []netip.Addr{addr}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		if got, want := a.domains["example.com"], []netip.Addr{addr}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// domains are explicitly downcased on set.
-	a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
-	a.Wait(ctx)
-	if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
+		// domains are explicitly downcased on set.
+		a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
+		a.Wait(ctx)
+		if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 	}
 }
 
 func TestUpdateRoutes(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"*.example.com"})
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		a.updateDomains([]string{"*.example.com"})
 
-	// This route should be collapsed into the range
-	a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
-	a.Wait(ctx)
+		// This route should be collapsed into the range
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		a.Wait(ctx)
 
-	if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
-		t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	}
+		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
+			t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		}
 
-	// This route should not be collapsed or removed
-	a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
-	a.Wait(ctx)
+		// This route should not be collapsed or removed
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		a.Wait(ctx)
 
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
-	a.updateRoutes(routes)
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
 
-	slices.SortFunc(rc.Routes(), prefixCompare)
-	rc.SetRoutes(slices.Compact(rc.Routes()))
-	slices.SortFunc(routes, prefixCompare)
+		slices.SortFunc(rc.Routes(), prefixCompare)
+		rc.SetRoutes(slices.Compact(rc.Routes()))
+		slices.SortFunc(routes, prefixCompare)
 
-	// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
-	}
+		// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
+		}
 
-	// Ensure that the contained /32 is removed, replaced by the /24.
-	wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
-	if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
-		t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		// Ensure that the contained /32 is removed, replaced by the /24.
+		wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
-	rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
-	a.updateRoutes(routes)
+	for _, shouldStore := range []bool{false, true} {
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
+		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+		a.updateRoutes(routes)
 
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		}
 	}
 }
 
 func TestDomainRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(context.Background())
+	for _, shouldStore := range []bool{false, true} {
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(context.Background())
 
-	want := map[string][]netip.Addr{
-		"example.com": {netip.MustParseAddr("192.0.0.8")},
-	}
+		want := map[string][]netip.Addr{
+			"example.com": {netip.MustParseAddr("192.0.0.8")},
+		}
 
-	if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
-		t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
+			t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		}
 	}
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
 
-	// a has no domains configured, so it should not advertise any routes
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a has no domains configured, so it should not advertise any routes
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// matches a routed domain.
-	a.updateDomains([]string{"www.example.com", "example.com"})
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// matches a routed domain.
+		a.updateDomains([]string{"www.example.com", "example.com"})
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// even if only found in the middle of the chain
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// even if only found in the middle of the chain
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// don't re-advertise routes that have already been advertised
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
+		// don't re-advertise routes that have already been advertised
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
 
-	// don't advertise addresses that are already in a control provided route
-	pfx := netip.MustParsePrefix("192.0.2.0/24")
-	a.updateRoutes([]netip.Prefix{pfx})
-	wantRoutes = append(wantRoutes, pfx)
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
-	if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
-		t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		// don't advertise addresses that are already in a control provided route
+		pfx := netip.MustParsePrefix("192.0.2.0/24")
+		a.updateRoutes([]netip.Prefix{pfx})
+		wantRoutes = append(wantRoutes, pfx)
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
+		if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
+			t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		}
 	}
 }
 
 func TestWildcardDomains(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
 
-	a.updateDomains([]string{"*.example.com"})
-	a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
-		t.Errorf("routes: got %v; want %v", got, want)
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com"})
+		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
+			t.Errorf("routes: got %v; want %v", got, want)
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if _, ok := a.domains["foo.example.com"]; !ok {
-		t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if _, ok := a.domains["foo.example.com"]; !ok {
+			t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if len(a.wildcards) != 1 {
-		t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if len(a.wildcards) != 1 {
+			t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		}
 	}
 }
 
@@ -310,3 +354,169 @@ func prefixCompare(a, b netip.Prefix) int {
 	}
 	return a.Addr().Compare(b.Addr())
 }
+
+func prefixes(in ...string) []netip.Prefix {
+	toRet := make([]netip.Prefix, len(in))
+	for i, s := range in {
+		toRet[i] = netip.MustParsePrefix(s)
+	}
+	return toRet
+}
+
+func TestUpdateRouteRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		// nothing has yet been advertised
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{}, prefixes("1.2.3.1/32", "1.2.3.2/32"))
+		a.Wait(ctx)
+		// the routes passed to UpdateDomainsAndRoutes have been advertised
+		assertRoutes("simple update", prefixes("1.2.3.1/32", "1.2.3.2/32"), []netip.Prefix{})
+
+		// one route the same, one different
+		a.UpdateDomainsAndRoutes([]string{}, prefixes("1.2.3.1/32", "1.2.3.3/32"))
+		a.Wait(ctx)
+		// old behavior: routes are not removed, resulting routes are both old and new
+		// (we have dupe 1.2.3.1 routes because the test RouteAdvertiser doesn't have the deduplication
+		// the real one does)
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.1/32", "1.2.3.3/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior: routes are removed, resulting routes are new only
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.1/32", "1.2.3.3/32")
+			wantRemovedRoutes = prefixes("1.2.3.2/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestUpdateDomainRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com", "b.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// adding domains doesn't immediately cause any routes to be advertised
+		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
+
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
+		a.Wait(ctx)
+		// observing dns responses causes routes to be advertised
+		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// old behavior, routes are not removed
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior, routes are removed for b.example.com
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.2/32")
+			wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestUpdateWildcardRouteRemoval(t *testing.T) {
+	for _, shouldStore := range []bool{false, true} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+
+		assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) {
+			if !slices.Equal(routes, rc.Routes()) {
+				t.Fatalf("%s: (shouldStore=%t) routes want %v, got %v", prefix, shouldStore, routes, rc.Routes())
+			}
+			if !slices.Equal(removedRoutes, rc.RemovedRoutes()) {
+				t.Fatalf("%s: (shouldStore=%t) removedRoutes want %v, got %v", prefix, shouldStore, removedRoutes, rc.RemovedRoutes())
+			}
+		}
+
+		var a *AppConnector
+		if shouldStore {
+			a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes)
+		} else {
+			a = NewAppConnector(t.Logf, rc, nil, nil)
+		}
+		assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com", "*.b.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// adding domains doesn't immediately cause any routes to be advertised
+		assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
+
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
+		a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
+		a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
+		a.Wait(ctx)
+		// observing dns responses causes routes to be advertised
+		assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
+
+		a.UpdateDomainsAndRoutes([]string{"a.example.com"}, []netip.Prefix{})
+		a.Wait(ctx)
+		// old behavior, routes are not removed
+		wantRoutes := prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32")
+		wantRemovedRoutes := []netip.Prefix{}
+		if shouldStore {
+			// new behavior, routes are removed for *.b.example.com
+			wantRoutes = prefixes("1.2.3.1/32", "1.2.3.2/32")
+			wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32")
+		}
+		assertRoutes("removal", wantRoutes, wantRemovedRoutes)
+	}
+}
+
+func TestRoutesWithout(t *testing.T) {
+	assert := func(msg string, got, want []netip.Prefix) {
+		if !slices.Equal(want, got) {
+			t.Errorf("%s: want %v, got %v", msg, want, got)
+		}
+	}
+
+	assert("empty routes", routesWithout([]netip.Prefix{}, []netip.Prefix{}), []netip.Prefix{})
+	assert("a empty", routesWithout([]netip.Prefix{}, prefixes("1.1.1.1/32", "1.1.1.2/32")), []netip.Prefix{})
+	assert("b empty", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), []netip.Prefix{}), prefixes("1.1.1.1/32", "1.1.1.2/32"))
+	assert("no overlap", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.3/32", "1.1.1.4/32")), prefixes("1.1.1.1/32", "1.1.1.2/32"))
+	assert("a has fewer", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32")), []netip.Prefix{})
+	assert("a has more", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32"), prefixes("1.1.1.1/32", "1.1.1.3/32")), prefixes("1.1.1.2/32", "1.1.1.4/32"))
+}

build_docker.sh

@@ -71,6 +71,22 @@ case "$TARGET" in
       --target="${PLATFORM}" \
       /usr/local/bin/operator
     ;;
+  k8s-nameserver)
+    DEFAULT_REPOS="tailscale/k8s-nameserver"
+    REPOS="${REPOS:-${DEFAULT_REPOS}}"
+    go run github.com/tailscale/mkctr \
+      --gopaths="tailscale.com/cmd/k8s-nameserver:/usr/local/bin/k8s-nameserver" \
+      --ldflags=" \
+        -X tailscale.com/version.longStamp=${VERSION_LONG} \
+        -X tailscale.com/version.shortStamp=${VERSION_SHORT} \
+        -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \
+      --base="${BASE}" \
+      --tags="${TAGS}" \
+      --repos="${REPOS}" \
+      --push="${PUSH}" \
+      --target="${PLATFORM}" \
+      /usr/local/bin/k8s-nameserver
+    ;;
   *)
     echo "unknown target: $TARGET"
     exit 1

client/tailscale/localclient.go

@@ -778,6 +778,17 @@ func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
 //
 // The ctx is only used for the duration of the call, not the lifetime of the net.Conn.
 func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
+	return lc.UserDial(ctx, "tcp", host, port)
+}
+
+// UserDial connects to the host's port via Tailscale for the given network.
+//
+// The host may be a base DNS name (resolved from the netmap inside tailscaled),
+// a FQDN, or an IP address.
+//
+// The ctx is only used for the duration of the call, not the lifetime of the
+// net.Conn.
+func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
 	connCh := make(chan net.Conn, 1)
 	trace := httptrace.ClientTrace{
 		GotConn: func(info httptrace.GotConnInfo) {
@@ -790,10 +801,11 @@ func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (n
 		return nil, err
 	}
 	req.Header = http.Header{
-		"Upgrade":    []string{"ts-dial"},
-		"Connection": []string{"upgrade"},
-		"Dial-Host":  []string{host},
-		"Dial-Port":  []string{fmt.Sprint(port)},
+		"Upgrade":      []string{"ts-dial"},
+		"Connection":   []string{"upgrade"},
+		"Dial-Host":    []string{host},
+		"Dial-Port":    []string{fmt.Sprint(port)},
+		"Dial-Network": []string{network},
 	}
 	res, err := lc.DoLocalRequest(req)
 	if err != nil {

client/tailscale/localclient_test.go

@@ -35,6 +35,7 @@ func TestDeps(t *testing.T) {
 		BadDeps: map[string]string{
 			// Make sure we don't again accidentally bring in a dependency on
 			// drive or its transitive dependencies
+			"testing":                        "do not use testing package in production code",
 			"tailscale.com/drive/driveimpl":  "https://github.com/tailscale/tailscale/pull/10631",
 			"github.com/studio-b12/gowebdav": "https://github.com/tailscale/tailscale/pull/10631",
 		},

client/web/web.go

@@ -1150,7 +1150,15 @@ func (s *Server) tailscaleUp(ctx context.Context, st *ipnstate.Status, opt tails
 		if !isRunning {
 			ipnOptions := ipn.Options{AuthKey: opt.AuthKey}
 			if opt.ControlURL != "" {
-				ipnOptions.UpdatePrefs = &ipn.Prefs{ControlURL: opt.ControlURL}
+				_, err := s.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+					Prefs: ipn.Prefs{
+						ControlURL: opt.ControlURL,
+					},
+					ControlURLSet: true,
+				})
+				if err != nil {
+					s.logf("edit prefs: %v", err)
+				}
 			}
 			if err := s.lc.Start(ctx, ipnOptions); err != nil {
 				s.logf("start: %v", err)

clientupdate/clientupdate.go

@@ -653,6 +653,9 @@ func (up *Updater) updateAlpineLike() (err error) {
 		return fmt.Errorf(`failed to parse latest version from "apk info tailscale": %w`, err)
 	}
 	if !up.confirm(ver) {
+		if err := checkOutdatedAlpineRepo(up.Logf, ver, up.Track); err != nil {
+			up.Logf("failed to check whether Alpine release is outdated: %v", err)
+		}
 		return nil
 	}
 
@@ -690,6 +693,37 @@ func parseAlpinePackageVersion(out []byte) (string, error) {
 	return "", errors.New("tailscale version not found in output")
 }
 
+var apkRepoVersionRE = regexp.MustCompile(`v[0-9]+\.[0-9]+`)
+
+func checkOutdatedAlpineRepo(logf logger.Logf, apkVer, track string) error {
+	latest, err := LatestTailscaleVersion(track)
+	if err != nil {
+		return err
+	}
+	if latest == apkVer {
+		// Actually on latest release.
+		return nil
+	}
+	f, err := os.Open("/etc/apk/repositories")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	// Read the first repo line. Typically, there are multiple repos that all
+	// contain the same version in the path, like:
+	//   https://dl-cdn.alpinelinux.org/alpine/v3.20/main
+	//   https://dl-cdn.alpinelinux.org/alpine/v3.20/community
+	s := bufio.NewScanner(f)
+	if !s.Scan() {
+		return s.Err()
+	}
+	alpineVer := apkRepoVersionRE.FindString(s.Text())
+	if alpineVer != "" {
+		logf("The latest Tailscale release for Linux is %q, but your apk repository only provides %q.\nYour Alpine version is %q, you may need to upgrade the system to get the latest Tailscale version: https://wiki.alpinelinux.org/wiki/Upgrading_Alpine", latest, apkVer, alpineVer)
+	}
+	return nil
+}
+
 func (up *Updater) updateMacSys() error {
 	return errors.New("NOTREACHED: On MacSys builds, `tailscale update` is handled in Swift to launch the GUI updater")
 }
@@ -1019,6 +1053,20 @@ func (up *Updater) updateLinuxBinary() error {
 	return nil
 }
 
+func restartSystemdUnit(ctx context.Context) error {
+	if _, err := exec.LookPath("systemctl"); err != nil {
+		// Likely not a systemd-managed distro.
+		return errors.ErrUnsupported
+	}
+	if out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput(); err != nil {
+		return fmt.Errorf("systemctl daemon-reload failed: %w\noutput: %s", err, out)
+	}
+	if out, err := exec.Command("systemctl", "restart", "tailscaled.service").CombinedOutput(); err != nil {
+		return fmt.Errorf("systemctl restart failed: %w\noutput: %s", err, out)
+	}
+	return nil
+}
+
 func (up *Updater) downloadLinuxTarball(ver string) (string, error) {
 	dlDir, err := os.UserCacheDir()
 	if err != nil {

clientupdate/systemd_linux.go

@@ -1,37 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package clientupdate
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"github.com/coreos/go-systemd/v22/dbus"
-)
-
-func restartSystemdUnit(ctx context.Context) error {
-	c, err := dbus.NewWithContext(ctx)
-	if err != nil {
-		// Likely not a systemd-managed distro.
-		return errors.ErrUnsupported
-	}
-	defer c.Close()
-	if err := c.ReloadContext(ctx); err != nil {
-		return fmt.Errorf("failed to reload tailscaled.service: %w", err)
-	}
-	ch := make(chan string, 1)
-	if _, err := c.RestartUnitContext(ctx, "tailscaled.service", "replace", ch); err != nil {
-		return fmt.Errorf("failed to restart tailscaled.service: %w", err)
-	}
-	select {
-	case res := <-ch:
-		if res != "done" {
-			return fmt.Errorf("systemd service restart failed with result %q", res)
-		}
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-	return nil
-}

clientupdate/systemd_other.go

@@ -1,15 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !linux
-
-package clientupdate
-
-import (
-	"context"
-	"errors"
-)
-
-func restartSystemdUnit(ctx context.Context) error {
-	return errors.ErrUnsupported
-}

cmd/containerboot/kube.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -18,20 +19,6 @@ import (
 	"tailscale.com/tailcfg"
 )
 
-// findKeyInKubeSecret inspects the kube secret secretName for a data
-// field called "authkey", and returns its value if present.
-func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
-	s, err := kc.GetSecret(ctx, secretName)
-	if err != nil {
-		return "", err
-	}
-	ak, ok := s.Data["authkey"]
-	if !ok {
-		return "", nil
-	}
-	return string(ak), nil
-}
-
 // storeDeviceInfo writes deviceID into the "device_id" data field of the kube
 // secret secretName.
 func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
@@ -88,9 +75,59 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc *kube.Client
+var kc kube.Client
 
-func initKube(root string) {
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kube.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
+
+func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.
@@ -101,9 +138,9 @@ func initKube(root string) {
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}
-	if root != "/" {
-		// If we are running in a test, we need to set the URL to the
-		// httptest server.
+	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		// Used to set http server in tests, or optionally enabled by flag
 		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
 	}
 }

cmd/containerboot/kube_test.go

@@ -0,0 +1,206 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/kube"
+)
+
+func TestSetupKube(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *settings
+		wantErr bool
+		wantCfg *settings
+		kc      kube.Client
+	}{
+		{
+			name: "TS_AUTHKEY set, state Secret exists",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, nil
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+		},
+		{
+			name: "TS_AUTHKEY set, state Secret does not exist, we have permissions to create it",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, true, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+		},
+		{
+			name: "TS_AUTHKEY set, state Secret does not exist, we do not have permissions to create it",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY set, we encounter a non-404 error when trying to retrieve the state Secret",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 403}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY set, we encounter a non-404 error when trying to check Secret permissions",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, errors.New("broken")
+				},
+			},
+			wantErr: true,
+		},
+		{
+			// Interactive login using URL in Pod logs
+			name: "TS_AUTHKEY not set, state Secret does not exist, we have permissions to create it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, true, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+		},
+		{
+			// Interactive login using URL in Pod logs
+			name: "TS_AUTHKEY not set, state Secret exists, but does not contain auth key",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{}, nil
+				},
+			},
+		},
+		{
+			name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				},
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY not set, state Secret contains auth key, we have RBAC to patch it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return true, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				},
+			},
+			wantCfg: &settings{
+				KubeSecret:         "foo",
+				AuthKey:            "foo",
+				KubernetesCanPatch: true,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		kc = tt.kc
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
+				t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {
+				t.Errorf("unexpected contents of settings after running settings.setupKube()\n(-got +want):\n%s", diff)
+			}
+		})
+	}
+}

cmd/containerboot/main.go

@@ -52,8 +52,10 @@
 //     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
 //     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
 //     and will be re-applied when it changes.
-//   - EXPERIMENTAL_TS_CONFIGFILE_PATH: if specified, a path to tailscaled
-//     config. If this is set, TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY,
+//   - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
+//     directory that containers tailscaled config in file. The config file needs to be
+//     named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
+//     TS_EXTRA_ARGS, TS_AUTHKEY,
 //     TS_ROUTES, TS_ACCEPT_DNS env vars must not be set. If this is set,
 //     containerboot only runs `tailscaled --config <path-to-this-configfile>`
 //     and not `tailscale up` or `tailscale set`.
@@ -92,6 +94,7 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"reflect"
 	"slices"
@@ -107,6 +110,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/conffile"
+	kubeutils "tailscale.com/k8s-operator"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/ptr"
@@ -145,7 +149,7 @@ func main() {
 		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
 		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
 		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
-		TailscaledConfigFilePath:              defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""),
+		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
 	}
@@ -171,44 +175,16 @@ func main() {
 		}
 	}
 
-	if cfg.InKubernetes {
-		initKube(cfg.Root)
-	}
-
 	// Context is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
 	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" {
-		canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret)
-		if err != nil {
-			log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
-		}
-		cfg.KubernetesCanPatch = canPatch
-
-		if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-			key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret)
-			if err != nil {
-				log.Fatalf("Getting authkey from kube secret: %v", err)
-			}
-			if key != "" {
-				// This behavior of pulling authkeys from kube secrets was added
-				// at the same time as the patch permission, so we can enforce
-				// that we must be able to patch out the authkey after
-				// authenticating if you want to use this feature. This avoids
-				// us having to deal with the case where we might leave behind
-				// an unnecessary reusable authkey in a secret, like a rake in
-				// the grass.
-				if !cfg.KubernetesCanPatch {
-					log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-				}
-				log.Print("Using authkey found in kube secret")
-				cfg.AuthKey = key
-			} else {
-				log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-			}
+	if cfg.InKubernetes {
+		initKubeClient(cfg.Root)
+		if err := cfg.setupKube(bootCtx); err != nil {
+			log.Fatalf("error setting up for running on Kubernetes: %v", err)
 		}
 	}
 
@@ -559,25 +535,26 @@ runLoop:
 					log.Println("Startup complete, waiting for shutdown signal")
 					startupTasksDone = true
 
-					// Reap all processes, since we are PID1 and need to collect zombies. We can
-					// only start doing this once we've stopped shelling out to things
-					// `tailscale up`, otherwise this goroutine can reap the CLI subprocesses
-					// and wedge bringup.
+					// Wait on tailscaled process. It won't
+					// be cleaned up by default when the
+					// container exits as it is not PID1.
+					// TODO (irbekrm): perhaps we can
+					// replace the reaper by a running
+					// cmd.Wait in a goroutine immediately
+					// after starting tailscaled?
 					reaper := func() {
 						defer wg.Done()
 						for {
 							var status unix.WaitStatus
-							pid, err := unix.Wait4(-1, &status, 0, nil)
+							_, err := unix.Wait4(daemonProcess.Pid, &status, 0, nil)
 							if errors.Is(err, unix.EINTR) {
 								continue
 							}
 							if err != nil {
-								log.Fatalf("Waiting for exited processes: %v", err)
-							}
-							if pid == daemonProcess.Pid {
-								log.Printf("Tailscaled exited")
-								os.Exit(0)
+								log.Fatalf("Waiting for tailscaled to exit: %v", err)
 							}
+							log.Print("tailscaled exited")
+							os.Exit(0)
 						}
 					}
 					wg.Add(1)
@@ -984,16 +961,23 @@ func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []ne
 		return err
 	}
 	var local netip.Addr
+	proxyHasIPv4Address := false
 	for _, pfx := range tsIPs {
 		if !pfx.IsSingleIP() {
 			continue
 		}
+		if pfx.Addr().Is4() {
+			proxyHasIPv4Address = true
+		}
 		if pfx.Addr().Is4() != dst.Is4() {
 			continue
 		}
 		local = pfx.Addr()
 		break
 	}
+	if proxyHasIPv4Address && dst.Is6() {
+		log.Printf("Warning: proxy backend ClusterIP is an IPv6 address and the proxy has a IPv4 tailnet address. You might need to disable IPv4 address allocation for the proxy for forwarding to work. See https://github.com/tailscale/tailscale/issues/12156")
+	}
 	if !local.IsValid() {
 		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
 	}
@@ -1124,6 +1108,13 @@ type settings struct {
 
 func (s *settings) validate() error {
 	if s.TailscaledConfigFilePath != "" {
+		dir, file := path.Split(s.TailscaledConfigFilePath)
+		if _, err := os.Stat(dir); err != nil {
+			return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err)
+		}
+		if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil {
+			return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err)
+		}
 		if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil {
 			return fmt.Errorf("error validating tailscaled configfile contents: %w", err)
 		}
@@ -1147,7 +1138,7 @@ func (s *settings) validate() error {
 		return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set")
 	}
 	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
-		return errors.New("EXPERIMENTAL_TS_CONFIGFILE_PATH cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
+		return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
 	}
 	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
 		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
@@ -1279,3 +1270,42 @@ func isTwoStepConfigAlwaysAuth(cfg *settings) bool {
 func isOneStepConfig(cfg *settings) bool {
 	return cfg.TailscaledConfigFilePath != ""
 }
+
+// tailscaledConfigFilePath returns the path to the tailscaled config file that
+// should be used for the current capability version. It is determined by the
+// TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a
+// file named cap-<capability_version>.hujson in the directory. It searches for
+// the highest capability version that is less than or equal to the current
+// capability version.
+func tailscaledConfigFilePath() string {
+	dir := os.Getenv("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR")
+	if dir == "" {
+		return ""
+	}
+	fe, err := os.ReadDir(dir)
+	if err != nil {
+		log.Fatalf("error reading tailscaled config directory %q: %v", dir, err)
+	}
+	maxCompatVer := tailcfg.CapabilityVersion(-1)
+	for _, e := range fe {
+		// We don't check if type if file as in most cases this will
+		// come from a mounted kube Secret, where the directory contents
+		// will be various symlinks.
+		if e.Type().IsDir() {
+			continue
+		}
+		cv, err := kubeutils.CapVerFromFileName(e.Name())
+		if err != nil {
+			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
+			continue
+		}
+		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
+			maxCompatVer = cv
+		}
+	}
+	if maxCompatVer == -1 {
+		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
+	}
+	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
+	return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer))
+}

cmd/containerboot/main_test.go

@@ -65,7 +65,7 @@ func TestContainerBoot(t *testing.T) {
 		"dev/net",
 		"proc/sys/net/ipv4",
 		"proc/sys/net/ipv6/conf/all",
-		"etc",
+		"etc/tailscaled",
 	}
 	for _, path := range dirs {
 		if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil {
@@ -80,7 +80,7 @@ func TestContainerBoot(t *testing.T) {
 		"dev/net/tun":                           []byte(""),
 		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
 		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
-		"etc/tailscaled":                        tailscaledConfBytes,
+		"etc/tailscaled/cap-95.hujson":          tailscaledConfBytes,
 	}
 	resetFiles := func() {
 		for path, content := range files {
@@ -638,14 +638,14 @@ func TestContainerBoot(t *testing.T) {
 			},
 		},
 		{
-			Name: "experimental tailscaled configfile",
+			Name: "experimental tailscaled config path",
 			Env: map[string]string{
-				"EXPERIMENTAL_TS_CONFIGFILE_PATH": filepath.Join(d, "etc/tailscaled"),
+				"TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(d, "etc/tailscaled/"),
 			},
 			Phases: []phase{
 				{
 					WantCmds: []string{
-						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled",
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson",
 					},
 				}, {
 					Notify: runningNotify,

cmd/derper/bootstrap_dns.go

@@ -5,35 +5,45 @@ package main
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"expvar"
 	"log"
+	"math/rand/v2"
 	"net"
 	"net/http"
+	"net/netip"
+	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/syncs"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/slicesx"
 )
 
 const refreshTimeout = time.Minute
 
-type dnsEntryMap map[string][]net.IP
+type dnsEntryMap struct {
+	IPs     map[string][]net.IP
+	Percent map[string]float64 // "foo.com" => 0.5 for 50%
+}
 
 var (
-	dnsCache            syncs.AtomicValue[dnsEntryMap]
+	dnsCache            atomic.Pointer[dnsEntryMap]
 	dnsCacheBytes       syncs.AtomicValue[[]byte] // of JSON
-	unpublishedDNSCache syncs.AtomicValue[dnsEntryMap]
+	unpublishedDNSCache atomic.Pointer[dnsEntryMap]
 	bootstrapLookupMap  syncs.Map[string, bool]
 )
 
 var (
-	bootstrapDNSRequests = expvar.NewInt("counter_bootstrap_dns_requests")
-	publishedDNSHits     = expvar.NewInt("counter_bootstrap_dns_published_hits")
-	publishedDNSMisses   = expvar.NewInt("counter_bootstrap_dns_published_misses")
-	unpublishedDNSHits   = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
-	unpublishedDNSMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+	bootstrapDNSRequests        = expvar.NewInt("counter_bootstrap_dns_requests")
+	publishedDNSHits            = expvar.NewInt("counter_bootstrap_dns_published_hits")
+	publishedDNSMisses          = expvar.NewInt("counter_bootstrap_dns_published_misses")
+	unpublishedDNSHits          = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
+	unpublishedDNSMisses        = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+	unpublishedDNSPercentMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_percent_misses")
 )
 
 func init() {
@@ -59,15 +69,13 @@ func refreshBootstrapDNS() {
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-	dnsEntries := resolveList(ctx, strings.Split(*bootstrapDNS, ","))
+	dnsEntries := resolveList(ctx, *bootstrapDNS)
 	// Randomize the order of the IPs for each name to avoid the client biasing
 	// to IPv6
-	for k := range dnsEntries {
-		ips := dnsEntries[k]
-		slicesx.Shuffle(ips)
-		dnsEntries[k] = ips
+	for _, vv := range dnsEntries.IPs {
+		slicesx.Shuffle(vv)
 	}
-	j, err := json.MarshalIndent(dnsEntries, "", "\t")
+	j, err := json.MarshalIndent(dnsEntries.IPs, "", "\t")
 	if err != nil {
 		// leave the old values in place
 		return
@@ -81,27 +89,50 @@ func refreshUnpublishedDNS() {
 	if *unpublishedDNS == "" {
 		return
 	}
-
 	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-
-	dnsEntries := resolveList(ctx, strings.Split(*unpublishedDNS, ","))
+	dnsEntries := resolveList(ctx, *unpublishedDNS)
 	unpublishedDNSCache.Store(dnsEntries)
 }
 
-func resolveList(ctx context.Context, names []string) dnsEntryMap {
-	dnsEntries := make(dnsEntryMap)
+// resolveList takes a comma-separated list of DNS names to resolve.
+//
+// If an entry contains a slash, it's two DNS names: the first is the one to
+// resolve and the second is that of a TXT recording containing the rollout
+// percentage in range "0".."100". If the TXT record doesn't exist or is
+// malformed, the percentage is 0. If the TXT record is not provided (there's no
+// slash), then the percentage is 100.
+func resolveList(ctx context.Context, list string) *dnsEntryMap {
+	ents := strings.Split(list, ",")
+
+	ret := &dnsEntryMap{}
 
 	var r net.Resolver
-	for _, name := range names {
+	for _, ent := range ents {
+		name, txtName, _ := strings.Cut(ent, "/")
 		addrs, err := r.LookupIP(ctx, "ip", name)
 		if err != nil {
 			log.Printf("bootstrap DNS lookup %q: %v", name, err)
 			continue
 		}
-		dnsEntries[name] = addrs
+		mak.Set(&ret.IPs, name, addrs)
+
+		if txtName == "" {
+			mak.Set(&ret.Percent, name, 1.0)
+			continue
+		}
+		vals, err := r.LookupTXT(ctx, txtName)
+		if err != nil {
+			log.Printf("bootstrap DNS lookup %q: %v", txtName, err)
+			continue
+		}
+		for _, v := range vals {
+			if v, err := strconv.Atoi(v); err == nil && v >= 0 && v <= 100 {
+				mak.Set(&ret.Percent, name, float64(v)/100)
+			}
+		}
 	}
-	return dnsEntries
+	return ret
 }
 
 func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
@@ -115,22 +146,36 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	// Try answering a query from our hidden map first
 	if q := r.URL.Query().Get("q"); q != "" {
 		bootstrapLookupMap.Store(q, true)
-		if ips, ok := unpublishedDNSCache.Load()[q]; ok && len(ips) > 0 {
+		if bootstrapLookupMap.Len() > 500 { // defensive
+			bootstrapLookupMap.Clear()
+		}
+		if m := unpublishedDNSCache.Load(); m != nil && len(m.IPs[q]) > 0 {
 			unpublishedDNSHits.Add(1)
 
-			// Only return the specific query, not everything.
-			m := dnsEntryMap{q: ips}
-			j, err := json.MarshalIndent(m, "", "\t")
-			if err == nil {
-				w.Write(j)
-				return
+			percent := m.Percent[q]
+			if remoteAddrMatchesPercent(r.RemoteAddr, percent) {
+				// Only return the specific query, not everything.
+				m := map[string][]net.IP{q: m.IPs[q]}
+				j, err := json.MarshalIndent(m, "", "\t")
+				if err == nil {
+					w.Write(j)
+					return
+				}
+			} else {
+				unpublishedDNSPercentMisses.Add(1)
 			}
 		}
 
 		// If we have a "q" query for a name in the published cache
 		// list, then track whether that's a hit/miss.
-		if m, ok := dnsCache.Load()[q]; ok {
-			if len(m) > 0 {
+		m := dnsCache.Load()
+		var inPub bool
+		var ips []net.IP
+		if m != nil {
+			ips, inPub = m.IPs[q]
+		}
+		if inPub {
+			if len(ips) > 0 {
 				publishedDNSHits.Add(1)
 			} else {
 				publishedDNSMisses.Add(1)
@@ -146,3 +191,29 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	j := dnsCacheBytes.Load()
 	w.Write(j)
 }
+
+// percent is [0.0, 1.0].
+func remoteAddrMatchesPercent(remoteAddr string, percent float64) bool {
+	if percent == 0 {
+		return false
+	}
+	if percent == 1 {
+		return true
+	}
+	reqIPStr, _, err := net.SplitHostPort(remoteAddr)
+	if err != nil {
+		return false
+	}
+	reqIP, err := netip.ParseAddr(reqIPStr)
+	if err != nil {
+		return false
+	}
+	if reqIP.IsLoopback() {
+		// For local testing.
+		return rand.Float64() < 0.5
+	}
+	reqIP16 := reqIP.As16()
+	rndSrc := rand.NewPCG(binary.LittleEndian.Uint64(reqIP16[:8]), binary.LittleEndian.Uint64(reqIP16[8:]))
+	rnd := rand.New(rndSrc)
+	return percent > rnd.Float64()
+}

cmd/derper/bootstrap_dns_test.go

@@ -4,15 +4,19 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/netip"
 	"net/url"
 	"reflect"
 	"testing"
 
 	"tailscale.com/tstest"
+	"tailscale.com/tstest/nettest"
 )
 
 func BenchmarkHandleBootstrapDNS(b *testing.B) {
@@ -37,7 +41,7 @@ func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p),
 
 func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {}
 
-func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
+func getBootstrapDNS(t *testing.T, q string) map[string][]net.IP {
 	t.Helper()
 	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape(q), nil)
 	w := httptest.NewRecorder()
@@ -47,14 +51,17 @@ func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
 	if res.StatusCode != 200 {
 		t.Fatalf("got status=%d; want %d", res.StatusCode, 200)
 	}
-	var ips dnsEntryMap
-	if err := json.NewDecoder(res.Body).Decode(&ips); err != nil {
-		t.Fatalf("error decoding response body: %v", err)
+	var m map[string][]net.IP
+	var buf bytes.Buffer
+	if err := json.NewDecoder(io.TeeReader(res.Body, &buf)).Decode(&m); err != nil {
+		t.Fatalf("error decoding response body %q: %v", buf.Bytes(), err)
 	}
-	return ips
+	return m
 }
 
 func TestUnpublishedDNS(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
+
 	const published = "login.tailscale.com"
 	const unpublished = "log.tailscale.io"
 
@@ -104,15 +111,21 @@ func resetMetrics() {
 // Verify that we don't count an empty list in the unpublishedDNSCache as a
 // cache hit in our metrics.
 func TestUnpublishedDNSEmptyList(t *testing.T) {
-	pub := dnsEntryMap{
-		"tailscale.com": {net.IPv4(10, 10, 10, 10)},
+	pub := &dnsEntryMap{
+		IPs: map[string][]net.IP{"tailscale.com": {net.IPv4(10, 10, 10, 10)}},
 	}
 	dnsCache.Store(pub)
 	dnsCacheBytes.Store([]byte(`{"tailscale.com":["10.10.10.10"]}`))
 
-	unpublishedDNSCache.Store(dnsEntryMap{
-		"log.tailscale.io":           {},
-		"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+	unpublishedDNSCache.Store(&dnsEntryMap{
+		IPs: map[string][]net.IP{
+			"log.tailscale.io":           {},
+			"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+		},
+		Percent: map[string]float64{
+			"log.tailscale.io":           1.0,
+			"controlplane.tailscale.com": 1.0,
+		},
 	})
 
 	t.Run("CacheMiss", func(t *testing.T) {
@@ -122,8 +135,8 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 			ips := getBootstrapDNS(t, q)
 
 			// Expected our public map to be returned on a cache miss
-			if !reflect.DeepEqual(ips, pub) {
-				t.Errorf("got ips=%+v; want %+v", ips, pub)
+			if !reflect.DeepEqual(ips, pub.IPs) {
+				t.Errorf("got ips=%+v; want %+v", ips, pub.IPs)
 			}
 			if v := unpublishedDNSHits.Value(); v != 0 {
 				t.Errorf("got hits=%d; want 0", v)
@@ -138,7 +151,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 	t.Run("CacheHit", func(t *testing.T) {
 		resetMetrics()
 		ips := getBootstrapDNS(t, "controlplane.tailscale.com")
-		want := dnsEntryMap{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
+		want := map[string][]net.IP{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
 		if !reflect.DeepEqual(ips, want) {
 			t.Errorf("got ips=%+v; want %+v", ips, want)
 		}
@@ -163,3 +176,54 @@ func TestLookupMetric(t *testing.T) {
 		t.Errorf("bootstrapLookupMap.Len() want=5, got %v", bootstrapLookupMap.Len())
 	}
 }
+
+func TestRemoteAddrMatchesPercent(t *testing.T) {
+	tests := []struct {
+		remoteAddr string
+		percent    float64
+		want       bool
+	}{
+		// 0% and 100%.
+		{"10.0.0.1:1234", 0.0, false},
+		{"10.0.0.1:1234", 1.0, true},
+
+		// Invalid IP.
+		{"", 1.0, true},
+		{"", 0.0, false},
+		{"", 0.5, false},
+
+		// Small manual sample at 50%. The func uses a deterministic PRNG seed.
+		{"1.2.3.4:567", 0.5, true},
+		{"1.2.3.5:567", 0.5, true},
+		{"1.2.3.6:567", 0.5, false},
+		{"1.2.3.7:567", 0.5, true},
+		{"1.2.3.8:567", 0.5, false},
+		{"1.2.3.9:567", 0.5, true},
+		{"1.2.3.10:567", 0.5, true},
+	}
+	for _, tt := range tests {
+		got := remoteAddrMatchesPercent(tt.remoteAddr, tt.percent)
+		if got != tt.want {
+			t.Errorf("remoteAddrMatchesPercent(%q, %v) = %v; want %v", tt.remoteAddr, tt.percent, got, tt.want)
+		}
+	}
+
+	var match, all int
+	const wantPercent = 0.5
+	for a := range 256 {
+		for b := range 256 {
+			all++
+			if remoteAddrMatchesPercent(
+				netip.AddrPortFrom(netip.AddrFrom4([4]byte{1, 2, byte(a), byte(b)}), 12345).String(),
+				wantPercent) {
+				match++
+			}
+		}
+	}
+	gotPercent := float64(match) / float64(all)
+	const tolerance = 0.005
+	t.Logf("got percent %v (goal %v)", gotPercent, wantPercent)
+	if gotPercent < wantPercent-tolerance || gotPercent > wantPercent+tolerance {
+		t.Errorf("got %v; want %v ± %v", gotPercent, wantPercent, tolerance)
+	}
+}

cmd/derper/depaware.txt

@@ -20,7 +20,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         github.com/google/uuid                                       from tailscale.com/util/fastuuid
         github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
-   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/interfaces+
+   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
    L 💣 github.com/mdlayher/netlink                                  from github.com/google/nftables+
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
@@ -47,7 +47,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/netipx                                               from tailscale.com/net/tsaddr+
-   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
+   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         google.golang.org/protobuf/encoding/protodelim               from github.com/prometheus/common/expfmt
         google.golang.org/protobuf/encoding/prototext                from github.com/prometheus/common/expfmt+
         google.golang.org/protobuf/encoding/protowire                from google.golang.org/protobuf/encoding/protodelim+
@@ -89,18 +89,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/health                                         from tailscale.com/net/tlsdial
-        tailscale.com/hostinfo                                       from tailscale.com/net/interfaces+
+        tailscale.com/health                                         from tailscale.com/net/tlsdial+
+        tailscale.com/hostinfo                                       from tailscale.com/net/netmon+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
-     💣 tailscale.com/net/interfaces                                 from tailscale.com/net/netmon+
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
-        tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
+     💣 tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
         tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale
         tailscale.com/net/packet                                     from tailscale.com/wgengine/filter
@@ -117,7 +116,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
@@ -138,6 +137,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/client/tailscale+
         tailscale.com/types/views                                    from tailscale.com/ipn+
+        tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/net/netmon+
         tailscale.com/util/cloudenv                                  from tailscale.com/hostinfo+
    W    tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy
@@ -148,7 +148,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns
-        tailscale.com/util/mak                                       from tailscale.com/net/interfaces+
+        tailscale.com/util/mak                                       from tailscale.com/health+
         tailscale.com/util/multierr                                  from tailscale.com/health+
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
         tailscale.com/util/set                                       from tailscale.com/derp+
@@ -235,7 +235,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper+
+        flag                                                         from tailscale.com/cmd/derper
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -253,7 +253,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid
+        math/rand/v2                                                 from tailscale.com/util/fastuuid+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -277,7 +277,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         runtime/debug                                                from github.com/prometheus/client_golang/prometheus+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof
-        runtime/trace                                                from net/http/pprof+
+        runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/ipn/ipnstate+
         sort                                                         from compress/flate+
         strconv                                                      from compress/flate+
@@ -285,7 +285,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync                                                         from compress/flate+
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from runtime/pprof
         time                                                         from compress/gzip+
         unicode                                                      from bytes+

cmd/derper/derper.go

@@ -55,7 +55,7 @@ var (
 	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 	bootstrapDNS    = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
-	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list")
+	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
 	verifyClients   = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 	verifyClientURL = flag.String("verify-client-url", "", "if non-empty, an admission controller URL for permitting client connections; see tailcfg.DERPAdmitClientRequest")
 	verifyFailOpen  = flag.Bool("verify-client-url-fail-open", true, "whether we fail open if --verify-client-url is unreachable")
@@ -191,7 +191,12 @@ func main() {
 			http.Error(w, "derp server disabled", http.StatusNotFound)
 		}))
 	}
-	mux.HandleFunc("/derp/probe", probeHandler)
+
+	// These two endpoints are the same. Different versions of the clients
+	// have assumes different paths over time so we support both.
+	mux.HandleFunc("/derp/probe", derphttp.ProbeHandler)
+	mux.HandleFunc("/derp/latency-check", derphttp.ProbeHandler)
+
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", tsweb.BrowserHeaderHandlerFunc(handleBootstrapDNS))
 	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -370,17 +375,6 @@ func isChallengeChar(c rune) bool {
 		c == '.' || c == '-' || c == '_'
 }
 
-// probeHandler is the endpoint that js/wasm clients hit to measure
-// DERP latency, since they can't do UDP STUN queries.
-func probeHandler(w http.ResponseWriter, r *http.Request) {
-	switch r.Method {
-	case "HEAD", "GET":
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-	default:
-		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
-	}
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -99,6 +99,7 @@ func TestNoContent(t *testing.T) {
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
+			"testing":                            "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/buffer":       "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",

cmd/derper/mesh.go

@@ -15,6 +15,7 @@ import (
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
+	"tailscale.com/net/netmon"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
@@ -36,7 +37,8 @@ func startMesh(s *derp.Server) error {
 
 func startMeshWithHost(s *derp.Server, host string) error {
 	logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
-	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf)
+	netMon := netmon.NewStatic() // good enough for cmd/derper; no need for netns fanciness
+	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf, netMon)
 	if err != nil {
 		return err
 	}

cmd/k8s-nameserver/main.go

@@ -0,0 +1,354 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// k8s-nameserver is a simple nameserver implementation meant to be used with
+// k8s-operator to allow to resolve magicDNS names associated with tailnet
+// proxies in cluster.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/miekg/dns"
+	operatorutils "tailscale.com/k8s-operator"
+	"tailscale.com/util/dnsname"
+)
+
+const (
+	// tsNetDomain is the domain that this DNS nameserver has registered a handler for.
+	tsNetDomain = "ts.net"
+	// addr is the the address that the UDP and TCP listeners will listen on.
+	addr = ":1053"
+
+	// The following constants are specific to the nameserver configuration
+	// provided by a mounted Kubernetes Configmap. The Configmap mounted at
+	// /config is the only supported way for configuring this nameserver.
+	defaultDNSConfigDir    = "/config"
+	kubeletMountedConfigLn = "..data"
+)
+
+// nameserver is a simple nameserver that responds to DNS queries for A records
+// for ts.net domain names over UDP or TCP. It serves DNS responses from
+// in-memory IPv4 host records. It is intended to be deployed on Kubernetes with
+// a ConfigMap mounted at /config that should contain the host records. It
+// dynamically reconfigures its in-memory mappings as the contents of the
+// mounted ConfigMap changes.
+type nameserver struct {
+	// configReader returns the latest desired configuration (host records)
+	// for the nameserver. By default it gets set to a reader that reads
+	// from a Kubernetes ConfigMap mounted at /config, but this can be
+	// overridden in tests.
+	configReader configReaderFunc
+	// configWatcher is a watcher that returns an event when the desired
+	// configuration has changed and the nameserver should update the
+	// in-memory records.
+	configWatcher <-chan string
+
+	mu sync.Mutex // protects following
+	// ip4 are the in-memory hostname -> IP4 mappings that the nameserver
+	// uses to respond to A record queries.
+	ip4 map[dnsname.FQDN][]net.IP
+}
+
+func main() {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Ensure that we watch the kube Configmap mounted at /config for
+	// nameserver configuration updates and send events when updates happen.
+	c := ensureWatcherForKubeConfigMap(ctx)
+
+	ns := &nameserver{
+		configReader:  configMapConfigReader,
+		configWatcher: c,
+	}
+
+	// Ensure that in-memory records get set up to date now and will get
+	// reset when the configuration changes.
+	ns.runRecordsReconciler(ctx)
+
+	// Register a DNS server handle for ts.net domain names. Not having a
+	// handle registered for any other domain names is how we enforce that
+	// this nameserver can only be used for ts.net domains - querying any
+	// other domain names returns Rcode Refused.
+	dns.HandleFunc(tsNetDomain, ns.handleFunc())
+
+	// Listen for DNS queries over UDP and TCP.
+	udpSig := make(chan os.Signal)
+	tcpSig := make(chan os.Signal)
+	go listenAndServe("udp", addr, udpSig)
+	go listenAndServe("tcp", addr, tcpSig)
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
+	s := <-sig
+	log.Printf("OS signal (%s) received, shutting down", s)
+	cancel()    // exit the records reconciler and configmap watcher goroutines
+	udpSig <- s // stop the UDP listener
+	tcpSig <- s // stop the TCP listener
+}
+
+// handleFunc is a DNS query handler that can respond to A record queries from
+// the nameserver's in-memory records.
+// - If an A record query is received and the
+// nameserver's in-memory records contain records for the queried domain name,
+// return a success response.
+// - If an A record query is received, but the
+// nameserver's in-memory records do not contain records for the queried domain name,
+// return NXDOMAIN.
+// - If an A record query is received, but the queried domain name is not valid, return Format Error.
+// - If a query is received for any other record type than A, return Not Implemented.
+func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) {
+	h := func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		defer func() {
+			w.WriteMsg(m)
+		}()
+		if len(r.Question) < 1 {
+			log.Print("[unexpected] nameserver received a request with no questions")
+			m = r.SetRcodeFormatError(r)
+			return
+		}
+		// TODO (irbekrm): maybe set message compression
+		switch r.Question[0].Qtype {
+		case dns.TypeA:
+			q := r.Question[0].Name
+			fqdn, err := dnsname.ToFQDN(q)
+			if err != nil {
+				m = r.SetRcodeFormatError(r)
+				return
+			}
+			// The only supported use of this nameserver is as a
+			// single source of truth for MagicDNS names by
+			// non-tailnet Kubernetes workloads.
+			m.Authoritative = true
+			m.RecursionAvailable = false
+
+			ips := n.lookupIP4(fqdn)
+			if ips == nil || len(ips) == 0 {
+				// As we are the authoritative nameserver for MagicDNS
+				// names, if we do not have a record for this MagicDNS
+				// name, it does not exist.
+				m = m.SetRcode(r, dns.RcodeNameError)
+				return
+			}
+			// TODO (irbekrm): TTL is currently set to 0, meaning
+			// that cluster workloads will not cache the DNS
+			// records. Revisit this in future when we understand
+			// the usage patterns better- is it putting too much
+			// load on kube DNS server or is this fine?
+			for _, ip := range ips {
+				rr := &dns.A{Hdr: dns.RR_Header{Name: q, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0}, A: ip}
+				m.SetRcode(r, dns.RcodeSuccess)
+				m.Answer = append(m.Answer, rr)
+			}
+		case dns.TypeAAAA:
+			// TODO (irbekrm): implement IPv6 support.
+			// Kubernetes distributions that I am most familiar with
+			// default to IPv4 for Pod CIDR ranges and often many cases don't
+			// support IPv6 at all, so this should not be crucial for now.
+			fallthrough
+		default:
+			log.Printf("[unexpected] nameserver received a query for an unsupported record type: %s", r.Question[0].String())
+			m.SetRcode(r, dns.RcodeNotImplemented)
+		}
+	}
+	return h
+}
+
+// runRecordsReconciler ensures that nameserver's in-memory records are
+// reset when the provided configuration changes.
+func (n *nameserver) runRecordsReconciler(ctx context.Context) {
+	log.Print("updating nameserver's records from the provided configuration...")
+	if err := n.resetRecords(); err != nil { // ensure records are up to date before the nameserver starts
+		log.Fatalf("error setting nameserver's records: %v", err)
+	}
+	log.Print("nameserver's records were updated")
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				log.Printf("context cancelled, exiting records reconciler")
+				return
+			case <-n.configWatcher:
+				log.Print("configuration update detected, resetting records")
+				if err := n.resetRecords(); err != nil {
+					// TODO (irbekrm): this runs in a
+					// container that will be thrown away,
+					// so this should be ok. But maybe still
+					// need to ensure that the DNS server
+					// terminates connections more
+					// gracefully.
+					log.Fatalf("error resetting records: %v", err)
+				}
+				log.Print("nameserver records were reset")
+			}
+		}
+	}()
+}
+
+// resetRecords sets the in-memory DNS records of this nameserver from the
+// provided configuration. It does not check for the diff, so the caller is
+// expected to ensure that this is only called when reset is needed.
+func (n *nameserver) resetRecords() error {
+	dnsCfgBytes, err := n.configReader()
+	if err != nil {
+		log.Printf("error reading nameserver's configuration: %v", err)
+		return err
+	}
+	if dnsCfgBytes == nil || len(dnsCfgBytes) < 1 {
+		log.Print("nameserver's configuration is empty, any in-memory records will be unset")
+		n.mu.Lock()
+		n.ip4 = make(map[dnsname.FQDN][]net.IP)
+		n.mu.Unlock()
+		return nil
+	}
+	dnsCfg := &operatorutils.Records{}
+	err = json.Unmarshal(dnsCfgBytes, dnsCfg)
+	if err != nil {
+		return fmt.Errorf("error unmarshalling nameserver configuration: %v\n", err)
+	}
+
+	if dnsCfg.Version != operatorutils.Alpha1Version {
+		return fmt.Errorf("unsupported configuration version %s, supported versions are %s\n", dnsCfg.Version, operatorutils.Alpha1Version)
+	}
+
+	ip4 := make(map[dnsname.FQDN][]net.IP)
+	defer func() {
+		n.mu.Lock()
+		defer n.mu.Unlock()
+		n.ip4 = ip4
+	}()
+
+	if len(dnsCfg.IP4) == 0 {
+		log.Print("nameserver's configuration contains no records, any in-memory records will be unset")
+		return nil
+	}
+
+	for fqdn, ips := range dnsCfg.IP4 {
+		fqdn, err := dnsname.ToFQDN(fqdn)
+		if err != nil {
+			log.Printf("invalid nameserver's configuration: %s is not a valid FQDN: %v; skipping this record", fqdn, err)
+			continue // one invalid hostname should not break the whole nameserver
+		}
+		for _, ipS := range ips {
+			ip := net.ParseIP(ipS).To4()
+			if ip == nil { // To4 returns nil if IP is not a IPv4 address
+				log.Printf("invalid nameserver's configuration: %v does not appear to be an IPv4 address; skipping this record", ipS)
+				continue // one invalid IP address should not break the whole nameserver
+			}
+			ip4[fqdn] = []net.IP{ip}
+		}
+	}
+	return nil
+}
+
+// listenAndServe starts a DNS server for the provided network and address.
+func listenAndServe(net, addr string, shutdown chan os.Signal) {
+	s := &dns.Server{Addr: addr, Net: net}
+	go func() {
+		<-shutdown
+		log.Printf("shutting down server for %s", net)
+		s.Shutdown()
+	}()
+	log.Printf("listening for %s queries on %s", net, addr)
+	if err := s.ListenAndServe(); err != nil {
+		log.Fatalf("error running %s server: %v", net, err)
+	}
+}
+
+// ensureWatcherForKubeConfigMap sets up a new file watcher for the ConfigMap
+// that's expected to be mounted at /config. Returns a channel that receives an
+// event every time the contents get updated.
+func ensureWatcherForKubeConfigMap(ctx context.Context) chan string {
+	c := make(chan string)
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatalf("error creating a new watcher for the mounted ConfigMap: %v", err)
+	}
+	// kubelet mounts configmap to a Pod using a series of symlinks, one of
+	// which is <mount-dir>/..data that Kubernetes recommends consumers to
+	// use if they need to monitor changes
+	// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
+	toWatch := filepath.Join(defaultDNSConfigDir, kubeletMountedConfigLn)
+	go func() {
+		defer watcher.Close()
+		log.Printf("starting file watch for %s", defaultDNSConfigDir)
+		for {
+			select {
+			case <-ctx.Done():
+				log.Print("context cancelled, exiting ConfigMap watcher")
+				return
+			case event, ok := <-watcher.Events:
+				if !ok {
+					log.Fatal("watcher finished; exiting")
+				}
+				if event.Name == toWatch {
+					msg := fmt.Sprintf("ConfigMap update received: %s", event)
+					log.Print(msg)
+					c <- msg
+				}
+			case err, ok := <-watcher.Errors:
+				if err != nil {
+					// TODO (irbekrm): this runs in a
+					// container that will be thrown away,
+					// so this should be ok. But maybe still
+					// need to ensure that the DNS server
+					// terminates connections more
+					// gracefully.
+					log.Fatalf("[unexpected] error watching configuration: %v", err)
+				}
+				if !ok {
+					// TODO (irbekrm): this runs in a
+					// container that will be thrown away,
+					// so this should be ok. But maybe still
+					// need to ensure that the DNS server
+					// terminates connections more
+					// gracefully.
+					log.Fatalf("[unexpected] errors watcher exited")
+				}
+			}
+		}
+	}()
+	if err = watcher.Add(defaultDNSConfigDir); err != nil {
+		log.Fatalf("failed setting up a watcher for the mounted ConfigMap: %v", err)
+	}
+	return c
+}
+
+// configReaderFunc is a function that returns the desired nameserver configuration.
+type configReaderFunc func() ([]byte, error)
+
+// configMapConfigReader reads the desired nameserver configuration from a
+// records.json file in a ConfigMap mounted at /config.
+var configMapConfigReader configReaderFunc = func() ([]byte, error) {
+	if contents, err := os.ReadFile(filepath.Join(defaultDNSConfigDir, operatorutils.DNSRecordsCMKey)); err == nil {
+		return contents, nil
+	} else if os.IsNotExist(err) {
+		return nil, nil
+	} else {
+		return nil, err
+	}
+}
+
+// lookupIP4 returns any IPv4 addresses for the given FQDN from nameserver's
+// in-memory records.
+func (n *nameserver) lookupIP4(fqdn dnsname.FQDN) []net.IP {
+	if n.ip4 == nil {
+		return nil
+	}
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	f := n.ip4[fqdn]
+	return f
+}

cmd/k8s-nameserver/main_test.go

@@ -0,0 +1,227 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"net"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/miekg/dns"
+	"tailscale.com/util/dnsname"
+)
+
+func TestNameserver(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		ip4      map[dnsname.FQDN][]net.IP
+		query    *dns.Msg
+		wantResp *dns.Msg
+	}{
+		{
+			name: "A record query, record exists",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}},
+				MsgHdr:   dns.MsgHdr{Id: 1, RecursionDesired: true},
+			},
+			wantResp: &dns.Msg{
+				Answer: []dns.RR{&dns.A{Hdr: dns.RR_Header{
+					Name: "foo.bar.com", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0},
+					A: net.IP{1, 2, 3, 4}}},
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}},
+				MsgHdr: dns.MsgHdr{
+					Id:                 1,
+					Rcode:              dns.RcodeSuccess,
+					RecursionAvailable: false,
+					RecursionDesired:   true,
+					Response:           true,
+					Opcode:             dns.OpcodeQuery,
+					Authoritative:      true,
+				}},
+		},
+		{
+			name: "A record query, record does not exist",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}},
+				MsgHdr: dns.MsgHdr{
+					Id:                 1,
+					Rcode:              dns.RcodeNameError,
+					RecursionAvailable: false,
+					Response:           true,
+					Opcode:             dns.OpcodeQuery,
+					Authoritative:      true,
+				}},
+		},
+		{
+			name: "A record query, but the name is not a valid FQDN",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}},
+				MsgHdr: dns.MsgHdr{
+					Id:       1,
+					Rcode:    dns.RcodeFormatError,
+					Response: true,
+					Opcode:   dns.OpcodeQuery,
+				}},
+		},
+		{
+			name: "AAAA record query",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr: dns.MsgHdr{
+					Id:       1,
+					Rcode:    dns.RcodeNotImplemented,
+					Response: true,
+					Opcode:   dns.OpcodeQuery,
+				}},
+		},
+		{
+			name: "AAAA record query",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				MsgHdr: dns.MsgHdr{
+					Id:       1,
+					Rcode:    dns.RcodeNotImplemented,
+					Response: true,
+					Opcode:   dns.OpcodeQuery,
+				}},
+		},
+		{
+			name: "CNAME record query",
+			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
+			query: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}},
+				MsgHdr:   dns.MsgHdr{Id: 1},
+			},
+			wantResp: &dns.Msg{
+				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}},
+				MsgHdr: dns.MsgHdr{
+					Id:       1,
+					Rcode:    dns.RcodeNotImplemented,
+					Response: true,
+					Opcode:   dns.OpcodeQuery,
+				}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ns := &nameserver{
+				ip4: tt.ip4,
+			}
+			handler := ns.handleFunc()
+			fakeRespW := &fakeResponseWriter{}
+			handler(fakeRespW, tt.query)
+			if diff := cmp.Diff(*fakeRespW.msg, *tt.wantResp); diff != "" {
+				t.Fatalf("unexpected response (-got +want): \n%s", diff)
+			}
+		})
+	}
+}
+
+func TestResetRecords(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   []byte
+		hasIp4   map[dnsname.FQDN][]net.IP
+		wantsIp4 map[dnsname.FQDN][]net.IP
+		wantsErr bool
+	}{
+		{
+			name:     "previously empty nameserver.ip4 gets set",
+			config:   []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
+			wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}},
+		},
+		{
+			name:     "nameserver.ip4 gets reset",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			config:   []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
+			wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}},
+		},
+		{
+			name:     "configuration with incompatible version",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			config:   []byte(`{"version": "v1beta1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
+			wantsIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			wantsErr: true,
+		},
+		{
+			name:     "nameserver.ip4 gets reset to empty config when no configuration is provided",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			wantsIp4: make(map[dnsname.FQDN][]net.IP),
+		},
+		{
+			name:     "nameserver.ip4 gets reset to empty config when the provided configuration is empty",
+			hasIp4:   map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
+			config:   []byte(`{"version": "v1alpha1", "ip4": {}}`),
+			wantsIp4: make(map[dnsname.FQDN][]net.IP),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ns := &nameserver{
+				ip4:          tt.hasIp4,
+				configReader: func() ([]byte, error) { return tt.config, nil },
+			}
+			if err := ns.resetRecords(); err == nil == tt.wantsErr {
+				t.Errorf("resetRecords() returned err: %v, wantsErr: %v", err, tt.wantsErr)
+			}
+			if diff := cmp.Diff(ns.ip4, tt.wantsIp4); diff != "" {
+				t.Fatalf("unexpected nameserver.ip4 contents (-got +want): \n%s", diff)
+			}
+		})
+	}
+}
+
+// fakeResponseWriter is a faked out dns.ResponseWriter that can be used in
+// tests that need to read the response message that was written.
+type fakeResponseWriter struct {
+	msg *dns.Msg
+}
+
+var _ dns.ResponseWriter = &fakeResponseWriter{}
+
+func (fr *fakeResponseWriter) WriteMsg(msg *dns.Msg) error {
+	fr.msg = msg
+	return nil
+}
+func (fr *fakeResponseWriter) LocalAddr() net.Addr {
+	return nil
+}
+func (fr *fakeResponseWriter) RemoteAddr() net.Addr {
+	return nil
+}
+func (fr *fakeResponseWriter) Write([]byte) (int, error) {
+	return 0, nil
+}
+func (fr *fakeResponseWriter) Close() error {
+	return nil
+}
+func (fr *fakeResponseWriter) TsigStatus() error {
+	return nil
+}
+func (fr *fakeResponseWriter) TsigTimersOnly(bool) {}
+func (fr *fakeResponseWriter) Hijack()             {}

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -21,6 +21,9 @@ spec:
       {{- end }}
       labels:
         app: operator
+        {{- with .Values.operatorConfig.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -24,6 +24,9 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["dnsconfigs", "dnsconfigs/status"]
+  verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -45,11 +48,14 @@ metadata:
   namespace: {{ .Release.Namespace }}
 rules:
 - apiGroups: [""]
-  resources: ["secrets"]
+  resources: ["secrets", "serviceaccounts", "configmaps"]
   verbs: ["*"]
 - apiGroups: ["apps"]
-  resources: ["statefulsets"]
+  resources: ["statefulsets", "deployments"]
   verbs: ["*"]
+- apiGroups: ["discovery.k8s.io"]
+  resources: ["endpointslices"]
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

cmd/k8s-operator/deploy/chart/values.yaml

@@ -37,6 +37,7 @@ operatorConfig:
   resources: {}
 
   podAnnotations: {}
+  podLabels: {}
 
   tolerations: []
 
@@ -50,6 +51,10 @@ operatorConfig:
 # proxies created by the operator.
 # https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
 # https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
+# Note that this section contains only a few global configuration options and
+# will not be updated with more configuration options in the future.
+# If you need more configuration options, take a look at ProxyClass:
+# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     repo: tailscale/tailscale

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -0,0 +1,96 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.13.0
+  name: dnsconfigs.tailscale.com
+spec:
+  group: tailscale.com
+  names:
+    kind: DNSConfig
+    listKind: DNSConfigList
+    plural: dnsconfigs
+    shortNames:
+      - dc
+    singular: dnsconfig
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Service IP address of the nameserver
+          jsonPath: .status.nameserver.ip
+          name: NameserverIP
+          type: string
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              type: object
+              required:
+                - nameserver
+              properties:
+                nameserver:
+                  type: object
+                  properties:
+                    image:
+                      type: object
+                      properties:
+                        repo:
+                          type: string
+                        tag:
+                          type: string
+            status:
+              type: object
+              properties:
+                conditions:
+                  type: array
+                  items:
+                    description: ConnectorCondition contains condition information for a Connector.
+                    type: object
+                    required:
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                        type: string
+                        format: date-time
+                      message:
+                        description: Message is a human readable description of the details of the last transition, complementing reason.
+                        type: string
+                      observedGeneration:
+                        description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                        type: integer
+                        format: int64
+                      reason:
+                        description: Reason is a brief machine readable explanation for the condition's last transition.
+                        type: string
+                      status:
+                        description: Status of the condition, one of ('True', 'False', 'Unknown').
+                        type: string
+                      type:
+                        description: Type of the condition, known values are (`SubnetRouterReady`).
+                        type: string
+                  x-kubernetes-list-map-keys:
+                    - type
+                  x-kubernetes-list-type: map
+                nameserver:
+                  type: object
+                  properties:
+                    ip:
+                      type: string
+      served: true
+      storage: true
+      subresources:
+        status: {}

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -37,9 +37,16 @@ spec:
             spec:
               description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
-              required:
-                - statefulSet
               properties:
+                metrics:
+                  description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. Note that the metrics are currently considered unstable and will likely change in breaking ways in the future - we only recommend that you use those for debugging purposes.
+                  type: object
+                  required:
+                    - enable
+                  properties:
+                    enable:
+                      description: Setting enable to true will make the proxy serve Tailscale metrics at <pod-ip>:9001/debug/metrics. Defaults to false.
+                      type: boolean
                 statefulSet:
                   description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).
                   type: object

cmd/k8s-operator/deploy/examples/dnsconfig.yaml

@@ -0,0 +1,9 @@
+apiVersion: tailscale.com/v1alpha1
+kind: DNSConfig 
+metadata:
+  name: ts-dns
+spec:
+  nameserver:
+    image:
+      repo: tailscale/k8s-nameserver
+      tag: unstable-v1.65

cmd/k8s-operator/deploy/examples/proxyclass.yaml

@@ -3,13 +3,15 @@ kind: ProxyClass
 metadata:
   name: prod
 spec:
+  metrics:
+    enable: true
   statefulSet:
     annotations:
-      platform-component: infra 
+      platform-component: infra
     pod:
       labels:
         team: eng
       nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
       imagePullSecrets:
       - name: "foo"

cmd/k8s-operator/deploy/manifests/nameserver/cm.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dnsrecords

cmd/k8s-operator/deploy/manifests/nameserver/deploy.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nameserver 
+spec:
+  replicas: 1
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels:
+      app: nameserver
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: nameserver
+    spec:
+      containers:
+      - imagePullPolicy: IfNotPresent
+        name: nameserver
+        ports:
+        - name: tcp
+          protocol: TCP
+          containerPort: 1053
+        - name: udp
+          protocol: UDP
+          containerPort: 1053
+        volumeMounts:
+        - name: dnsrecords
+          mountPath: /config
+      restartPolicy: Always
+      serviceAccount: nameserver
+      serviceAccountName: nameserver
+      volumes:
+      - name: dnsrecords
+        configMap:
+          name: dnsrecords

cmd/k8s-operator/deploy/manifests/nameserver/sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nameserver

cmd/k8s-operator/deploy/manifests/nameserver/svc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nameserver
+spec:
+  selector:
+    app: nameserver
+  ports:
+  - name: udp
+    targetPort: 1053
+    port: 53
+    protocol: UDP
+  - name: tcp
+    targetPort: 1053
+    port: 53
+    protocol: TCP 

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -159,6 +159,103 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+    annotations:
+        controller-gen.kubebuilder.io/version: v0.13.0
+    name: dnsconfigs.tailscale.com
+spec:
+    group: tailscale.com
+    names:
+        kind: DNSConfig
+        listKind: DNSConfigList
+        plural: dnsconfigs
+        shortNames:
+            - dc
+        singular: dnsconfig
+    scope: Cluster
+    versions:
+        - additionalPrinterColumns:
+            - description: Service IP address of the nameserver
+              jsonPath: .status.nameserver.ip
+              name: NameserverIP
+              type: string
+          name: v1alpha1
+          schema:
+            openAPIV3Schema:
+                properties:
+                    apiVersion:
+                        description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+                        type: string
+                    kind:
+                        description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                        type: string
+                    metadata:
+                        type: object
+                    spec:
+                        properties:
+                            nameserver:
+                                properties:
+                                    image:
+                                        properties:
+                                            repo:
+                                                type: string
+                                            tag:
+                                                type: string
+                                        type: object
+                                type: object
+                        required:
+                            - nameserver
+                        type: object
+                    status:
+                        properties:
+                            conditions:
+                                items:
+                                    description: ConnectorCondition contains condition information for a Connector.
+                                    properties:
+                                        lastTransitionTime:
+                                            description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                                            format: date-time
+                                            type: string
+                                        message:
+                                            description: Message is a human readable description of the details of the last transition, complementing reason.
+                                            type: string
+                                        observedGeneration:
+                                            description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                                            format: int64
+                                            type: integer
+                                        reason:
+                                            description: Reason is a brief machine readable explanation for the condition's last transition.
+                                            type: string
+                                        status:
+                                            description: Status of the condition, one of ('True', 'False', 'Unknown').
+                                            type: string
+                                        type:
+                                            description: Type of the condition, known values are (`SubnetRouterReady`).
+                                            type: string
+                                    required:
+                                        - status
+                                        - type
+                                    type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                    - type
+                                x-kubernetes-list-type: map
+                            nameserver:
+                                properties:
+                                    ip:
+                                        type: string
+                                type: object
+                        type: object
+                required:
+                    - spec
+                type: object
+          served: true
+          storage: true
+          subresources:
+            status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
     annotations:
         controller-gen.kubebuilder.io/version: v0.13.0
@@ -193,6 +290,15 @@ spec:
                     spec:
                         description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
+                            metrics:
+                                description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. Note that the metrics are currently considered unstable and will likely change in breaking ways in the future - we only recommend that you use those for debugging purposes.
+                                properties:
+                                    enable:
+                                        description: Setting enable to true will make the proxy serve Tailscale metrics at <pod-ip>:9001/debug/metrics. Defaults to false.
+                                        type: boolean
+                                required:
+                                    - enable
+                                type: object
                             statefulSet:
                                 description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).
                                 properties:
@@ -1157,8 +1263,6 @@ spec:
                                                 type: array
                                         type: object
                                 type: object
-                        required:
-                            - statefulSet
                         type: object
                     status:
                         description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
@@ -1245,6 +1349,16 @@ rules:
         - list
         - watch
         - update
+    - apiGroups:
+        - tailscale.com
+      resources:
+        - dnsconfigs
+        - dnsconfigs/status
+      verbs:
+        - get
+        - list
+        - watch
+        - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -1269,14 +1383,25 @@ rules:
         - ""
       resources:
         - secrets
+        - serviceaccounts
+        - configmaps
       verbs:
         - '*'
     - apiGroups:
         - apps
       resources:
         - statefulsets
+        - deployments
       verbs:
         - '*'
+    - apiGroups:
+        - discovery.k8s.io
+      resources:
+        - endpointslices
+      verbs:
+        - get
+        - list
+        - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -14,10 +14,8 @@ spec:
         - name: sysctler
           securityContext:
             privileged: true
-          command: ["/bin/sh"]
-          args:
-            - -c
-            - sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1
+          command: ["/bin/sh", "-c"]
+          args: [sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi]
       resources:
         requests:
           cpu: 1m

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -20,3 +20,7 @@ spec:
           env:
             - name: TS_USERSPACE
               value: "true"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP

cmd/k8s-operator/dnsrecords.go

@@ -0,0 +1,337 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// tailscale-operator provides a way to expose services running in a Kubernetes
+// cluster to your Tailnet and to make Tailscale nodes available to cluster
+// workloads
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/net"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	operatorutils "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/util/mak"
+)
+
+const (
+	dnsRecordsRecocilerFinalizer = "tailscale.com/dns-records-reconciler"
+	annotationTSMagicDNSName     = "tailscale.com/magic-dnsname"
+)
+
+// dnsRecordsReconciler knows how to update dnsrecords ConfigMap with DNS
+// records.
+// The records that it creates are:
+//   - For tailscale Ingress, a mapping of the Ingress's MagicDNSName to the IP address of
+//     the ingress proxy Pod.
+//   - For egress proxies configured via tailscale.com/tailnet-fqdn annotation, a
+//     mapping of the tailnet FQDN to the IP address of the egress proxy Pod.
+//
+// Records will only be created if there is exactly one ready
+// tailscale.com/v1alpha1.DNSConfig instance in the cluster (so that we know
+// that there is a ts.net nameserver deployed in the cluster).
+type dnsRecordsReconciler struct {
+	client.Client
+	tsNamespace           string // namespace in which we provision tailscale resources
+	logger                *zap.SugaredLogger
+	isDefaultLoadBalancer bool // true if operator is the default ingress controller in this cluster
+}
+
+// Reconcile takes a reconcile.Request for a headless Service fronting a
+// tailscale proxy and updates DNS Records in dnsrecords ConfigMap for the
+// in-cluster ts.net nameserver if required.
+func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	logger := dnsRR.logger.With("Service", req.NamespacedName)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	headlessSvc := new(corev1.Service)
+	err = dnsRR.Client.Get(ctx, req.NamespacedName, headlessSvc)
+	if apierrors.IsNotFound(err) {
+		logger.Debugf("Service not found")
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get Service: %w", err)
+	}
+	if !(isManagedByType(headlessSvc, "svc") || isManagedByType(headlessSvc, "ingress")) {
+		logger.Debugf("Service is not a headless Service for a tailscale ingress or egress proxy; do nothing")
+		return reconcile.Result{}, nil
+	}
+
+	if !headlessSvc.DeletionTimestamp.IsZero() {
+		logger.Debug("Service is being deleted, clean up resources")
+		return reconcile.Result{}, dnsRR.maybeCleanup(ctx, headlessSvc, logger)
+	}
+
+	// Check that there is a ts.net nameserver deployed to the cluster by
+	// checking that there is tailscale.com/v1alpha1.DNSConfig resource in a
+	// Ready state.
+	dnsCfgLst := new(tsapi.DNSConfigList)
+	if err = dnsRR.List(ctx, dnsCfgLst); err != nil {
+		return reconcile.Result{}, fmt.Errorf("error listing DNSConfigs: %w", err)
+	}
+	if len(dnsCfgLst.Items) == 0 {
+		logger.Debugf("DNSConfig does not exist, not creating DNS records")
+		return reconcile.Result{}, nil
+	}
+	if len(dnsCfgLst.Items) > 1 {
+		logger.Errorf("Invalid cluster state - more than one DNSConfig found in cluster. Please ensure no more than one exists")
+		return reconcile.Result{}, nil
+	}
+	dnsCfg := dnsCfgLst.Items[0]
+	if !operatorutils.DNSCfgIsReady(&dnsCfg) {
+		logger.Info("DNSConfig is not ready yet, waiting...")
+		return reconcile.Result{}, nil
+	}
+
+	return reconcile.Result{}, dnsRR.maybeProvision(ctx, headlessSvc, logger)
+}
+
+// maybeProvision ensures that dnsrecords ConfigMap contains a record for the
+// proxy associated with the headless Service.
+// The record is only provisioned if the proxy is for a tailscale Ingress or
+// egress configured via tailscale.com/tailnet-fqdn annotation.
+//
+// For Ingress, the record is a mapping between the MagicDNSName of the Ingress, retrieved from
+// ingress.status.loadBalancer.ingress.hostname field and the proxy Pod IP addresses
+// retrieved from the EndpoinSlice associated with this headless Service, i.e
+// Records{IP4: <MagicDNS name of the Ingress>: <[IPs of the ingress proxy Pods]>}
+//
+// For egress, the record is a mapping between tailscale.com/tailnet-fqdn
+// annotation and the proxy Pod IP addresses, retrieved from the EndpointSlice
+// associated with this headless Service, i.e
+// Records{IP4: {<tailscale.com/tailnet-fqdn>: <[IPs of the egress proxy Pods]>}
+//
+// If records need to be created for this proxy, maybeProvision will also:
+// - update the headless Service with a tailscale.com/magic-dnsname annotation
+// - update the headless Service with a finalizer
+func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error {
+	if headlessSvc == nil {
+		logger.Info("[unexpected] maybeProvision called with a nil Service")
+		return nil
+	}
+	isEgressFQDNSvc, err := dnsRR.isSvcForFQDNEgressProxy(ctx, headlessSvc)
+	if err != nil {
+		return fmt.Errorf("error checking whether the Service is for an egress proxy: %w", err)
+	}
+	if !(isEgressFQDNSvc || isManagedByType(headlessSvc, "ingress")) {
+		logger.Debug("Service is not fronting a proxy that we create DNS records for; do nothing")
+		return nil
+	}
+	fqdn, err := dnsRR.fqdnForDNSRecord(ctx, headlessSvc, logger)
+	if err != nil {
+		return fmt.Errorf("error determining DNS name for record: %w", err)
+	}
+	if fqdn == "" {
+		logger.Debugf("MagicDNS name does not (yet) exist, not provisioning DNS record")
+		return nil // a new reconcile will be triggered once it's added
+	}
+
+	oldHeadlessSvc := headlessSvc.DeepCopy()
+	// Ensure that headless Service is annotated with a finalizer to help
+	// with records cleanup when proxy resources are deleted.
+	if !slices.Contains(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) {
+		headlessSvc.Finalizers = append(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
+	}
+	// Ensure that headless Service is annotated with the current MagicDNS
+	// name to help with records cleanup when proxy resources are deleted or
+	// MagicDNS name changes.
+	oldFqdn := headlessSvc.Annotations[annotationTSMagicDNSName]
+	if oldFqdn != "" && oldFqdn != fqdn { // i.e user has changed the value of tailscale.com/tailnet-fqdn annotation
+		logger.Debugf("MagicDNS name has changed, remvoving record for %s", oldFqdn)
+		updateFunc := func(rec *operatorutils.Records) {
+			delete(rec.IP4, oldFqdn)
+		}
+		if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
+			return fmt.Errorf("error removing record for %s: %w", oldFqdn, err)
+		}
+	}
+	mak.Set(&headlessSvc.Annotations, annotationTSMagicDNSName, fqdn)
+	if !apiequality.Semantic.DeepEqual(oldHeadlessSvc, headlessSvc) {
+		logger.Infof("provisioning DNS record for MagicDNS name: %s", fqdn) // this will be printed exactly once
+		if err := dnsRR.Update(ctx, headlessSvc); err != nil {
+			return fmt.Errorf("error updating proxy headless Service metadata: %w", err)
+		}
+	}
+
+	// Get the Pod IP addresses for the proxy from the EndpointSlice for the
+	// headless Service.
+	labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
+	eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
+	if err != nil {
+		return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
+	}
+	if eps == nil {
+		logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
+		return nil
+	}
+	// An EndpointSlice for a Service can have a list of endpoints that each
+	// can have multiple addresses - these are the IP addresses of any Pods
+	// selected by that Service. Pick all the IPv4 addresses.
+	ips := make([]string, 0)
+	for _, ep := range eps.Endpoints {
+		for _, ip := range ep.Addresses {
+			if !net.IsIPv4String(ip) {
+				logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
+			} else {
+				ips = append(ips, ip)
+			}
+		}
+	}
+	if len(ips) == 0 {
+		logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
+		return nil
+	}
+	updateFunc := func(rec *operatorutils.Records) {
+		mak.Set(&rec.IP4, fqdn, ips)
+	}
+	if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
+		return fmt.Errorf("error updating DNS records: %w", err)
+	}
+	return nil
+}
+
+// maybeCleanup ensures that the DNS record for the proxy has been removed from
+// dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
+// has been removed from the Service. If the record is not found in the
+// ConfigMap, the ConfigMap does not exist, or the Service does not have
+// tailscale.com/magic-dnsname annotation, just remove the finalizer.
+func (h *dnsRecordsReconciler) maybeCleanup(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error {
+	ix := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
+	if ix == -1 {
+		logger.Debugf("no finalizer, nothing to do")
+		return nil
+	}
+	cm := &corev1.ConfigMap{}
+	err := h.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: h.tsNamespace}, cm)
+	if apierrors.IsNotFound(err) {
+		logger.Debug("'dsnrecords' ConfigMap not found")
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	if err != nil {
+		return fmt.Errorf("error retrieving 'dnsrecords' ConfigMap: %w", err)
+	}
+	if cm.Data == nil {
+		logger.Debug("'dnsrecords' ConfigMap contains no records")
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	_, ok := cm.Data[operatorutils.DNSRecordsCMKey]
+	if !ok {
+		logger.Debug("'dnsrecords' ConfigMap contains no records")
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	fqdn, _ := headlessSvc.GetAnnotations()[annotationTSMagicDNSName]
+	if fqdn == "" {
+		return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+	}
+	logger.Infof("removing DNS record for MagicDNS name %s", fqdn)
+	updateFunc := func(rec *operatorutils.Records) {
+		delete(rec.IP4, fqdn)
+	}
+	if err = h.updateDNSConfig(ctx, updateFunc); err != nil {
+		return fmt.Errorf("error updating DNS config: %w", err)
+	}
+	return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
+}
+
+func (dnsRR *dnsRecordsReconciler) removeHeadlessSvcFinalizer(ctx context.Context, headlessSvc *corev1.Service) error {
+	idx := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
+	if idx == -1 {
+		return nil
+	}
+	headlessSvc.Finalizers = append(headlessSvc.Finalizers[:idx], headlessSvc.Finalizers[idx+1:]...)
+	return dnsRR.Update(ctx, headlessSvc)
+}
+
+// fqdnForDNSRecord returns MagicDNS name associated with a given headless Service.
+// If the headless Service is for a tailscale Ingress proxy, returns ingress.status.loadBalancer.ingress.hostname.
+// If the headless Service is for an tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value.
+// This function is not expected to be called with headless Services for other
+// proxy types, or any other Services, but it just returns an empty string if
+// that happens.
+func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) (string, error) {
+	parentName := parentFromObjectLabels(headlessSvc)
+	if isManagedByType(headlessSvc, "ingress") {
+		ing := new(networkingv1.Ingress)
+		if err := dnsRR.Get(ctx, parentName, ing); err != nil {
+			return "", err
+		}
+		if len(ing.Status.LoadBalancer.Ingress) == 0 {
+			return "", nil
+		}
+		return ing.Status.LoadBalancer.Ingress[0].Hostname, nil
+	}
+	if isManagedByType(headlessSvc, "svc") {
+		svc := new(corev1.Service)
+		if err := dnsRR.Get(ctx, parentName, svc); apierrors.IsNotFound(err) {
+			logger.Info("[unexpected] parent Service for egress proxy %s not found", headlessSvc.Name)
+			return "", nil
+		} else if err != nil {
+			return "", err
+		}
+		return svc.Annotations[AnnotationTailnetTargetFQDN], nil
+	}
+	return "", nil
+}
+
+// updateDNSConfig runs the provided update function against dnsrecords
+// ConfigMap. At this point the in-cluster ts.net nameserver is expected to be
+// successfully created together with the ConfigMap.
+func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error {
+	cm := &corev1.ConfigMap{}
+	err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm)
+	if apierrors.IsNotFound(err) {
+		dnsRR.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an isue and attach operator logs.")
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("error retrieving dnsrecords ConfigMap: %w", err)
+	}
+	dnsRecords := operatorutils.Records{Version: operatorutils.Alpha1Version, IP4: map[string][]string{}}
+	if cm.Data != nil && cm.Data[operatorutils.DNSRecordsCMKey] != "" {
+		if err := json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil {
+			return err
+		}
+	}
+	update(&dnsRecords)
+	dnsRecordsBs, err := json.Marshal(dnsRecords)
+	if err != nil {
+		return fmt.Errorf("error marshalling DNS records: %w", err)
+	}
+	mak.Set(&cm.Data, operatorutils.DNSRecordsCMKey, string(dnsRecordsBs))
+	return dnsRR.Update(ctx, cm)
+}
+
+// isSvcForFQDNEgressProxy returns true if the Service is a headless Service
+// created for a proxy for a tailscale egress Service configured via
+// tailscale.com/tailnet-fqdn annotation.
+func (dnsRR *dnsRecordsReconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Service) (bool, error) {
+	if !isManagedByType(svc, "svc") {
+		return false, nil
+	}
+	parentName := parentFromObjectLabels(svc)
+	parentSvc := new(corev1.Service)
+	if err := dnsRR.Get(ctx, parentName, parentSvc); apierrors.IsNotFound(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	annots := parentSvc.Annotations
+	return annots != nil && annots[AnnotationTailnetTargetFQDN] != "", nil
+}

cmd/k8s-operator/dnsrecords_test.go

@@ -0,0 +1,198 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	operatorutils "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
+)
+
+func TestDNSRecordsReconciler(t *testing.T) {
+	// Preconfigure a cluster with a DNSConfig
+	dnsConfig := &tsapi.DNSConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+		TypeMeta: metav1.TypeMeta{Kind: "DNSConfig"},
+		Spec: tsapi.DNSConfigSpec{
+			Nameserver: &tsapi.Nameserver{},
+		}}
+	ing := &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ts-ingress",
+			Namespace: "test",
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+		},
+		Status: networkingv1.IngressStatus{
+			LoadBalancer: networkingv1.IngressLoadBalancerStatus{
+				Ingress: []networkingv1.IngressLoadBalancerIngress{{
+					Hostname: "cluster.ingress.ts.net"}},
+			},
+		},
+	}
+	cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", Namespace: "tailscale"}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(cm).
+		WithObjects(dnsConfig).
+		WithObjects(ing).
+		WithStatusSubresource(dnsConfig, ing).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	// Set the ready condition of the DNSConfig
+	mustUpdateStatus[tsapi.DNSConfig](t, fc, "", "test", func(c *tsapi.DNSConfig) {
+		operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated, 0, cl, zl.Sugar())
+	})
+	dnsRR := &dnsRecordsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		tsNamespace: "tailscale",
+	}
+
+	// 1. DNS record is created for an egress proxy configured via
+	// tailscale.com/tailnet-fqdn annotation
+	egressSvcFQDN := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "egress-fqdn",
+			Namespace:   "test",
+			Annotations: map[string]string{"tailscale.com/tailnet-fqdn": "foo.bar.ts.net"},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "unused",
+			Type:         corev1.ServiceTypeExternalName,
+		},
+	}
+	headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
+	ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
+	mustCreate(t, fc, egressSvcFQDN)
+	mustCreate(t, fc, headlessForEgressSvcFQDN)
+	mustCreate(t, fc, ep)
+	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
+	// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
+	wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
+	// value changes
+	mustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) {
+		svc.Annotations["tailscale.com/tailnet-fqdn"] = "baz.bar.ts.net"
+	})
+	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
+	wantHosts = map[string][]string{"baz.bar.ts.net": {"10.9.8.7"}}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 3. DNS record is updated if the IP address of the proxy Pod changes.
+	ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
+	})
+	expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
+	wantHosts = map[string][]string{"baz.bar.ts.net": {"10.6.5.4"}}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 4. DNS record is created for an ingress proxy configured via Ingress
+	headlessForIngress := headlessSvcForParent(ing, "ingress")
+	ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
+	mustCreate(t, fc, headlessForIngress)
+	mustCreate(t, fc, ep)
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
+	wantHosts["cluster.ingress.ts.net"] = []string{"10.9.8.7"}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 5. DNS records are updated if Ingress's MagicDNS name changes (i.e users changed spec.tls.hosts[0])
+	t.Log("test case 5")
+	mustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) {
+		ing.Status.LoadBalancer.Ingress[0].Hostname = "another.ingress.ts.net"
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
+	delete(wantHosts, "cluster.ingress.ts.net")
+	wantHosts["another.ingress.ts.net"] = []string{"10.9.8.7"}
+	expectHostsRecords(t, fc, wantHosts)
+
+	// 6. DNS records are updated if Ingress proxy's Pod IP changes
+	mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
+		ep.Endpoints[0].Addresses = []string{"7.8.9.10"}
+	})
+	expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
+	wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
+	expectHostsRecords(t, fc, wantHosts)
+}
+
+func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      o.GetName(),
+			Namespace: "tailscale",
+			Labels: map[string]string{
+				LabelManaged:         "true",
+				LabelParentName:      o.GetName(),
+				LabelParentNamespace: o.GetNamespace(),
+				LabelParentType:      typ,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "None",
+			Type:      corev1.ServiceTypeClusterIP,
+			Selector:  map[string]string{"foo": "bar"},
+		},
+	}
+}
+
+func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
+	return &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      svc.Name,
+			Namespace: svc.Namespace,
+			Labels:    map[string]string{discoveryv1.LabelServiceName: svc.Name},
+		},
+		Endpoints: []discoveryv1.Endpoint{{
+			Addresses: []string{ip},
+		}},
+	}
+}
+
+func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]string) {
+	t.Helper()
+	cm := new(corev1.ConfigMap)
+	if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
+		t.Fatalf("getting dnsconfig ConfigMap: %v", err)
+	}
+	if cm.Data == nil {
+		t.Fatal("dnsconfig ConfigMap has no data")
+	}
+	dnsConfigString, ok := cm.Data[operatorutils.DNSRecordsCMKey]
+	if !ok {
+		t.Fatal("dnsconfig ConfigMap does not contain dnsconfig")
+	}
+	dnsConfig := &operatorutils.Records{}
+	if err := json.Unmarshal([]byte(dnsConfigString), dnsConfig); err != nil {
+		t.Fatalf("unmarshaling dnsconfig: %v", err)
+	}
+	if diff := cmp.Diff(dnsConfig.IP4, wantsHosts); diff != "" {
+		t.Fatalf("unexpected dns config (-got +want):\n%s", diff)
+	}
+}

cmd/k8s-operator/generate/main.go

@@ -22,9 +22,11 @@ const (
 	operatorDeploymentFilesPath   = "cmd/k8s-operator/deploy"
 	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
+	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
+	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -36,10 +38,10 @@ func main() {
 	}
 	repoRoot := "../../"
 	switch os.Args[1] {
-	case "helmcrd": // insert CRD to Helm templates behind a installCRDs=true conditional check
-		log.Print("Adding Connector CRD to Helm templates")
+	case "helmcrd": // insert CRDs to Helm templates behind a installCRDs=true conditional check
+		log.Print("Adding CRDs to Helm templates")
 		if err := generate("./"); err != nil {
-			log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+			log.Fatalf("error adding CRDs to Helm templates: %v", err)
 		}
 		return
 	case "staticmanifests": // generate static manifests from Helm templates (including the CRD)
@@ -108,7 +110,7 @@ func main() {
 	}
 }
 
-// generate places tailscale.com CRDs (currently Connector and ProxyClass) into
+// generate places tailscale.com CRDs (currently Connector, ProxyClass and DNSConfig) into
 // the Helm chart templates behind .Values.installCRDs=true condition (true by
 // default).
 func generate(baseDir string) error {
@@ -140,6 +142,9 @@ func generate(baseDir string) error {
 	if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
 		return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
 	}
+	if err := addCRDToHelm(dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath); err != nil {
+		return fmt.Errorf("error adding DNSConfig CRD to Helm templates: %w", err)
+	}
 	return nil
 }
 
@@ -151,5 +156,8 @@ func cleanup(baseDir string) error {
 	if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
 	}
+	if err := os.Remove(filepath.Join(baseDir, dnsConfigCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up DNSConfig CRD template: %w", err)
+	}
 	return nil
 }

cmd/k8s-operator/generate/main_test.go

@@ -56,6 +56,9 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: proxyclasses.tailscale.com") {
 		t.Errorf("ProxyClass CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: dnsconfigs.tailscale.com") {
+		t.Errorf("DNSConfig CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -71,4 +74,7 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") {
 		t.Errorf("ProxyClass CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: dnsconfigs.tailscale.com") {
+		t.Errorf("DNSConfig CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/nameserver.go

@@ -0,0 +1,283 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sync"
+
+	_ "embed"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/yaml"
+	tsoperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+const (
+	reasonNameserverCreationFailed  = "NameserverCreationFailed"
+	reasonMultipleDNSConfigsPresent = "MultipleDNSConfigsPresent"
+
+	reasonNameserverCreated = "NameserverCreated"
+
+	messageNameserverCreationFailed  = "Failed creating nameserver resources: %v"
+	messageMultipleDNSConfigsPresent = "Multiple DNSConfig resources found in cluster. Please ensure no more than one is present."
+
+	defaultNameserverImageRepo = "tailscale/k8s-nameserver"
+	// TODO (irbekrm): once we start publishing nameserver images for stable
+	// track, replace 'unstable' here with the version of this operator
+	// instance.
+	defaultNameserverImageTag = "unstable"
+)
+
+// NameserverReconciler knows how to create nameserver resources in cluster in
+// response to users applying DNSConfig.
+type NameserverReconciler struct {
+	client.Client
+	logger      *zap.SugaredLogger
+	recorder    record.EventRecorder
+	clock       tstime.Clock
+	tsNamespace string
+
+	mu                 sync.Mutex           // protects following
+	managedNameservers set.Slice[types.UID] // one or none
+}
+
+var (
+	gaugeNameserverResources = clientmetric.NewGauge("k8s_nameserver_resources")
+)
+
+func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
+	logger := a.logger.With("dnsConfig", req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	var dnsCfg tsapi.DNSConfig
+	err = a.Get(ctx, req.NamespacedName, &dnsCfg)
+	if apierrors.IsNotFound(err) {
+		// Request object not found, could have been deleted after reconcile request.
+		logger.Debugf("dnsconfig not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get dnsconfig: %w", err)
+	}
+	if !dnsCfg.DeletionTimestamp.IsZero() {
+		ix := xslices.Index(dnsCfg.Finalizers, FinalizerName)
+		if ix < 0 {
+			logger.Debugf("no finalizer, nothing to do")
+			return reconcile.Result{}, nil
+		}
+		logger.Info("Cleaning up DNSConfig resources")
+		if err := a.maybeCleanup(ctx, &dnsCfg, logger); err != nil {
+			logger.Errorf("error cleaning up reconciler resource: %v", err)
+			return res, err
+		}
+		dnsCfg.Finalizers = append(dnsCfg.Finalizers[:ix], dnsCfg.Finalizers[ix+1:]...)
+		if err := a.Update(ctx, &dnsCfg); err != nil {
+			logger.Errorf("error removing finalizer: %v", err)
+			return reconcile.Result{}, err
+		}
+		logger.Infof("Nameserver resources cleaned up")
+		return reconcile.Result{}, nil
+	}
+
+	oldCnStatus := dnsCfg.Status.DeepCopy()
+	setStatus := func(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+		tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, a.clock, logger)
+		if !apiequality.Semantic.DeepEqual(oldCnStatus, dnsCfg.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil {
+				err = errors.Wrap(err, updateErr.Error())
+			}
+		}
+		return res, err
+	}
+	var dnsCfgs tsapi.DNSConfigList
+	if err := a.List(ctx, &dnsCfgs); err != nil {
+		return res, fmt.Errorf("error listing DNSConfigs: %w", err)
+	}
+	if len(dnsCfgs.Items) > 1 { // enforce DNSConfig to be a singleton
+		msg := "invalid cluster configuration: more than one tailscale.com/dnsconfigs found. Please ensure that no more than one is created."
+		logger.Error(msg)
+		a.recorder.Event(&dnsCfg, corev1.EventTypeWarning, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
+		setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
+	}
+
+	if !slices.Contains(dnsCfg.Finalizers, FinalizerName) {
+		logger.Infof("ensuring nameserver resources")
+		dnsCfg.Finalizers = append(dnsCfg.Finalizers, FinalizerName)
+		if err := a.Update(ctx, &dnsCfg); err != nil {
+			msg := fmt.Sprintf(messageNameserverCreationFailed, err)
+			logger.Error(msg)
+			return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
+		}
+	}
+	if err := a.maybeProvision(ctx, &dnsCfg, logger); err != nil {
+		return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
+	}
+
+	a.mu.Lock()
+	a.managedNameservers.Add(dnsCfg.UID)
+	a.mu.Unlock()
+	gaugeNameserverResources.Set(int64(a.managedNameservers.Len()))
+
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: a.tsNamespace},
+	}
+	if err := a.Client.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil {
+		return res, fmt.Errorf("error getting Service: %w", err)
+	}
+	if ip := svc.Spec.ClusterIP; ip != "" && ip != "None" {
+		dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
+			IP: ip,
+		}
+		return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated)
+	}
+	logger.Info("nameserver Service does not have an IP address allocated, waiting...")
+	return reconcile.Result{}, nil
+}
+
+func nameserverResourceLabels(name, namespace string) map[string]string {
+	labels := childResourceLabels(name, namespace, "nameserver")
+	labels["app.kubernetes.io/name"] = "tailscale"
+	labels["app.kubernetes.io/component"] = "nameserver"
+	return labels
+}
+
+func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error {
+	labels := nameserverResourceLabels(tsDNSCfg.Name, a.tsNamespace)
+	dCfg := &deployConfig{
+		ownerRefs: []metav1.OwnerReference{*metav1.NewControllerRef(tsDNSCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))},
+		namespace: a.tsNamespace,
+		labels:    labels,
+		imageRepo: defaultNameserverImageRepo,
+		imageTag:  defaultNameserverImageTag,
+	}
+	if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Repo != "" {
+		dCfg.imageRepo = tsDNSCfg.Spec.Nameserver.Image.Repo
+	}
+	if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" {
+		dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag
+	}
+	for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} {
+		if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil {
+			return fmt.Errorf("error reconciling %s: %w", deployable.kind, err)
+		}
+	}
+	return nil
+}
+
+// maybeCleanup removes DNSConfig from being tracked. The cluster resources
+// created, will be automatically garbage collected as they are owned by the
+// DNSConfig.
+func (a *NameserverReconciler) maybeCleanup(ctx context.Context, dnsCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error {
+	a.mu.Lock()
+	a.managedNameservers.Remove(dnsCfg.UID)
+	a.mu.Unlock()
+	gaugeNameserverResources.Set(int64(a.managedNameservers.Len()))
+	return nil
+}
+
+type deployable struct {
+	kind      string
+	updateObj func(context.Context, *deployConfig, client.Client) error
+}
+
+type deployConfig struct {
+	imageRepo string
+	imageTag  string
+	labels    map[string]string
+	ownerRefs []metav1.OwnerReference
+	namespace string
+}
+
+var (
+	//go:embed deploy/manifests/nameserver/cm.yaml
+	cmYaml []byte
+	//go:embed deploy/manifests/nameserver/deploy.yaml
+	deployYaml []byte
+	//go:embed deploy/manifests/nameserver/sa.yaml
+	saYaml []byte
+	//go:embed deploy/manifests/nameserver/svc.yaml
+	svcYaml []byte
+
+	deployDeployable = deployable{
+		kind: "Deployment",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			d := new(appsv1.Deployment)
+			if err := yaml.Unmarshal(deployYaml, &d); err != nil {
+				return fmt.Errorf("error unmarshalling Deployment yaml: %w", err)
+			}
+			d.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", cfg.imageRepo, cfg.imageTag)
+			d.ObjectMeta.Namespace = cfg.namespace
+			d.ObjectMeta.Labels = cfg.labels
+			d.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			updateF := func(oldD *appsv1.Deployment) {
+				oldD.Spec = d.Spec
+			}
+			_, err := createOrUpdate[appsv1.Deployment](ctx, kubeClient, cfg.namespace, d, updateF)
+			return err
+		},
+	}
+	saDeployable = deployable{
+		kind: "ServiceAccount",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			sa := new(corev1.ServiceAccount)
+			if err := yaml.Unmarshal(saYaml, &sa); err != nil {
+				return fmt.Errorf("error unmarshalling ServiceAccount yaml: %w", err)
+			}
+			sa.ObjectMeta.Labels = cfg.labels
+			sa.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			sa.ObjectMeta.Namespace = cfg.namespace
+			_, err := createOrUpdate(ctx, kubeClient, cfg.namespace, sa, func(*corev1.ServiceAccount) {})
+			return err
+		},
+	}
+	svcDeployable = deployable{
+		kind: "Service",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			svc := new(corev1.Service)
+			if err := yaml.Unmarshal(svcYaml, &svc); err != nil {
+				return fmt.Errorf("error unmarshalling Service yaml: %w", err)
+			}
+			svc.ObjectMeta.Labels = cfg.labels
+			svc.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			svc.ObjectMeta.Namespace = cfg.namespace
+			_, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {})
+			return err
+		},
+	}
+	cmDeployable = deployable{
+		kind: "ConfigMap",
+		updateObj: func(ctx context.Context, cfg *deployConfig, kubeClient client.Client) error {
+			cm := new(corev1.ConfigMap)
+			if err := yaml.Unmarshal(cmYaml, &cm); err != nil {
+				return fmt.Errorf("error unmarshalling ConfigMap yaml: %w", err)
+			}
+			cm.ObjectMeta.Labels = cfg.labels
+			cm.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			cm.ObjectMeta.Namespace = cfg.namespace
+			_, err := createOrUpdate[corev1.ConfigMap](ctx, kubeClient, cfg.namespace, cm, func(cm *corev1.ConfigMap) {})
+			return err
+		},
+	}
+)

cmd/k8s-operator/nameserver_test.go

@@ -0,0 +1,127 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// tailscale-operator provides a way to expose services running in a Kubernetes
+// cluster to your Tailnet and to make Tailscale nodes available to cluster
+// workloads
+package main
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/yaml"
+	operatorutils "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+	"tailscale.com/util/mak"
+)
+
+func TestNameserverReconciler(t *testing.T) {
+	dnsCfg := &tsapi.DNSConfig{
+		TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+		},
+		Spec: tsapi.DNSConfigSpec{
+			Nameserver: &tsapi.Nameserver{
+				Image: &tsapi.Image{
+					Repo: "test",
+					Tag:  "v0.0.1",
+				},
+			},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(dnsCfg).
+		WithStatusSubresource(dnsCfg).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	nr := &NameserverReconciler{
+		Client:      fc,
+		clock:       cl,
+		logger:      zl.Sugar(),
+		tsNamespace: "tailscale",
+	}
+	expectReconciled(t, nr, "", "test")
+	// Verify that nameserver Deployment has been created and has the expected fields.
+	wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}}
+	if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil {
+		t.Fatalf("unmarshalling yaml: %v", err)
+	}
+	dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
+	wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef}
+	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1"
+	wantsDeploy.Namespace = "tailscale"
+	labels := nameserverResourceLabels("test", "tailscale")
+	wantsDeploy.ObjectMeta.Labels = labels
+	expectEqual(t, fc, wantsDeploy, nil)
+
+	// Verify that DNSConfig advertizes the nameserver's Service IP address,
+	// has the ready status condition and tailscale finalizer.
+	mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
+		svc.Spec.ClusterIP = "1.2.3.4"
+	})
+	expectReconciled(t, nr, "", "test")
+	dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
+		IP: "1.2.3.4",
+	}
+	dnsCfg.Finalizers = []string{FinalizerName}
+	dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, tsapi.ConnectorCondition{
+		Type:               tsapi.NameserverReady,
+		Status:             metav1.ConditionTrue,
+		Reason:             reasonNameserverCreated,
+		Message:            reasonNameserverCreated,
+		LastTransitionTime: &metav1.Time{Time: cl.Now().Truncate(time.Second)},
+	})
+	expectEqual(t, fc, dnsCfg, nil)
+
+	// // Verify that nameserver image gets updated to match DNSConfig spec.
+	mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
+		dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2"
+	})
+	expectReconciled(t, nr, "", "test")
+	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
+	expectEqual(t, fc, wantsDeploy, nil)
+
+	// Verify that when another actor sets ConfigMap data, it does not get
+	// overwritten by nameserver reconciler.
+	dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}}
+	bs, err := json.Marshal(dnsRecords)
+	if err != nil {
+		t.Fatalf("error marshalling ConfigMap contents: %v", err)
+	}
+	mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
+		mak.Set(&cm.Data, "records.json", string(bs))
+	})
+	expectReconciled(t, nr, "", "test")
+	wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords",
+		Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}},
+		TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+		Data:     map[string]string{"records.json": string(bs)},
+	}
+	expectEqual(t, fc, wantCm, nil)
+
+	// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
+	// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
+	mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
+		dnsCfg.Spec.Nameserver.Image = nil
+	})
+	expectReconciled(t, nr, "", "test")
+	wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
+	expectEqual(t, fc, wantsDeploy, nil)
+}

cmd/k8s-operator/operator.go

@@ -20,6 +20,7 @@ import (
 	"golang.org/x/oauth2/clientcredentials"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
@@ -44,12 +45,12 @@ import (
 	"tailscale.com/version"
 )
 
-// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
-//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
-
 // Generate Connector and ProxyClass CustomResourceDefinition yamls from their Go types.
 //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen crd schemapatch:manifests=./deploy/crds output:dir=./deploy/crds paths=../../k8s-operator/apis/...
 
+// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
+//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
+
 // Generate CRD docs from the yamls
 //go:generate go run fybrik.io/crdoc --resources=./deploy/crds --output=../../k8s-operator/api.md
 
@@ -59,12 +60,13 @@ func main() {
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	var (
-		tsNamespace       = defaultEnv("OPERATOR_NAMESPACE", "")
-		tslogging         = defaultEnv("OPERATOR_LOGGING", "info")
-		image             = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
-		priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
-		tags              = defaultEnv("PROXY_TAGS", "tag:k8s")
-		tsFirewallMode    = defaultEnv("PROXY_FIREWALL_MODE", "")
+		tsNamespace           = defaultEnv("OPERATOR_NAMESPACE", "")
+		tslogging             = defaultEnv("OPERATOR_LOGGING", "info")
+		image                 = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
+		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
+		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
+		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
 	var opts []kzap.Opts
@@ -93,9 +95,19 @@ func main() {
 	defer s.Close()
 	restConfig := config.GetConfigOrDie()
 	maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
-	// TODO (irbekrm): gather the reconciler options into an opts struct
-	// rather than passing a million of them in one by one.
-	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode)
+	rOpts := reconcilerOpts{
+		log:                           zlog,
+		tsServer:                      s,
+		tsClient:                      tsClient,
+		tailscaleNamespace:            tsNamespace,
+		restConfig:                    restConfig,
+		proxyImage:                    image,
+		proxyPriorityClassName:        priorityClassName,
+		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
+		proxyTags:                     tags,
+		proxyFirewallMode:             tsFirewallMode,
+	}
+	runReconcilers(rOpts)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -203,11 +215,8 @@ waitOnline:
 
 // runReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler. It blocks forever.
-func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) {
-	var (
-		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
-	)
-	startlog := zlog.Named("startReconcilers")
+func runReconcilers(opts reconcilerOpts) {
+	startlog := opts.log.Named("startReconcilers")
 	// For secrets and statefulsets, we only get permission to touch the objects
 	// in the controller's own namespace. This cannot be expressed by
 	// .Watches(...) below, instead you have to add a per-type field selector to
@@ -215,7 +224,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 	// implicitly filter what parts of the world the builder code gets to see at
 	// all.
 	nsFilter := cache.ByObject{
-		Field: client.InNamespace(tsNamespace).AsSelector(),
+		Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(),
 	}
 	mgrOpts := manager.Options{
 		// TODO (irbekrm): stricter filtering what we watch/cache/call
@@ -223,33 +232,37 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		// resources that we GET via the controller manager's client.
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
-				&corev1.Secret{}:      nsFilter,
-				&appsv1.StatefulSet{}: nsFilter,
+				&corev1.Secret{}:             nsFilter,
+				&corev1.ServiceAccount{}:     nsFilter,
+				&corev1.ConfigMap{}:          nsFilter,
+				&appsv1.StatefulSet{}:        nsFilter,
+				&appsv1.Deployment{}:         nsFilter,
+				&discoveryv1.EndpointSlice{}: nsFilter,
 			},
 		},
 		Scheme: tsapi.GlobalScheme,
 	}
-	mgr, err := manager.New(restConfig, mgrOpts)
+	mgr, err := manager.New(opts.restConfig, mgrOpts)
 	if err != nil {
 		startlog.Fatalf("could not create manager: %v", err)
 	}
 
 	svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler)
 	svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc"))
-	// If a ProxyClassChanges, enqueue all Services labeled with that
+	// If a ProxyClass changes, enqueue all Services labeled with that
 	// ProxyClass's name.
 	proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc(mgr.GetClient(), startlog))
 
 	eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
 	ssr := &tailscaleSTSReconciler{
 		Client:                 mgr.GetClient(),
-		tsnetServer:            s,
-		tsClient:               tsClient,
-		defaultTags:            strings.Split(tags, ","),
-		operatorNamespace:      tsNamespace,
-		proxyImage:             image,
-		proxyPriorityClassName: priorityClassName,
-		tsFirewallMode:         tsFirewallMode,
+		tsnetServer:            opts.tsServer,
+		tsClient:               opts.tsClient,
+		defaultTags:            strings.Split(opts.proxyTags, ","),
+		operatorNamespace:      opts.tailscaleNamespace,
+		proxyImage:             opts.proxyImage,
+		proxyPriorityClassName: opts.proxyPriorityClassName,
+		tsFirewallMode:         opts.proxyFirewallMode,
 	}
 	err = builder.
 		ControllerManagedBy(mgr).
@@ -261,10 +274,10 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Complete(&ServiceReconciler{
 			ssr:                   ssr,
 			Client:                mgr.GetClient(),
-			logger:                zlog.Named("service-reconciler"),
-			isDefaultLoadBalancer: isDefaultLoadBalancer,
+			logger:                opts.log.Named("service-reconciler"),
+			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
 			recorder:              eventRecorder,
-			tsNamespace:           tsNamespace,
+			tsNamespace:           opts.tailscaleNamespace,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -286,7 +299,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			ssr:      ssr,
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
-			logger:   zlog.Named("ingress-reconciler"),
+			logger:   opts.log.Named("ingress-reconciler"),
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -305,29 +318,201 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			ssr:      ssr,
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
-			logger:   zlog.Named("connector-reconciler"),
+			logger:   opts.log.Named("connector-reconciler"),
 			clock:    tstime.DefaultClock{},
 		})
 	if err != nil {
-		startlog.Fatal("could not create connector reconciler: %v", err)
+		startlog.Fatalf("could not create connector reconciler: %v", err)
+	}
+	// TODO (irbekrm): switch to metadata-only watches for resources whose
+	// spec we don't need to inspect to reduce memory consumption.
+	// https://github.com/kubernetes-sigs/controller-runtime/issues/1159
+	nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver"))
+	err = builder.ControllerManagedBy(mgr).
+		For(&tsapi.DNSConfig{}).
+		Watches(&appsv1.Deployment{}, nameserverFilter).
+		Watches(&corev1.ConfigMap{}, nameserverFilter).
+		Watches(&corev1.Service{}, nameserverFilter).
+		Watches(&corev1.ServiceAccount{}, nameserverFilter).
+		Complete(&NameserverReconciler{
+			recorder:    eventRecorder,
+			tsNamespace: opts.tailscaleNamespace,
+			Client:      mgr.GetClient(),
+			logger:      opts.log.Named("nameserver-reconciler"),
+			clock:       tstime.DefaultClock{},
+		})
+	if err != nil {
+		startlog.Fatalf("could not create nameserver reconciler: %v", err)
 	}
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
 		Complete(&ProxyClassReconciler{
 			Client:   mgr.GetClient(),
 			recorder: eventRecorder,
-			logger:   zlog.Named("proxyclass-reconciler"),
+			logger:   opts.log.Named("proxyclass-reconciler"),
 			clock:    tstime.DefaultClock{},
 		})
 	if err != nil {
 		startlog.Fatal("could not create proxyclass reconciler: %v", err)
 	}
+	logger := startlog.Named("dns-records-reconciler-event-handlers")
+	// On EndpointSlice events, if it is an EndpointSlice for an
+	// ingress/egress proxy headless Service, reconcile the headless
+	// Service.
+	dnsRREpsOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerEndpointSliceHandler)
+	// On DNSConfig changes, reconcile all headless Services for
+	// ingress/egress proxies in operator namespace.
+	dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(opts.tailscaleNamespace, mgr.GetClient(), logger))
+	// On Service events, if it is an ingress/egress proxy headless Service, reconcile it.
+	dnsRRServiceOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerServiceHandler)
+	// On Ingress events, if it is a tailscale Ingress or if tailscale is the default ingress controller, reconcile the proxy
+	// headless Service.
+	dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(opts.tailscaleNamespace, opts.proxyActAsDefaultLoadBalancer, mgr.GetClient(), logger))
+	err = builder.ControllerManagedBy(mgr).
+		Named("dns-records-reconciler").
+		Watches(&corev1.Service{}, dnsRRServiceOpts).
+		Watches(&networkingv1.Ingress{}, dnsRRIngressOpts).
+		Watches(&discoveryv1.EndpointSlice{}, dnsRREpsOpts).
+		Watches(&tsapi.DNSConfig{}, dnsRRDNSConfigOpts).
+		Complete(&dnsRecordsReconciler{
+			Client:                mgr.GetClient(),
+			tsNamespace:           opts.tailscaleNamespace,
+			logger:                opts.log.Named("dns-records-reconciler"),
+			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create DNS records reconciler: %v", err)
+	}
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
 	}
 }
 
+type reconcilerOpts struct {
+	log                *zap.SugaredLogger
+	tsServer           *tsnet.Server
+	tsClient           *tailscale.Client
+	tailscaleNamespace string       // namespace in which operator resources will be deployed
+	restConfig         *rest.Config // config for connecting to the kube API server
+	proxyImage         string       // <proxy-image-repo>:<proxy-image-tag>
+	// proxyPriorityClassName isPriorityClass to be set for proxy Pods. This
+	// is a legacy mechanism for cluster resource configuration options -
+	// going forward use ProxyClass.
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass
+	proxyPriorityClassName string
+	// proxyTags are ACL tags to tag proxy auth keys. Multiple tags should
+	// be provided as a string with comma-separated tag values. Proxy tags
+	// default to tag:k8s.
+	// https://tailscale.com/kb/1085/auth-keys
+	proxyTags string
+	// proxyActAsDefaultLoadBalancer determines whether this operator
+	// instance should act as the default ingress controller when looking at
+	// Ingress resources with unset ingress.spec.ingressClassName.
+	// TODO (irbekrm): this setting does not respect the default
+	// IngressClass.
+	// https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class
+	// We should fix that and preferably integrate with that mechanism as
+	// well - perhaps make the operator itself create the default
+	// IngressClass if this is set to true.
+	proxyActAsDefaultLoadBalancer bool
+	// proxyFirewallMode determines whether non-userspace proxies should use
+	// iptables or nftables for firewall configuration. Accepted values are
+	// iptables, nftables and auto. If set to auto, proxy will automatically
+	// determine which mode is supported for a given host (prefer nftables).
+	// Auto is usually the best choice, unless you want to explicitly set
+	// specific mode for debugging purposes.
+	proxyFirewallMode string
+}
+
+// enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
+// ingress/egress proxy headless Service found in the provided namespace.
+func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, _ client.Object) []reconcile.Request {
+		reqs := make([]reconcile.Request, 0)
+
+		// Get all headless Services for proxies configured using Service.
+		svcProxyLabels := map[string]string{
+			LabelManaged:    "true",
+			LabelParentType: "svc",
+		}
+		svcHeadlessSvcList := &corev1.ServiceList{}
+		if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
+			logger.Errorf("error listing headless Services for tailscale ingress/egress Services in operator namespace: %v", err)
+			return nil
+		}
+		for _, svc := range svcHeadlessSvcList.Items {
+			reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}})
+		}
+
+		// Get all headless Services for proxies configured using Ingress.
+		ingProxyLabels := map[string]string{
+			LabelManaged:    "true",
+			LabelParentType: "ingress",
+		}
+		ingHeadlessSvcList := &corev1.ServiceList{}
+		if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
+			logger.Errorf("error listing headless Services for tailscale Ingresses in operator namespace: %v", err)
+			return nil
+		}
+		for _, svc := range ingHeadlessSvcList.Items {
+			reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}})
+		}
+		return reqs
+	}
+}
+
+// dnsRecordsReconciler filters EndpointSlice events for which
+// dns-records-reconciler should reconcile a headless Service. The only events
+// it should reconcile are those for EndpointSlices associated with proxy
+// headless Services.
+func dnsRecordsReconcilerEndpointSliceHandler(ctx context.Context, o client.Object) []reconcile.Request {
+	if !isManagedByType(o, "svc") && !isManagedByType(o, "ingress") {
+		return nil
+	}
+	headlessSvcName, ok := o.GetLabels()[discoveryv1.LabelServiceName] // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
+	if !ok {
+		return nil
+	}
+	return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: headlessSvcName}}}
+}
+
+// dnsRecordsReconcilerServiceHandler filters Service events for which
+// dns-records-reconciler should reconcile. If the event is for a cluster
+// ingress/cluster egress proxy's headless Service, returns the Service for
+// reconcile.
+func dnsRecordsReconcilerServiceHandler(ctx context.Context, o client.Object) []reconcile.Request {
+	if isManagedByType(o, "svc") || isManagedByType(o, "ingress") {
+		return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: o.GetName()}}}
+	}
+	return nil
+}
+
+// dnsRecordsReconcilerIngressHandler filters Ingress events to ensure that
+// dns-records-reconciler only reconciles on tailscale Ingress events. When an
+// event is observed on a tailscale Ingress, reconcile the proxy headless Service.
+func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		ing, ok := o.(*networkingv1.Ingress)
+		if !ok {
+			return nil
+		}
+		if !isDefaultLoadBalancer && (ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != "tailscale") {
+			return nil
+		}
+		proxyResourceLabels := childResourceLabels(ing.Name, ing.Namespace, "ingress")
+		headlessSvc, err := getSingleObject[corev1.Service](ctx, cl, ns, proxyResourceLabels)
+		if err != nil {
+			logger.Errorf("error getting headless Service from parent labels: %v", err)
+			return nil
+		}
+		if headlessSvc == nil {
+			return nil
+		}
+		return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: headlessSvc.Namespace, Name: headlessSvc.Name}}}
+	}
+}
+
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
 	DeleteDevice(ctx context.Context, nodeStableID string) error

cmd/k8s-operator/operator_test.go

@@ -1182,7 +1182,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1",
+		confFileHash:    "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e",
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 
@@ -1192,11 +1192,10 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "1a087f887825d2b75d3673c7c2b0131f8ec1f0b1cb761d33e236dd28350dfe23"
+	o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }
-
 func Test_isMagicDNSName(t *testing.T) {
 	tests := []struct {
 		in   string

cmd/k8s-operator/proxyclass.go

@@ -3,8 +3,6 @@
 
 //go:build !plan9
 
-// tailscale-operator provides a way to expose services running in a Kubernetes
-// cluster to your Tailnet.
 package main
 
 import (

cmd/k8s-operator/sts.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/yaml"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	kubeutils "tailscale.com/k8s-operator"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/net/netutil"
@@ -92,10 +93,6 @@ const (
 	podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
 	// podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents.
 	podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash"
-
-	// tailscaledConfigKey is the name of the key in proxy Secret Data that
-	// holds the tailscaled config contents.
-	tailscaledConfigKey = "tailscaled"
 )
 
 var (
@@ -174,11 +171,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
 		return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
 	}
 
-	secretName, tsConfigHash, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
+	secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
 	}
-	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash)
+	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash, configs)
 	if err != nil {
 		return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
 	}
@@ -291,7 +288,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, string, error) {
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaleConfigs, _ error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			// Hardcode a -0 suffix so that in future, if we support
@@ -307,25 +304,23 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
 		orig = secret.DeepCopy()
 	} else if !apierrors.IsNotFound(err) {
-		return "", "", err
+		return "", "", nil, err
 	}
 
-	var (
-		authKey, hash string
-	)
+	var authKey string
 	if orig == nil {
 		// Initially it contains only tailscaled config, but when the
 		// proxy starts, it will also store there the state, certs and
 		// ACME account key.
 		sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
 		if err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 		if sts != nil {
 			// StatefulSet exists, so we have already created the secret.
 			// If the secret is missing, they should delete the StatefulSet.
 			logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
-			return "", "", nil
+			return "", "", nil, nil
 		}
 		// Create API Key secret which is going to be used by the statefulset
 		// to authenticate with Tailscale.
@@ -336,45 +331,58 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		}
 		authKey, err = a.newAuthKey(ctx, tags)
 		if err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 	}
-	confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig)
+	configs, err := tailscaledConfig(stsC, authKey, orig)
 	if err != nil {
-		return "", "", fmt.Errorf("error creating tailscaled config: %w", err)
+		return "", "", nil, fmt.Errorf("error creating tailscaled config: %w", err)
+	}
+	hash, err = tailscaledConfigHash(configs)
+	if err != nil {
+		return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err)
+	}
+
+	latest := tailcfg.CapabilityVersion(-1)
+	var latestConfig ipn.ConfigVAlpha
+	for key, val := range configs {
+		fn := kubeutils.TailscaledConfigFileNameForCap(key)
+		b, err := json.Marshal(val)
+		if err != nil {
+			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
+		}
+		mak.Set(&secret.StringData, fn, string(b))
+		if key > latest {
+			latest = key
+			latestConfig = val
+		}
 	}
-	hash = h
-	mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes))
 
 	if stsC.ServeConfig != nil {
 		j, err := json.Marshal(stsC.ServeConfig)
 		if err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 		mak.Set(&secret.StringData, "serve-config", string(j))
 	}
 
 	if orig != nil {
-		logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(secret.Data[tailscaledConfigKey]))
+		logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
 		if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 	} else {
-		logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes([]byte(secret.StringData[tailscaledConfigKey])))
+		logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
 		if err := a.Create(ctx, secret); err != nil {
-			return "", "", err
+			return "", "", nil, err
 		}
 	}
-	return secret.Name, hash, nil
+	return secret.Name, hash, configs, nil
 }
 
 // sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
 // auth key.
-func sanitizeConfigBytes(bs []byte) string {
-	c := &ipn.ConfigVAlpha{}
-	if err := json.Unmarshal(bs, c); err != nil {
-		return "invalid config"
-	}
+func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
 	if c.AuthKey != nil {
 		c.AuthKey = ptr.To("**redacted**")
 	}
@@ -437,7 +445,7 @@ var proxyYaml []byte
 //go:embed deploy/manifests/userspace-proxy.yaml
 var userspaceProxyYaml []byte
 
-func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) {
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -493,9 +501,15 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Value: proxySecret,
 		},
 		corev1.EnvVar{
+			// Old tailscaled config key is still used for backwards compatibility.
 			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
 			Value: "/etc/tsconfig/tailscaled",
 		},
+		corev1.EnvVar{
+			// New style is in the form of cap-<capability-version>.hujson.
+			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
+			Value: "/etc/tsconfig",
+		},
 	)
 	if sts.ForwardClusterTrafficViaL7IngressProxy {
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -505,18 +519,16 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	}
 	// Configure containeboot to run tailscaled with a configfile read from the state Secret.
 	mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash)
-	pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+
+	configVolume := corev1.Volume{
 		Name: "tailscaledconfig",
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
 				SecretName: proxySecret,
-				Items: []corev1.KeyToPath{{
-					Key:  tailscaledConfigKey,
-					Path: tailscaledConfigKey,
-				}},
 			},
 		},
-	})
+	}
+	pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
 	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
 		Name:      "tailscaledconfig",
 		ReadOnly:  true,
@@ -571,10 +583,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: proxySecret,
-					Items: []corev1.KeyToPath{{
-						Key:  "serve-config",
-						Path: "serve-config",
-					}},
+					Items:      []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
 				},
 			},
 		})
@@ -582,7 +591,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
 	if sts.ProxyClass != "" {
 		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
-		ss = applyProxyClassToStatefulSet(proxyClass, ss)
+		ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger)
 	}
 	updateSS := func(s *appsv1.StatefulSet) {
 		s.Spec = ss.Spec
@@ -613,8 +622,28 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [
 	return custom
 }
 
-func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet {
-	if pc == nil || ss == nil || pc.Spec.StatefulSet == nil {
+func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet {
+	if pc == nil || ss == nil {
+		return ss
+	}
+	if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
+		if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+			enableMetrics(ss, pc)
+		} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
+			// TODO (irbekrm): fix this
+			// For Ingress proxies that have been configured with
+			// tailscale.com/experimental-forward-cluster-traffic-via-ingress
+			// annotation, all cluster traffic is forwarded to the
+			// Ingress backend(s).
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		} else {
+			// TODO (irbekrm): fix this
+			// For egress proxies, currently all cluster traffic is forwarded to the tailnet target.
+			logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
+		}
+	}
+
+	if pc.Spec.StatefulSet == nil {
 		return ss
 	}
 
@@ -681,42 +710,97 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet)
 	return ss
 }
 
+func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) {
+	for i, c := range ss.Spec.Template.Spec.Containers {
+		if c.Name == "tailscale" {
+			// Serve metrics on on <pod-ip>:9001/debug/metrics. If
+			// we didn't specify Pod IP here, the proxy would, in
+			// some cases, also listen to its Tailscale IP- we don't
+			// want folks to start relying on this side-effect as a
+			// feature.
+			ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
+			ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001})
+			break
+		}
+	}
+}
+
+func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
+	origConf := &ipn.ConfigVAlpha{}
+	if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil {
+		return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err)
+	}
+	return origConf.AuthKey, nil
+}
+
 // tailscaledConfig takes a proxy config, a newly generated auth key if
 // generated and a Secret with the previous proxy state and auth key and
-// produces returns tailscaled configuration and a hash of that configuration.
-func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) ([]byte, string, error) {
-	conf := ipn.ConfigVAlpha{
-		Version:      "alpha0",
-		AcceptDNS:    "false",
-		AcceptRoutes: "false", // AcceptRoutes defaults to true
-		Locked:       "false",
-		Hostname:     &stsC.Hostname,
+// returns tailscaled configuration and a hash of that configuration.
+//
+// As of 2024-05-09 it also returns legacy tailscaled config without the
+// later added NoStatefulFilter field to support proxies older than cap95.
+// TODO (irbekrm): remove the legacy config once we no longer need to support
+// versions older than cap94,
+// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaleConfigs, error) {
+	conf := &ipn.ConfigVAlpha{
+		Version:             "alpha0",
+		AcceptDNS:           "false",
+		AcceptRoutes:        "false", // AcceptRoutes defaults to true
+		Locked:              "false",
+		Hostname:            &stsC.Hostname,
+		NoStatefulFiltering: "false",
+	}
+
+	// For egress proxies only, we need to ensure that stateful filtering is
+	// not in place so that traffic from cluster can be forwarded via
+	// Tailscale IPs.
+	if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
+		conf.NoStatefulFiltering = "true"
 	}
 	if stsC.Connector != nil {
 		routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
 		if err != nil {
-			return nil, "", fmt.Errorf("error calculating routes: %w", err)
+			return nil, fmt.Errorf("error calculating routes: %w", err)
 		}
 		conf.AdvertiseRoutes = routes
 	}
 	if newAuthkey != "" {
 		conf.AuthKey = &newAuthkey
-	} else if oldSecret != nil && len(oldSecret.Data[tailscaledConfigKey]) > 0 { // write to StringData, read from Data as StringData is write-only
-		origConf := &ipn.ConfigVAlpha{}
-		if err := json.Unmarshal([]byte(oldSecret.Data[tailscaledConfigKey]), origConf); err != nil {
-			return nil, "", fmt.Errorf("error unmarshaling previous tailscaled config: %w", err)
+	} else if oldSecret != nil {
+		var err error
+		latest := tailcfg.CapabilityVersion(-1)
+		latestStr := ""
+		for k, data := range oldSecret.Data {
+			// write to StringData, read from Data as StringData is write-only
+			if len(data) == 0 {
+				continue
+			}
+			v, err := kubeutils.CapVerFromFileName(k)
+			if err != nil {
+				continue
+			}
+			if v > latest {
+				latestStr = k
+				latest = v
+			}
+		}
+		// Allow for configs that don't contain an auth key. Perhaps
+		// users have some mechanisms to delete them. Auth key is
+		// normally not needed after the initial login.
+		if latestStr != "" {
+			conf.AuthKey, err = readAuthKey(oldSecret, latestStr)
+			if err != nil {
+				return nil, err
+			}
 		}
-		conf.AuthKey = origConf.AuthKey
 	}
-	confFileBytes, err := json.Marshal(conf)
-	if err != nil {
-		return nil, "", fmt.Errorf("error marshaling tailscaled config : %w", err)
-	}
-	hash, err := hashBytes(confFileBytes)
-	if err != nil {
-		return nil, "", fmt.Errorf("error calculating config hash: %w", err)
-	}
-	return confFileBytes, hash, nil
+	capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
+	capVerConfigs[95] = *conf
+	// legacy config should not contain NoStatefulFiltering field.
+	conf.NoStatefulFiltering.Clear()
+	capVerConfigs[94] = *conf
+	return capVerConfigs, nil
 }
 
 // ptrObject is a type constraint for pointer types that implement
@@ -726,7 +810,9 @@ type ptrObject[T any] interface {
 	*T
 }
 
-// hashBytes produces a hash for the provided bytes that is the same across
+type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha
+
+// hashBytes produces a hash for the provided tailscaled config that is the same across
 // different invocations of this code. We do not use the
 // tailscale.com/deephash.Hash here because that produces a different hash for
 // the same value in different tailscale builds. The hash we are producing here
@@ -735,10 +821,13 @@ type ptrObject[T any] interface {
 // thing that changed is operator version (the hash is also exposed to users via
 // an annotation and might be confusing if it changes without the config having
 // changed).
-func hashBytes(b []byte) (string, error) {
-	h := sha256.New()
-	_, err := h.Write(b)
+func tailscaledConfigHash(c tailscaleConfigs) (string, error) {
+	b, err := json.Marshal(c)
 	if err != nil {
+		return "", fmt.Errorf("error marshalling tailscaled configs: %w", err)
+	}
+	h := sha256.New()
+	if _, err = h.Write(b); err != nil {
 		return "", fmt.Errorf("error calculating hash: %w", err)
 	}
 	return fmt.Sprintf("%x", h.Sum(nil)), nil

cmd/k8s-operator/sts_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -51,6 +52,10 @@ func Test_statefulSetNameBase(t *testing.T) {
 }
 
 func Test_applyProxyClassToStatefulSet(t *testing.T) {
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Setup
 	proxyClassAllOpts := &tsapi.ProxyClass{
 		Spec: tsapi.ProxyClassSpec{
@@ -105,6 +110,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 			},
 		},
 	}
+	proxyClassMetrics := &tsapi.ProxyClass{
+		Spec: tsapi.ProxyClassSpec{
+			Metrics: &tsapi.Metrics{Enable: true},
+		},
+	}
+
 	var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
 	if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil {
 		t.Fatalf("unmarshaling userspace proxy template: %v", err)
@@ -149,7 +160,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
 
-	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy())
+	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
@@ -162,7 +173,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
 	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
-	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy())
+	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
 	}
@@ -183,7 +194,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
-	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy())
+	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
@@ -195,10 +206,19 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
 	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
-	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy())
+	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
 		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
+
+	// 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet.
+	wantSS = nonUserspaceProxySS.DeepCopy()
+	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
+	wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}}
+	gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
+	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
+		t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
+	}
 }
 
 func mergeMapKeys(a, b map[string]string) map[string]string {

cmd/k8s-operator/svc.go

@@ -90,7 +90,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	} else if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
-	targetIP := a.tailnetTargetAnnotation(svc)
+	targetIP := tailnetTargetAnnotation(svc)
 	targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN]
 	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && targetIP == "" && targetFQDN == "" {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
@@ -161,7 +161,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	}
 	if violations := validateService(svc); len(violations) > 0 {
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", "))
-		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVCICE", msg)
+		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
 		a.logger.Error(msg)
 		return nil
 	}
@@ -216,7 +216,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		sts.ClusterTargetDNSName = svc.Spec.ExternalName
 		a.managedIngressProxies.Add(svc.UID)
 		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
-	} else if ip := a.tailnetTargetAnnotation(svc); ip != "" {
+	} else if ip := tailnetTargetAnnotation(svc); ip != "" {
 		sts.TailnetTargetIP = ip
 		a.managedEgressProxies.Add(svc.UID)
 		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
@@ -250,7 +250,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	if !a.hasLoadBalancerClass(svc) {
+	if !isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) {
 		logger.Debugf("service is not a LoadBalancer, so not updating ingress")
 		return nil
 	}
@@ -310,29 +310,27 @@ func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
 	return a.shouldExposeClusterIP(svc) || a.shouldExposeDNSName(svc)
 }
 
+func (a *ServiceReconciler) shouldExposeDNSName(svc *corev1.Service) bool {
+	return hasExposeAnnotation(svc) && svc.Spec.Type == corev1.ServiceTypeExternalName && svc.Spec.ExternalName != ""
+}
+
 func (a *ServiceReconciler) shouldExposeClusterIP(svc *corev1.Service) bool {
-	// Headless services can't be exposed, since there is no ClusterIP to
-	// forward to.
 	if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
 		return false
 	}
-	return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc)
+	return isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) || hasExposeAnnotation(svc)
 }
 
-func (a *ServiceReconciler) shouldExposeDNSName(svc *corev1.Service) bool {
-	return a.hasExposeAnnotation(svc) && svc.Spec.Type == corev1.ServiceTypeExternalName && svc.Spec.ExternalName != ""
-}
-
-func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
+func isTailscaleLoadBalancerService(svc *corev1.Service, isDefaultLoadBalancer bool) bool {
 	return svc != nil &&
 		svc.Spec.Type == corev1.ServiceTypeLoadBalancer &&
 		(svc.Spec.LoadBalancerClass != nil && *svc.Spec.LoadBalancerClass == "tailscale" ||
-			svc.Spec.LoadBalancerClass == nil && a.isDefaultLoadBalancer)
+			svc.Spec.LoadBalancerClass == nil && isDefaultLoadBalancer)
 }
 
 // hasExposeAnnotation reports whether Service has the tailscale.com/expose
 // annotation set
-func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
+func hasExposeAnnotation(svc *corev1.Service) bool {
 	return svc != nil && svc.Annotations[AnnotationExpose] == "true"
 }
 
@@ -340,7 +338,7 @@ func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
 // annotation or of the deprecated tailscale.com/ts-tailnet-target-ip
 // annotation. If neither is set, it returns an empty string. If both are set,
 // it returns the value of the new annotation.
-func (a *ServiceReconciler) tailnetTargetAnnotation(svc *corev1.Service) string {
+func tailnetTargetAnnotation(svc *corev1.Service) string {
 	if svc == nil {
 		return ""
 	}

cmd/k8s-operator/testutils_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -54,6 +55,10 @@ type configOpts struct {
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
 	t.Helper()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
 	tsContainer := corev1.Container{
 		Name:  "tailscale",
 		Image: "tailscale/tailscale",
@@ -62,6 +67,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
+			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
 			Capabilities: &corev1.Capabilities{
@@ -84,12 +90,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: opts.secretName,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "tailscaled",
-							Path: "tailscaled",
-						},
-					},
 				},
 			},
 		},
@@ -139,9 +139,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			Name:  "TS_SERVE_CONFIG",
 			Value: "/etc/tailscaled/serve-config",
 		})
-		volumes = append(volumes, corev1.Volume{
-			Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Path: "serve-config", Key: "serve-config"}}}},
-		})
+		volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
 		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
 	}
 	ss := &appsv1.StatefulSet{
@@ -184,8 +182,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 						{
 							Name:    "sysctler",
 							Image:   "tailscale/tailscale",
-							Command: []string{"/bin/sh"},
-							Args:    []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
+							Command: []string{"/bin/sh", "-c"},
+							Args:    []string{"sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi"},
 							SecurityContext: &corev1.SecurityContext{
 								Privileged: ptr.To(true),
 							},
@@ -205,20 +203,26 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
 			t.Fatalf("error getting ProxyClass: %v", err)
 		}
-		return applyProxyClassToStatefulSet(proxyClass, ss)
+		return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar())
 	}
 	return ss
 }
 
 func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
 	t.Helper()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
 	tsContainer := corev1.Container{
 		Name:  "tailscale",
 		Image: "tailscale/tailscale",
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
+			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
 			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
+			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 		},
 		ImagePullPolicy: "Always",
@@ -233,20 +237,12 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: opts.secretName,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "tailscaled",
-							Path: "tailscaled",
-						},
-					},
 				},
 			},
 		},
 		{Name: "serve-config",
 			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName,
-					Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}},
-		},
+				Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}},
 	}
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
@@ -301,7 +297,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 		if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
 			t.Fatalf("error getting ProxyClass: %v", err)
 		}
-		return applyProxyClassToStatefulSet(proxyClass, ss)
+		return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar())
 	}
 	return ss
 }
@@ -378,7 +374,17 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
+	if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
+		conf.NoStatefulFiltering = "true"
+	} else {
+		conf.NoStatefulFiltering = "false"
+	}
+	bn, err := json.Marshal(conf)
+	if err != nil {
+		t.Fatalf("error marshalling tailscaled config")
+	}
 	mak.Set(&s.StringData, "tailscaled", string(b))
+	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
 	labels := map[string]string{
 		"tailscale.com/managed":              "true",
 		"tailscale.com/parent-resource":      "test",
@@ -453,7 +459,7 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client,
 // they are not present in the passed object and use the modify func to remove
 // them from the cluster object. If no such modifications are needed, you can
 // pass nil in place of the modify function.
-func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modify func(O)) {
+func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) {
 	t.Helper()
 	got := O(new(T))
 	if err := client.Get(context.Background(), types.NamespacedName{
@@ -467,8 +473,8 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 	// so just remove it from both got and want.
 	got.SetResourceVersion("")
 	want.SetResourceVersion("")
-	if modify != nil {
-		modify(got)
+	if modifier != nil {
+		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
 		t.Fatalf("unexpected object (-got +want):\n%s", diff)

cmd/pgproxy/pgproxy.go

@@ -28,7 +28,6 @@ import (
 	"tailscale.com/metrics"
 	"tailscale.com/tsnet"
 	"tailscale.com/tsweb"
-	"tailscale.com/types/logger"
 )
 
 var (
@@ -58,8 +57,6 @@ func main() {
 	ts := &tsnet.Server{
 		Dir:      *tailscaleDir,
 		Hostname: *hostname,
-		// Make the stdout logs a clean audit log of connections.
-		Logf: logger.Discard,
 	}
 
 	if os.Getenv("TS_AUTHKEY") == "" {

cmd/sniproxy/sniproxy_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"log"
 	"net"
 	"net/http/httptest"
 	"net/netip"
@@ -24,6 +25,7 @@ import (
 	"tailscale.com/tsnet"
 	"tailscale.com/tstest/integration"
 	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/tstest/nettest"
 	"tailscale.com/types/appctype"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
@@ -98,8 +100,8 @@ func startNode(t *testing.T, ctx context.Context, controlURL, hostname string) (
 		Store:      new(mem.Store),
 		Ephemeral:  true,
 	}
-	if !*verboseNodes {
-		s.Logf = logger.Discard
+	if *verboseNodes {
+		s.Logf = log.Printf
 	}
 	t.Cleanup(func() { s.Close() })
 
@@ -111,6 +113,7 @@ func startNode(t *testing.T, ctx context.Context, controlURL, hostname string) (
 }
 
 func TestSNIProxyWithNetmapConfig(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
 	c, controlURL := startControl(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -189,6 +192,7 @@ func TestSNIProxyWithNetmapConfig(t *testing.T) {
 }
 
 func TestSNIProxyWithFlagConfig(t *testing.T) {
+	nettest.SkipIfNoNetwork(t)
 	_, controlURL := startControl(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()

cmd/stunc/stunc.go

@@ -20,7 +20,7 @@ func main() {
 	}
 	host := os.Args[1]
 
-	uaddr, err := net.ResolveUDPAddr("udp", host+":3478")
+	uaddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(host, "3478"))
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/tailscale/cli/cli.go

@@ -78,7 +78,9 @@ func CleanUpArgs(args []string) []string {
 	return out
 }
 
-var localClient tailscale.LocalClient
+var localClient = tailscale.LocalClient{
+	Socket: paths.DefaultTailscaledSocket(),
+}
 
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) (err error) {
@@ -139,13 +141,6 @@ func Run(args []string) (err error) {
 		return
 	}
 
-	localClient.Socket = rootArgs.socket
-	rootCmd.FlagSet.Visit(func(f *flag.Flag) {
-		if f.Name == "socket" {
-			localClient.UseSocketOnly = true
-		}
-	})
-
 	err = rootCmd.Run(context.Background())
 	if tailscale.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" {
 		return fmt.Errorf("%v\n\nUse 'sudo tailscale %s' or 'tailscale up --operator=$USER' to not require root.", err, strings.Join(args, " "))
@@ -158,7 +153,12 @@ func Run(args []string) (err error) {
 
 func newRootCmd() *ffcli.Command {
 	rootfs := newFlagSet("tailscale")
-	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled socket")
+	rootfs.Func("socket", "path to tailscaled socket", func(s string) error {
+		localClient.Socket = s
+		localClient.UseSocketOnly = true
+		return nil
+	})
+	rootfs.Lookup("socket").DefValue = localClient.Socket
 
 	rootCmd := &ffcli.Command{
 		Name:       "tailscale",
@@ -236,10 +236,6 @@ func fatalf(format string, a ...any) {
 // Fatalf, if non-nil, is used instead of log.Fatalf.
 var Fatalf func(format string, a ...any)
 
-var rootArgs struct {
-	socket string
-}
-
 type cmdWalk struct {
 	*ffcli.Command
 	parents []*ffcli.Command

cmd/tailscale/cli/cli_test.go

@@ -24,6 +24,7 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 	"tailscale.com/version/distro"
@@ -176,9 +177,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "bare_up_means_up",
 			flags: []string{},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         false,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -186,12 +188,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "losing_hostname",
 			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      false,
-				Hostname:         "foo",
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         false,
+				Hostname:            "foo",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
 		},
@@ -199,11 +201,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "hostname_changing_explicitly",
 			flags: []string{"--hostname=bar"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
-				Hostname:         "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -211,11 +213,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "hostname_changing_empty_explicitly",
 			flags: []string{"--hostname="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
-				Hostname:         "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -231,11 +233,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "implicit_operator_change",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				OperatorUser:     "alice",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				OperatorUser:        "alice",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
 			want:    accidentalUpPrefix + " --hostname=foo --operator=alice",
@@ -244,11 +246,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "implicit_operator_matches_shell_user",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				OperatorUser:     "alice",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "alice",
 			want:    "",
@@ -257,15 +259,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "error_advertised_routes_exit_node_removed",
 			flags: []string{"--advertise-routes=10.0.42.0/24"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.42.0/24"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
 		},
@@ -273,15 +275,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertised_routes_exit_node_removed_explicit",
 			flags: []string{"--advertise-routes=10.0.42.0/24", "--advertise-exit-node=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.42.0/24"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -289,15 +291,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
 			flags: []string{"--advertise-routes=11.1.43.0/24,0.0.0.0/0,::/0"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.42.0/24"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -305,10 +307,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node", // Issue 1859
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -316,14 +318,14 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node_over_existing_routes",
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -331,15 +333,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node_over_existing_routes_and_exit_node",
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -347,12 +349,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "exit_node_clearing", // Issue 1777
 			flags: []string{"--exit-node="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeID: "fooID",
+				ExitNodeID:          "fooID",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -360,59 +362,59 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "remove_all_implicit",
 			flags: []string{"--force-reauth"},
 			curPrefs: &ipn.Prefs{
-				WantRunning:      true,
-				ControlURL:       ipn.DefaultControlURL,
-				RouteAll:         true,
-				AllowSingleHosts: false,
-				ExitNodeIP:       netip.MustParseAddr("100.64.5.6"),
-				CorpDNS:          false,
-				ShieldsUp:        true,
-				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
-				Hostname:         "myhostname",
-				ForceDaemon:      true,
+				WantRunning:   true,
+				ControlURL:    ipn.DefaultControlURL,
+				RouteAll:      true,
+				ExitNodeIP:    netip.MustParseAddr("100.64.5.6"),
+				CorpDNS:       false,
+				ShieldsUp:     true,
+				AdvertiseTags: []string{"tag:foo", "tag:bar"},
+				Hostname:      "myhostname",
+				ForceDaemon:   true,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.0.0/16"),
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
-			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
+			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
 		},
 		{
 			name:  "remove_all_implicit_except_hostname",
 			flags: []string{"--hostname=newhostname"},
 			curPrefs: &ipn.Prefs{
-				WantRunning:      true,
-				ControlURL:       ipn.DefaultControlURL,
-				RouteAll:         true,
-				AllowSingleHosts: false,
-				ExitNodeIP:       netip.MustParseAddr("100.64.5.6"),
-				CorpDNS:          false,
-				ShieldsUp:        true,
-				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
-				Hostname:         "myhostname",
-				ForceDaemon:      true,
+				WantRunning:   true,
+				ControlURL:    ipn.DefaultControlURL,
+				RouteAll:      true,
+				ExitNodeIP:    netip.MustParseAddr("100.64.5.6"),
+				CorpDNS:       false,
+				ShieldsUp:     true,
+				AdvertiseTags: []string{"tag:foo", "tag:bar"},
+				Hostname:      "myhostname",
+				ForceDaemon:   true,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.0.0/16"),
 				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
-			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --netfilter-mode=nodivert --operator=alice --shields-up",
+			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --netfilter-mode=nodivert --operator=alice --shields-up",
 		},
 		{
 			name:  "loggedout_is_implicit",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				LoggedOut:        true,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				LoggedOut:           true,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "", // not an error. LoggedOut is implicit.
 		},
@@ -422,10 +424,9 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "make_windows_exit_node",
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				RouteAll:         true,
+				ControlURL: ipn.DefaultControlURL,
+				CorpDNS:    true,
+				RouteAll:   true,
 
 				// And assume this no-op accidental pre-1.8 value:
 				NoSNAT: true,
@@ -437,8 +438,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_netfilter_change_non_linux",
 			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
+				ControlURL: ipn.DefaultControlURL,
 
 				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
 			},
@@ -449,15 +449,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "operator_losing_routes_step1", // https://twitter.com/EXPbits/status/1390418145047887877
 			flags: []string{"--operator=expbits"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --operator=expbits --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -465,15 +465,15 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "operator_losing_routes_step2", // https://twitter.com/EXPbits/status/1390418145047887877
 			flags: []string{"--operator=expbits", "--advertise-routes=1.2.0.0/16"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-routes=1.2.0.0/16 --operator=expbits --advertise-exit-node",
 		},
@@ -481,13 +481,13 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "errors_preserve_explicit_flags",
 			flags: []string{"--reset", "--force-reauth=false", "--authkey=secretrand"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      false,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				AllowSingleHosts: true,
+				ControlURL:    ipn.DefaultControlURL,
+				WantRunning:   false,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				Hostname: "foo",
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --auth-key=secretrand --force-reauth=false --reset --hostname=foo",
 		},
@@ -495,12 +495,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "error_exit_node_omit_with_ip_pref",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeIP: netip.MustParseAddr("100.64.5.4"),
+				ExitNodeIP:          netip.MustParseAddr("100.64.5.4"),
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.4",
 		},
@@ -509,12 +509,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			flags:         []string{"--hostname=foo"},
 			curExitNodeIP: netip.MustParseAddr("100.64.5.7"),
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeID: "some_stable_id",
+				ExitNodeID:          "some_stable_id",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
@@ -523,13 +523,13 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			flags:         []string{"--hostname=foo"},
 			curExitNodeIP: netip.MustParseAddr("100.2.3.4"),
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 
 				ExitNodeAllowLANAccess: true,
 				ExitNodeID:             "some_stable_id",
+				NoStatefulFiltering:    opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node-allow-lan-access --exit-node=100.2.3.4",
 		},
@@ -537,10 +537,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_login_server_synonym",
 			flags: []string{"--login-server=https://controlplane.tailscale.com"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "", // not an error
 		},
@@ -548,10 +548,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_login_server_synonym_on_other_change",
 			flags: []string{"--netfilter-mode=off"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          false,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             false,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
 		},
@@ -561,11 +561,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "synology_permit_omit_accept_routes",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				RouteAll:         true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				RouteAll:            true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos:   "linux",
 			distro: distro.Synology,
@@ -577,11 +577,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "not_synology_dont_permit_omit_accept_routes",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				RouteAll:         true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				RouteAll:            true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos:   "linux",
 			distro: "", // not Synology
@@ -591,11 +591,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "profile_name_ignored_in_up",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				NetfilterMode:    preftype.NetfilterOn,
-				ProfileName:      "foo",
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				ProfileName:         "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos: "linux",
 			want: "",
@@ -655,12 +655,12 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			goos: "linux",
 			args: upArgsFromOSArgs("linux"),
 			want: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      true,
-				NoSNAT:           false,
-				NetfilterMode:    preftype.NetfilterOn,
-				CorpDNS:          true,
-				AllowSingleHosts: true,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         true,
+				NoSNAT:              false,
+				NoStatefulFiltering: "true",
+				NetfilterMode:       preftype.NetfilterOn,
+				CorpDNS:             true,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -671,12 +671,13 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			goos: "windows",
 			args: upArgsFromOSArgs("windows"),
 			want: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      true,
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				RouteAll:         true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         true,
+				CorpDNS:             true,
+				RouteAll:            true,
+				NoSNAT:              false,
+				NoStatefulFiltering: "true",
+				NetfilterMode:       preftype.NetfilterOn,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -686,15 +687,15 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			name: "advertise_default_route",
 			args: upArgsFromOSArgs("linux", "--advertise-exit-node"),
 			want: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				WantRunning:      true,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
+				ControlURL:  ipn.DefaultControlURL,
+				WantRunning: true,
+				CorpDNS:     true,
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
-				NetfilterMode: preftype.NetfilterOn,
+				NoStatefulFiltering: "true",
+				NetfilterMode:       preftype.NetfilterOn,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -781,9 +782,10 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			},
 			wantWarn: "netfilter=nodivert; add iptables calls to ts-* chains manually.",
 			want: &ipn.Prefs{
-				WantRunning:   true,
-				NetfilterMode: preftype.NetfilterNoDivert,
-				NoSNAT:        true,
+				WantRunning:         true,
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -797,9 +799,10 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			},
 			wantWarn: "netfilter=off; configure iptables yourself.",
 			want: &ipn.Prefs{
-				WantRunning:   true,
-				NetfilterMode: preftype.NetfilterOff,
-				NoSNAT:        true,
+				WantRunning:         true,
+				NetfilterMode:       preftype.NetfilterOff,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
 				},
@@ -813,8 +816,9 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				netfilterMode:   "off",
 			},
 			want: &ipn.Prefs{
-				WantRunning: true,
-				NoSNAT:      true,
+				WantRunning:         true,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("fd7a:115c:a1e0:b1a::bb:10.0.0.0/112"),
 				},
@@ -831,8 +835,9 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				netfilterMode:   "off",
 			},
 			want: &ipn.Prefs{
-				WantRunning: true,
-				NoSNAT:      true,
+				WantRunning:         true,
+				NoSNAT:              true,
+				NoStatefulFiltering: "true",
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("fd7a:115c:a1e0:b1a::aabb:10.0.0.0/112"),
 				},
@@ -914,6 +919,9 @@ func TestPrefFlagMapping(t *testing.T) {
 			continue
 		}
 		switch prefName {
+		case "AllowSingleHosts":
+			// Fake pref for downgrade compat. See #12058.
+			continue
 		case "WantRunning", "Persist", "LoggedOut":
 			// All explicitly handled (ignored) by checkForAccidentalSettingReverts.
 			continue
@@ -1021,7 +1029,6 @@ func TestUpdatePrefs(t *testing.T) {
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AdvertiseRoutesSet:        true,
 				AdvertiseTagsSet:          true,
-				AllowSingleHostsSet:       true,
 				AppConnectorSet:           true,
 				ControlURLSet:             true,
 				CorpDNSSet:                true,
@@ -1031,6 +1038,7 @@ func TestUpdatePrefs(t *testing.T) {
 				HostnameSet:               true,
 				NetfilterModeSet:          true,
 				NoSNATSet:                 true,
+				NoStatefulFilteringSet:    true,
 				OperatorUserSet:           true,
 				RouteAllSet:               true,
 				RunSSHSet:                 true,
@@ -1053,11 +1061,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "change_login_server",
 			flags: []string{"--login-server=https://localhost:1000"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env:            upCheckEnv{backendState: "Running"},
 			wantSimpleUp:   true,
@@ -1068,11 +1076,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "change_tags",
 			flags: []string{"--advertise-tags=tag:foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env: upCheckEnv{backendState: "Running"},
 		},
@@ -1081,11 +1089,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "explicit_empty_operator",
 			flags: []string{"--operator="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				CorpDNS:          true,
-				AllowSingleHosts: true,
-				NetfilterMode:    preftype.NetfilterOn,
-				OperatorUser:     "somebody",
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				OperatorUser:        "somebody",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env: upCheckEnv{user: "somebody", backendState: "Running"},
 			wantJustEditMP: &ipn.MaskedPrefs{
@@ -1102,11 +1110,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "enable_ssh",
 			flags: []string{"--ssh"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1123,12 +1131,12 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "disable_ssh",
 			flags: []string{"--ssh=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				RunSSH:           true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				RunSSH:              true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1148,12 +1156,12 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=false"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
-				RunSSH:           true,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				RunSSH:              true,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1172,11 +1180,11 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=true"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1195,11 +1203,11 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=true", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1217,12 +1225,12 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=false", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				Persist:          &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				RunSSH:           true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				RunSSH:              true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1240,10 +1248,10 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--force-reauth"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env:          upCheckEnv{backendState: "Running"},
 			wantErrSubtr: "aborted, no changes made",
@@ -1253,10 +1261,10 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--force-reauth", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:       "https://login.tailscale.com",
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: nil,
 			env:            upCheckEnv{backendState: "Running"},
@@ -1265,10 +1273,10 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "advertise_connector",
 			flags: []string{"--advertise-connector"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AppConnectorSet: true,
@@ -1285,13 +1293,13 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "no_advertise_connector",
 			flags: []string{"--advertise-connector=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:       ipn.DefaultControlURL,
-				AllowSingleHosts: true,
-				CorpDNS:          true,
-				NetfilterMode:    preftype.NetfilterOn,
+				ControlURL:    ipn.DefaultControlURL,
+				CorpDNS:       true,
+				NetfilterMode: preftype.NetfilterOn,
 				AppConnector: ipn.AppConnectorPrefs{
 					Advertise: true,
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AppConnectorSet: true,

cmd/tailscale/cli/ffcomplete/complete.go

@@ -97,7 +97,7 @@ func Inject(root *ffcli.Command, hide func(*ffcli.Command), usageFunc func(*ffcl
 		&ffcli.Command{
 			Name:       "completion",
 			ShortUsage: root.Name + " completion <shell> [--flags] [--descs]",
-			ShortHelp:  "Shell tab-completion scripts.",
+			ShortHelp:  "Shell tab-completion scripts",
 			LongHelp:   fmt.Sprintf(cobra.UsageTemplate, root.Name),
 
 			// Print help if run without args.

cmd/tailscale/cli/ffcomplete/internal/complete.go

@@ -193,12 +193,26 @@ walk:
 	}
 
 	// Strip any descriptions if they were suppressed.
-	if !descs {
-		for i := range words {
-			words[i], _, _ = strings.Cut(words[i], "\t")
+	clean := words[:0]
+	for _, w := range words {
+		if !descs {
+			w, _, _ = strings.Cut(w, "\t")
 		}
+		w = cutAny(w, "\n\r")
+		if w == "" || w[0] == '\t' {
+			continue
+		}
+		clean = append(clean, w)
 	}
-	return words, dir, nil
+	return clean, dir, nil
+}
+
+func cutAny(s, cutset string) string {
+	i := strings.IndexAny(s, cutset)
+	if i == -1 {
+		return s
+	}
+	return s[:i]
 }
 
 // splitFlagArgs separates a list of command-line arguments into arguments

cmd/tailscale/cli/ffcomplete/internal/complete_test.go

@@ -50,11 +50,17 @@ func TestComplete(t *testing.T) {
 				cmd := &ffcli.Command{
 					Name: "ping",
 					FlagSet: newFlagSet("prog ping", flag.ContinueOnError, func(fs *flag.FlagSet) {
-						fs.String("until", "", "when pinging should end")
+						fs.String("until", "", "when pinging should end\nline break!")
 						ffcomplete.Flag(fs, "until", ffcomplete.Fixed("forever", "direct"))
 					}),
 				}
-				ffcomplete.Args(cmd, ffcomplete.Fixed("jupiter", "neptune", "venus"))
+				ffcomplete.Args(cmd, ffcomplete.Fixed(
+					"jupiter\t5th planet\nand largets",
+					"neptune\t8th planet",
+					"venus\t2nd planet",
+					"\tonly description",
+					"\nonly line break",
+				))
 				return cmd
 			}(),
 		},
@@ -170,9 +176,9 @@ func TestComplete(t *testing.T) {
 			showDescs: true,
 			wantComp: []string{
 				"--until\twhen pinging should end",
-				"jupiter",
-				"neptune",
-				"venus",
+				"jupiter\t5th planet",
+				"neptune\t8th planet",
+				"venus\t2nd planet",
 			},
 			wantDir: ffcomplete.ShellCompDirectiveNoFileComp,
 		},

cmd/tailscale/cli/netcheck.go

@@ -53,6 +53,7 @@ func runNetcheck(ctx context.Context, args []string) error {
 		return err
 	}
 	c := &netcheck.Client{
+		NetMon:      netMon,
 		PortMapper:  portmapper.NewClient(logf, netMon, nil, nil, nil),
 		UseDNSCache: false, // always resolve, don't cache
 	}
@@ -126,13 +127,13 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 
 	printf("\nReport:\n")
 	printf("\t* UDP: %v\n", report.UDP)
-	if report.GlobalV4 != "" {
-		printf("\t* IPv4: yes, %v\n", report.GlobalV4)
+	if report.GlobalV4.IsValid() {
+		printf("\t* IPv4: yes, %s\n", report.GlobalV4)
 	} else {
 		printf("\t* IPv4: (no addr found)\n")
 	}
-	if report.GlobalV6 != "" {
-		printf("\t* IPv6: yes, %v\n", report.GlobalV6)
+	if report.GlobalV6.IsValid() {
+		printf("\t* IPv6: yes, %s\n", report.GlobalV6)
 	} else if report.IPv6 {
 		printf("\t* IPv6: (no addr found)\n")
 	} else if report.OSHasIPv6 {
@@ -141,7 +142,6 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 		printf("\t* IPv6: no, unavailable in OS\n")
 	}
 	printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
-	printf("\t* HairPinning: %v\n", report.HairPinning)
 	printf("\t* PortMapping: %v\n", portMapping(report))
 	if report.CaptivePortal != "" {
 		printf("\t* CaptivePortal: %v\n", report.CaptivePortal)

cmd/tailscale/cli/network-lock.go

@@ -222,7 +222,8 @@ func runNetworkLockStatus(ctx context.Context, args []string) error {
 
 	if st.Enabled && st.NodeKey != nil && !st.PublicKey.IsZero() {
 		if st.NodeKeySigned {
-			fmt.Println("This node is accessible under tailnet lock.")
+			fmt.Println("This node is accessible under tailnet lock. Node signature:")
+			fmt.Println(st.NodeKeySignature.String())
 		} else {
 			fmt.Println("This node is LOCKED OUT by tailnet-lock, and action is required to establish connectivity.")
 			fmt.Printf("Run the following command on a node with a trusted key:\n\ttailscale lock sign %v %s\n", st.NodeKey, st.PublicKey.CLIString())

cmd/tailscale/cli/set.go

@@ -58,6 +58,9 @@ type setArgsT struct {
 	updateCheck            bool
 	updateApply            bool
 	postureChecking        bool
+	snat                   bool
+	statefulFiltering      bool
+	netfilterMode          string
 }
 
 func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet {
@@ -98,6 +101,10 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet {
 		setf.StringVar(&setArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
 	}
 	switch goos {
+	case "linux":
+		setf.BoolVar(&setArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
+		setf.BoolVar(&setArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)")
+		setf.StringVar(&setArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
 	case "windows":
 		setf.BoolVar(&setArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")
 	}
@@ -121,6 +128,9 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 		return err
 	}
 
+	// Note that even though we set the values here regardless of whether the
+	// user passed the flag, the value is only used if the user passed the flag.
+	// See updateMaskedPrefsFromUpOrSetFlag.
 	maskedPrefs := &ipn.MaskedPrefs{
 		Prefs: ipn.Prefs{
 			ProfileName:            setArgs.profileName,
@@ -132,6 +142,7 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 			RunWebClient:           setArgs.runWebClient,
 			Hostname:               setArgs.hostname,
 			OperatorUser:           setArgs.opUser,
+			NoSNAT:                 !setArgs.snat,
 			ForceDaemon:            setArgs.forceDaemon,
 			AutoUpdate: ipn.AutoUpdatePrefs{
 				Check: setArgs.updateCheck,
@@ -140,10 +151,22 @@ func runSet(ctx context.Context, args []string) (retErr error) {
 			AppConnector: ipn.AppConnectorPrefs{
 				Advertise: setArgs.advertiseConnector,
 			},
-			PostureChecking: setArgs.postureChecking,
+			PostureChecking:     setArgs.postureChecking,
+			NoStatefulFiltering: opt.NewBool(!setArgs.statefulFiltering),
 		},
 	}
 
+	if effectiveGOOS() == "linux" {
+		nfMode, warning, err := netfilterModeFromFlag(setArgs.netfilterMode)
+		if err != nil {
+			return err
+		}
+		if warning != "" {
+			warnf(warning)
+		}
+		maskedPrefs.Prefs.NetfilterMode = nfMode
+	}
+
 	if setArgs.exitNodeIP != "" {
 		if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil {
 			var e ipn.ExitNodeLocalIPError

cmd/tailscale/cli/ssh.go

@@ -110,8 +110,8 @@ func runSSH(ctx context.Context, args []string) error {
 	// mode, so 'nc' isn't very useful.
 	if runtime.GOOS != "darwin" {
 		socketArg := ""
-		if rootArgs.socket != "" && rootArgs.socket != paths.DefaultTailscaledSocket() {
-			socketArg = fmt.Sprintf("--socket=%q", rootArgs.socket)
+		if localClient.Socket != "" && localClient.Socket != paths.DefaultTailscaledSocket() {
+			socketArg = fmt.Sprintf("--socket=%q", localClient.Socket)
 		}
 
 		argv = append(argv,

cmd/tailscale/cli/status.go

@@ -23,7 +23,7 @@ import (
 	"golang.org/x/net/idna"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/net/interfaces"
+	"tailscale.com/net/netmon"
 	"tailscale.com/util/dnsname"
 )
 
@@ -102,7 +102,7 @@ func runStatus(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		statusURL := interfaces.HTTPOfListener(ln)
+		statusURL := netmon.HTTPOfListener(ln)
 		printf("Serving Tailscale status at %v ...\n", statusURL)
 		go func() {
 			<-ctx.Done()

cmd/tailscale/cli/up.go

@@ -20,7 +20,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -105,7 +104,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet {
 	upf.StringVar(&upArgs.server, "login-server", ipn.DefaultControlURL, "base URL of control server")
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
-	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, hidden+"install host routes to other Tailscale nodes")
+	upf.Var(notFalseVar{}, "host-routes", hidden+"install host routes to other Tailscale nodes (must be true as of Tailscale 1.67+)")
 	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
@@ -122,6 +121,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet {
 	switch goos {
 	case "linux":
 		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
+		upf.BoolVar(&upArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)")
 		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
 	case "windows":
 		upf.BoolVar(&upArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")
@@ -143,6 +143,18 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet {
 	return upf
 }
 
+// notFalseVar is is a flag.Value that can only be "true", if set.
+type notFalseVar struct{}
+
+func (notFalseVar) IsBoolFlag() bool { return true }
+func (notFalseVar) Set(v string) error {
+	if v != "true" {
+		return fmt.Errorf("unsupported value; only 'true' is allowed")
+	}
+	return nil
+}
+func (notFalseVar) String() string { return "true" }
+
 func defaultNetfilterMode() string {
 	if distro.Get() == distro.Synology {
 		return "off"
@@ -156,7 +168,6 @@ type upArgsT struct {
 	server                 string
 	acceptRoutes           bool
 	acceptDNS              bool
-	singleRoutes           bool
 	exitNodeIP             string
 	exitNodeAllowLANAccess bool
 	shieldsUp              bool
@@ -169,6 +180,7 @@ type upArgsT struct {
 	advertiseTags          string
 	advertiseConnector     bool
 	snat                   bool
+	statefulFiltering      bool
 	netfilterMode          string
 	authKeyOrFile          string // "secret" or "file:/path/to/secret"
 	hostname               string
@@ -277,7 +289,6 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 
 	prefs.ExitNodeAllowLANAccess = upArgs.exitNodeAllowLANAccess
 	prefs.CorpDNS = upArgs.acceptDNS
-	prefs.AllowSingleHosts = upArgs.singleRoutes
 	prefs.ShieldsUp = upArgs.shieldsUp
 	prefs.RunSSH = upArgs.runSSH
 	prefs.RunWebClient = upArgs.runWebClient
@@ -292,24 +303,44 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	if goos == "linux" {
 		prefs.NoSNAT = !upArgs.snat
 
-		switch upArgs.netfilterMode {
-		case "on":
-			prefs.NetfilterMode = preftype.NetfilterOn
-		case "nodivert":
-			prefs.NetfilterMode = preftype.NetfilterNoDivert
-			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
-		case "off":
-			prefs.NetfilterMode = preftype.NetfilterOff
-			if defaultNetfilterMode() != "off" {
-				warnf("netfilter=off; configure iptables yourself.")
-			}
-		default:
-			return nil, fmt.Errorf("invalid value --netfilter-mode=%q", upArgs.netfilterMode)
+		// Backfills for NoStatefulFiltering occur when loading a profile; just set it explicitly here.
+		prefs.NoStatefulFiltering.Set(!upArgs.statefulFiltering)
+		v, warning, err := netfilterModeFromFlag(upArgs.netfilterMode)
+		if err != nil {
+			return nil, err
+		}
+		prefs.NetfilterMode = v
+		if warning != "" {
+			warnf(warning)
 		}
 	}
 	return prefs, nil
 }
 
+// netfilterModeFromFlag returns the preftype.NetfilterMode for the provided
+// flag value. It returns a warning if there is something the user should know
+// about the value.
+func netfilterModeFromFlag(v string) (_ preftype.NetfilterMode, warning string, _ error) {
+	switch v {
+	case "on", "nodivert", "off":
+	default:
+		return preftype.NetfilterOn, "", fmt.Errorf("invalid value --netfilter-mode=%q", v)
+	}
+	m, err := preftype.ParseNetfilterMode(v)
+	if err != nil {
+		return preftype.NetfilterOn, "", err
+	}
+	switch m {
+	case preftype.NetfilterNoDivert:
+		warning = "netfilter=nodivert; add iptables calls to ts-* chains manually."
+	case preftype.NetfilterOff:
+		if defaultNetfilterMode() != "off" {
+			warning = "netfilter=off; configure iptables yourself."
+		}
+	}
+	return m, warning, nil
+}
+
 // updatePrefs returns how to edit preferences based on the
 // flag-provided 'prefs' and the currently active 'curPrefs'.
 //
@@ -406,6 +437,11 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 	// printAuthURL reports whether we should print out the
 	// provided auth URL from an IPN notify.
 	printAuthURL := func(url string) bool {
+		if url == "" {
+			// Probably unnecessary but we used to have a bug where tailscaled
+			// could send an empty URL over the IPN bus. ~Harmless to keep.
+			return false
+		}
 		if upArgs.authKeyOrFile != "" {
 			// Issue 1755: when using an authkey, don't
 			// show an authURL that might still be pending
@@ -477,11 +513,6 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 
 	watchCtx, cancelWatch := context.WithCancel(ctx)
 	defer cancelWatch()
-	watcher, err := localClient.WatchIPNBus(watchCtx, 0)
-	if err != nil {
-		return err
-	}
-	defer watcher.Close()
 
 	go func() {
 		interrupt := make(chan os.Signal, 1)
@@ -494,29 +525,62 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 	}()
 
 	running := make(chan bool, 1) // gets value once in state ipn.Running
-	pumpErr := make(chan error, 1)
+	watchErr := make(chan error, 1)
 
-	// localAPIMu should be held while doing mutable LocalAPI calls
-	// to the backend. In particular, it prevents StartLoginInteractive from
-	// being called from the watcher goroutine while the Start call from
-	// the other goroutine is in progress.
-	// See https://github.com/tailscale/tailscale/issues/7036#issuecomment-2053771466
-	// TODO(bradfitz): simplify this once #11649 is cleaned up and Start is
-	// hopefully removed.
-	var localAPIMu sync.Mutex
+	// Special case: bare "tailscale up" means to just start
+	// running, if there's ever been a login.
+	if simpleUp {
+		_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
+			Prefs: ipn.Prefs{
+				WantRunning: true,
+			},
+			WantRunningSet: true,
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		if err := localClient.CheckPrefs(ctx, prefs); err != nil {
+			return err
+		}
 
-	startLoginInteractive := sync.OnceFunc(func() {
-		localAPIMu.Lock()
-		defer localAPIMu.Unlock()
-		localClient.StartLoginInteractive(ctx)
-	})
+		authKey, err := upArgs.getAuthKey()
+		if err != nil {
+			return err
+		}
+		authKey, err = resolveAuthKey(ctx, authKey, upArgs.advertiseTags)
+		if err != nil {
+			return err
+		}
+		err = localClient.Start(ctx, ipn.Options{
+			AuthKey:     authKey,
+			UpdatePrefs: prefs,
+		})
+		if err != nil {
+			return err
+		}
+		if upArgs.forceReauth || !st.HaveNodeKey {
+			err := localClient.StartLoginInteractive(ctx)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	watcher, err := localClient.WatchIPNBus(watchCtx, ipn.NotifyInitialState)
+	if err != nil {
+		return err
+	}
+	defer watcher.Close()
 
 	go func() {
 		var printed bool // whether we've yet printed anything to stdout or stderr
+		var lastURLPrinted string
+
 		for {
 			n, err := watcher.Next()
 			if err != nil {
-				pumpErr <- err
+				watchErr <- err
 				return
 			}
 			if n.ErrMessage != nil {
@@ -525,8 +589,6 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 			}
 			if s := n.State; s != nil {
 				switch *s {
-				case ipn.NeedsLogin:
-					startLoginInteractive()
 				case ipn.NeedsMachineAuth:
 					printed = true
 					if env.upArgs.json {
@@ -549,12 +611,17 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 					cancelWatch()
 				}
 			}
-			if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
+			if url := n.BrowseToURL; url != nil {
+				authURL := *url
+				if !printAuthURL(authURL) || authURL == lastURLPrinted {
+					continue
+				}
 				printed = true
+				lastURLPrinted = authURL
 				if upArgs.json {
-					js := &upOutputJSON{AuthURL: *url, BackendState: st.BackendState}
+					js := &upOutputJSON{AuthURL: authURL, BackendState: st.BackendState}
 
-					q, err := qrcode.New(*url, qrcode.Medium)
+					q, err := qrcode.New(authURL, qrcode.Medium)
 					if err == nil {
 						png, err := q.PNG(128)
 						if err == nil {
@@ -569,9 +636,9 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 						outln(string(data))
 					}
 				} else {
-					fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+					fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", authURL)
 					if upArgs.qr {
-						q, err := qrcode.New(*url, qrcode.Medium)
+						q, err := qrcode.New(authURL, qrcode.Medium)
 						if err != nil {
 							log.Printf("QR code error: %v", err)
 						} else {
@@ -583,47 +650,6 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 		}
 	}()
 
-	// Special case: bare "tailscale up" means to just start
-	// running, if there's ever been a login.
-	if simpleUp {
-		localAPIMu.Lock()
-		_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
-			Prefs: ipn.Prefs{
-				WantRunning: true,
-			},
-			WantRunningSet: true,
-		})
-		localAPIMu.Unlock()
-		if err != nil {
-			return err
-		}
-	} else {
-		if err := localClient.CheckPrefs(ctx, prefs); err != nil {
-			return err
-		}
-
-		authKey, err := upArgs.getAuthKey()
-		if err != nil {
-			return err
-		}
-		authKey, err = resolveAuthKey(ctx, authKey, upArgs.advertiseTags)
-		if err != nil {
-			return err
-		}
-		localAPIMu.Lock()
-		err = localClient.Start(ctx, ipn.Options{
-			AuthKey:     authKey,
-			UpdatePrefs: prefs,
-		})
-		localAPIMu.Unlock()
-		if err != nil {
-			return err
-		}
-		if upArgs.forceReauth {
-			startLoginInteractive()
-		}
-	}
-
 	// This whole 'up' mechanism is too complicated and results in
 	// hairy stuff like this select. We're ultimately waiting for
 	// 'running' to be done, but even in the case where
@@ -647,7 +673,7 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
 		default:
 		}
 		return watchCtx.Err()
-	case err := <-pumpErr:
+	case err := <-watchErr:
 		select {
 		case <-running:
 			return nil
@@ -724,12 +750,12 @@ func init() {
 	addPrefFlagMapping("accept-dns", "CorpDNS")
 	addPrefFlagMapping("accept-routes", "RouteAll")
 	addPrefFlagMapping("advertise-tags", "AdvertiseTags")
-	addPrefFlagMapping("host-routes", "AllowSingleHosts")
 	addPrefFlagMapping("hostname", "Hostname")
 	addPrefFlagMapping("login-server", "ControlURL")
 	addPrefFlagMapping("netfilter-mode", "NetfilterMode")
 	addPrefFlagMapping("shields-up", "ShieldsUp")
 	addPrefFlagMapping("snat-subnet-routes", "NoSNAT")
+	addPrefFlagMapping("stateful-filtering", "NoStatefulFiltering")
 	addPrefFlagMapping("exit-node-allow-lan-access", "ExitNodeAllowLANAccess")
 	addPrefFlagMapping("unattended", "ForceDaemon")
 	addPrefFlagMapping("operator", "OperatorUser")
@@ -762,7 +788,7 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 // correspond to an ipn.Pref.
 func preflessFlag(flagName string) bool {
 	switch flagName {
-	case "auth-key", "force-reauth", "reset", "qr", "json", "timeout", "accept-risk":
+	case "auth-key", "force-reauth", "reset", "qr", "json", "timeout", "accept-risk", "host-routes":
 		return true
 	}
 	return false
@@ -859,11 +885,26 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck
 			// Issue 6811. Ignore on Synology.
 			continue
 		}
+		if flagName == "stateful-filtering" && valCur == true && valNew == false && env.goos == "linux" {
+			// See https://github.com/tailscale/tailscale/issues/12307
+			// Stateful filtering was on by default in tailscale 1.66.0-1.66.3, then off in 1.66.4.
+			// This broke Tailscale installations in containerized
+			// environments that use the default containerboot
+			// configuration that configures tailscale using
+			// 'tailscale up' command, which requires that all
+			// previously set flags are explicitly provided on
+			// subsequent restarts.
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
 		return nil
 	}
+
+	// Some previously provided flags are missing. This run of 'tailscale
+	// up' will error out.
+
 	sort.Strings(missing)
 
 	// Compute the stringification of the explicitly provided args in flagSet
@@ -914,7 +955,7 @@ func applyImplicitPrefs(prefs, oldPrefs *ipn.Prefs, env upCheckEnv) {
 
 func flagAppliesToOS(flag, goos string) bool {
 	switch flag {
-	case "netfilter-mode", "snat-subnet-routes":
+	case "netfilter-mode", "snat-subnet-routes", "stateful-filtering":
 		return goos == "linux"
 	case "unattended":
 		return goos == "windows"
@@ -958,8 +999,6 @@ func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]any) {
 			set(prefs.ControlURL)
 		case "accept-routes":
 			set(prefs.RouteAll)
-		case "host-routes":
-			set(prefs.AllowSingleHosts)
 		case "accept-dns":
 			set(prefs.CorpDNS)
 		case "shields-up":
@@ -989,6 +1028,16 @@ func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]any) {
 			set(prefs.AppConnector.Advertise)
 		case "snat-subnet-routes":
 			set(!prefs.NoSNAT)
+		case "stateful-filtering":
+			// We only set the stateful-filtering flag to false if
+			// the pref (negated!) is explicitly set to true; unset
+			// or false is treated as enabled.
+			val, ok := prefs.NoStatefulFiltering.Get()
+			if ok && val {
+				set(false)
+			} else {
+				set(true)
+			}
 		case "netfilter-mode":
 			set(prefs.NetfilterMode.String())
 		case "unattended":

cmd/tailscale/depaware.txt

@@ -6,11 +6,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
-   L    github.com/coreos/go-systemd/v22/dbus                        from tailscale.com/clientupdate
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/pe+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-   L 💣 github.com/godbus/dbus/v5                                    from github.com/coreos/go-systemd/v22/dbus
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
    L    github.com/google/nftables                                   from tailscale.com/util/linuxfw
    L 💣 github.com/google/nftables/alignedbuff                       from github.com/google/nftables/xt
@@ -23,7 +21,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         github.com/gorilla/securecookie                              from github.com/gorilla/csrf
         github.com/hdevalence/ed25519consensus                       from tailscale.com/clientupdate/distsign+
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
-   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/interfaces+
+   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
      💣 github.com/mattn/go-colorable                                from tailscale.com/cmd/tailscale/cli
@@ -60,7 +58,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/netipx                                               from tailscale.com/net/tsaddr+
-   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
+   W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         k8s.io/client-go/util/homedir                                from tailscale.com/cmd/tailscale/cli
         nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
         nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
@@ -88,7 +86,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/drive                                          from tailscale.com/client/tailscale+
         tailscale.com/envknob                                        from tailscale.com/client/tailscale+
-        tailscale.com/health                                         from tailscale.com/net/tlsdial
+        tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/health/healthmsg                               from tailscale.com/cmd/tailscale/cli
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale+
@@ -99,12 +97,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlhttp+
         tailscale.com/net/dnsfallback                                from tailscale.com/control/controlhttp
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
-     💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netcheck                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/net/neterror                                   from tailscale.com/net/netcheck+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
-        tailscale.com/net/netmon                                     from tailscale.com/cmd/tailscale/cli+
+     💣 tailscale.com/net/netmon                                     from tailscale.com/cmd/tailscale/cli+
         tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
         tailscale.com/net/packet                                     from tailscale.com/wgengine/capture+
@@ -123,7 +120,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale+
         tailscale.com/tempfork/spf13/cobra                           from tailscale.com/cmd/tailscale/cli/ffcomplete+
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon
         tailscale.com/tstime                                         from tailscale.com/control/controlhttp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
@@ -142,6 +139,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/tkatype                                  from tailscale.com/types/key+
         tailscale.com/types/views                                    from tailscale.com/tailcfg+
+        tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck+
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dnscache+
         tailscale.com/util/cmpver                                    from tailscale.com/net/tshttpproxy+
@@ -271,7 +269,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         image/png                                                    from github.com/skip2/go-qrcode
         io                                                           from archive/tar+
         io/fs                                                        from archive/tar+
-        io/ioutil                                                    from github.com/godbus/dbus/v5+
+        io/ioutil                                                    from github.com/mitchellh/go-ps+
         log                                                          from expvar+
         log/internal                                                 from log
         maps                                                         from tailscale.com/clientupdate+
@@ -301,7 +299,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
         runtime/debug                                                from nhooyr.io/websocket/internal/xsync+
-        runtime/trace                                                from testing
         slices                                                       from tailscale.com/client/web+
         sort                                                         from archive/tar+
         strconv                                                      from archive/tar+
@@ -309,7 +306,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         sync                                                         from archive/tar+
         sync/atomic                                                  from context+
         syscall                                                      from archive/tar+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from github.com/peterbourgon/ff/v3/ffcli+
         text/template                                                from html/template
         text/template/parse                                          from html/template+

cmd/tailscale/tailscale_test.go

@@ -12,6 +12,7 @@ import (
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
+			"testing":                            "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/buffer":       "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",

cmd/tailscaled/debug.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	"tailscale.com/derp/derphttp"
+	"tailscale.com/health"
 	"tailscale.com/ipn"
-	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
@@ -72,7 +72,7 @@ func debugMode(args []string) error {
 }
 
 func runMonitor(ctx context.Context, loop bool) error {
-	dump := func(st *interfaces.State) {
+	dump := func(st *netmon.State) {
 		j, _ := json.MarshalIndent(st, "", "    ")
 		os.Stderr.Write(j)
 	}
@@ -157,6 +157,7 @@ func getURL(ctx context.Context, urlStr string) error {
 }
 
 func checkDerp(ctx context.Context, derpRegion string) (err error) {
+	ht := new(health.Tracker)
 	req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil)
 	if err != nil {
 		return fmt.Errorf("create derp map request: %w", err)
@@ -195,6 +196,8 @@ func checkDerp(ctx context.Context, derpRegion string) (err error) {
 
 	c1 := derphttp.NewRegionClient(priv1, log.Printf, nil, getRegion)
 	c2 := derphttp.NewRegionClient(priv2, log.Printf, nil, getRegion)
+	c1.HealthTracker = ht
+	c2.HealthTracker = ht
 	defer func() {
 		if err != nil {
 			c1.Close()

cmd/tailscaled/depaware.txt

@@ -80,7 +80,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
         github.com/bits-and-blooms/bitset                            from github.com/gaissmai/bart
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
-   L    github.com/coreos/go-systemd/v22/dbus                        from tailscale.com/clientupdate
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
    W 💣 github.com/dblohm7/wingoes/com                               from tailscale.com/cmd/tailscaled+
@@ -90,7 +89,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LW 💣 github.com/digitalocean/go-smbios/smbios                     from tailscale.com/posture
      💣 github.com/djherbis/times                                    from tailscale.com/drive/driveimpl
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
-        github.com/gaissmai/bart                                     from tailscale.com/net/tstun
+        github.com/gaissmai/bart                                     from tailscale.com/net/tstun+
         github.com/go-json-experiment/json/internal                  from github.com/go-json-experiment/json/internal/jsonflags+
         github.com/go-json-experiment/json/internal/jsonflags        from github.com/go-json-experiment/json/internal/jsonopts+
         github.com/go-json-experiment/json/internal/jsonopts         from github.com/go-json-experiment/json/jsontext
@@ -98,7 +97,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/go-json-experiment/json/jsontext                  from tailscale.com/logtail
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
-   L 💣 github.com/godbus/dbus/v5                                    from github.com/coreos/go-systemd/v22/dbus+
+   L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns+
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header+
    L    github.com/google/nftables                                   from tailscale.com/util/linuxfw
@@ -119,7 +118,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/jellydator/ttlcache/v3                            from tailscale.com/drive/driveimpl/compositedav
    L    github.com/jmespath/go-jmespath                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
-   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/interfaces+
+   L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/netmon
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/klauspost/compress                                from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
@@ -145,6 +144,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/pierrec/lz4/v4/internal/xxh32                     from github.com/pierrec/lz4/v4/internal/lz4stream
   LD    github.com/pkg/sftp                                          from tailscale.com/ssh/tailssh
   LD    github.com/pkg/sftp/internal/encoding/ssh/filexfer           from github.com/pkg/sftp
+   D    github.com/prometheus-community/pro-bing                     from tailscale.com/wgengine/netstack
    L 💣 github.com/safchain/ethtool                                  from tailscale.com/net/netkernelconf+
    W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
    W 💣 github.com/tailscale/go-winio                                from tailscale.com/safesocket
@@ -295,13 +295,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
         tailscale.com/net/dnsfallback                                from tailscale.com/cmd/tailscaled+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
-     💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock+
         tailscale.com/net/neterror                                   from tailscale.com/net/dns/resolver+
         tailscale.com/net/netkernelconf                              from tailscale.com/ipn/ipnlocal
         tailscale.com/net/netknob                                    from tailscale.com/logpolicy+
-        tailscale.com/net/netmon                                     from tailscale.com/cmd/tailscaled+
+     💣 tailscale.com/net/netmon                                     from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netns                                      from tailscale.com/cmd/tailscaled+
    W 💣 tailscale.com/net/netstat                                    from tailscale.com/portlist
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
@@ -321,6 +320,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
         tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
+        tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
@@ -333,7 +333,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
         tailscale.com/tka                                            from tailscale.com/client/tailscale+
-   W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+   W    tailscale.com/tsconst                                        from tailscale.com/net/netmon
         tailscale.com/tsd                                            from tailscale.com/cmd/tailscaled+
         tailscale.com/tstime                                         from tailscale.com/control/controlclient+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
@@ -358,6 +358,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/tkatype                                  from tailscale.com/tka+
         tailscale.com/types/views                                    from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/cibuild                                   from tailscale.com/health
         tailscale.com/util/clientmetric                              from tailscale.com/control/controlclient+
         tailscale.com/util/cloudenv                                  from tailscale.com/net/dns/resolver+
         tailscale.com/util/cmpver                                    from tailscale.com/net/dns+
@@ -440,7 +441,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/http2                                       from golang.org/x/net/http2/h2c+
         golang.org/x/net/http2/h2c                                   from tailscale.com/ipn/ipnlocal
         golang.org/x/net/http2/hpack                                 from golang.org/x/net/http2+
-        golang.org/x/net/icmp                                        from tailscale.com/net/ping
+        golang.org/x/net/icmp                                        from tailscale.com/net/ping+
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/ipv4                                        from github.com/miekg/dns+
         golang.org/x/net/ipv6                                        from github.com/miekg/dns+
@@ -553,7 +554,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+
         runtime/pprof                                                from net/http/pprof+
-        runtime/trace                                                from net/http/pprof+
+        runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/appc+
         sort                                                         from archive/tar+
         strconv                                                      from archive/tar+
@@ -561,7 +562,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         sync                                                         from archive/tar+
         sync/atomic                                                  from context+
         syscall                                                      from archive/tar+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from runtime/pprof
         text/template                                                from html/template
         text/template/parse                                          from html/template+

cmd/tailscaled/tailscaled.go

@@ -35,6 +35,7 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive/driveimpl"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/conffile"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/ipnserver"
@@ -117,7 +118,7 @@ var args struct {
 	tunname string
 
 	cleanUp        bool
-	confFile       string
+	confFile       string // empty, file path, or "vm:user-data"
 	debug          string
 	port           uint16
 	statepath      string
@@ -168,7 +169,7 @@ func main() {
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
 	flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support")
-	flag.StringVar(&args.confFile, "config", "", "path to config file")
+	flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2)")
 
 	if len(os.Args) > 0 && filepath.Base(os.Args[0]) == "tailscale" && beCLI != nil {
 		beCLI()
@@ -358,7 +359,7 @@ func run() (err error) {
 		sys.Set(netMon)
 	}
 
-	pol := logpolicy.New(logtail.CollectionNode, netMon, nil /* use log.Printf */)
+	pol := logpolicy.New(logtail.CollectionNode, netMon, sys.HealthTracker(), nil /* use log.Printf */)
 	pol.SetVerbosityLevel(args.verbose)
 	logPol = pol
 	defer func() {
@@ -393,8 +394,8 @@ func run() (err error) {
 	// Always clean up, even if we're going to run the server. This covers cases
 	// such as when a system was rebooted without shutting down, or tailscaled
 	// crashed, and would for example restore system DNS configuration.
-	dns.CleanUp(logf, args.tunname)
-	router.CleanUp(logf, args.tunname)
+	dns.CleanUp(logf, netMon, args.tunname)
+	router.CleanUp(logf, netMon, args.tunname)
 	// If the cleanUp flag was passed, then exit.
 	if args.cleanUp {
 		return nil
@@ -480,6 +481,15 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID,
 		lb, err := getLocalBackend(ctx, logf, logID, sys)
 		if err == nil {
 			logf("got LocalBackend in %v", time.Since(t0).Round(time.Millisecond))
+			if lb.Prefs().Valid() {
+				if err := lb.Start(ipn.Options{}); err != nil {
+					logf("LocalBackend.Start: %v", err)
+					lb.Shutdown()
+					lbErr.Store(err)
+					cancel()
+					return
+				}
+			}
 			srv.SetLocalBackend(lb)
 			close(wgEngineCreated)
 			return
@@ -538,14 +548,25 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 			return ok
 		}
 		dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
-			// Note: don't just return ns.DialContextTCP or we'll
-			// return an interface containing a nil pointer.
+			// Note: don't just return ns.DialContextTCP or we'll return
+			// *gonet.TCPConn(nil) instead of a nil interface which trips up
+			// callers.
 			tcpConn, err := ns.DialContextTCP(ctx, dst)
 			if err != nil {
 				return nil, err
 			}
 			return tcpConn, nil
 		}
+		dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+			// Note: don't just return ns.DialContextUDP or we'll return
+			// *gonet.UDPConn(nil) instead of a nil interface which trips up
+			// callers.
+			udpConn, err := ns.DialContextUDP(ctx, dst)
+			if err != nil {
+				return nil, err
+			}
+			return udpConn, nil
+		}
 	}
 	if socksListener != nil || httpProxyListener != nil {
 		var addrs []string
@@ -651,6 +672,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 	conf := wgengine.Config{
 		ListenPort:    args.port,
 		NetMon:        sys.NetMon.Get(),
+		HealthTracker: sys.HealthTracker(),
 		Dialer:        sys.Dialer.Get(),
 		SetSubsystem:  sys.Set,
 		ControlKnobs:  sys.ControlKnobs(),
@@ -676,7 +698,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 			// configuration being unavailable (from the noop
 			// manager). More in Issue 4017.
 			// TODO(bradfitz): add a Synology-specific DNS manager.
-			conf.DNS, err = dns.NewOSConfigurator(logf, "") // empty interface name
+			conf.DNS, err = dns.NewOSConfigurator(logf, sys.HealthTracker(), "") // empty interface name
 			if err != nil {
 				return false, fmt.Errorf("dns.NewOSConfigurator: %w", err)
 			}
@@ -698,13 +720,13 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 			return false, err
 		}
 
-		r, err := router.New(logf, dev, sys.NetMon.Get())
+		r, err := router.New(logf, dev, sys.NetMon.Get(), sys.HealthTracker())
 		if err != nil {
 			dev.Close()
 			return false, fmt.Errorf("creating router: %w", err)
 		}
 
-		d, err := dns.NewOSConfigurator(logf, devName)
+		d, err := dns.NewOSConfigurator(logf, sys.HealthTracker(), devName)
 		if err != nil {
 			dev.Close()
 			r.Close()

cmd/tailscaled/tailscaled_test.go

@@ -20,6 +20,7 @@ func TestDeps(t *testing.T) {
 		GOOS:   "darwin",
 		GOARCH: "arm64",
 		BadDeps: map[string]string{
+			"testing":                        "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658",
 		},
 	}.Check(t)
@@ -28,6 +29,7 @@ func TestDeps(t *testing.T) {
 		GOOS:   "linux",
 		GOARCH: "arm64",
 		BadDeps: map[string]string{
+			"testing":                        "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658",
 		},
 	}.Check(t)

cmd/tsconnect/wasm/wasm_js.go

@@ -104,9 +104,10 @@ func newIPN(jsConfig js.Value) map[string]any {
 	sys.Set(store)
 	dialer := &tsdial.Dialer{Logf: logf}
 	eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
-		Dialer:       dialer,
-		SetSubsystem: sys.Set,
-		ControlKnobs: sys.ControlKnobs(),
+		Dialer:        dialer,
+		SetSubsystem:  sys.Set,
+		ControlKnobs:  sys.ControlKnobs(),
+		HealthTracker: sys.HealthTracker(),
 	})
 	if err != nil {
 		log.Fatal(err)
@@ -297,11 +298,10 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 	go func() {
 		err := i.lb.Start(ipn.Options{
 			UpdatePrefs: &ipn.Prefs{
-				ControlURL:       i.controlURL,
-				RouteAll:         false,
-				AllowSingleHosts: true,
-				WantRunning:      true,
-				Hostname:         i.hostname,
+				ControlURL:  i.controlURL,
+				RouteAll:    false,
+				WantRunning: true,
+				Hostname:    i.hostname,
 			},
 			AuthKey: i.authKey,
 		})

cmd/tsidp/tsidp.go

@@ -40,7 +40,6 @@ import (
 	"tailscale.com/tsnet"
 	"tailscale.com/types/key"
 	"tailscale.com/types/lazy"
-	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
@@ -95,8 +94,8 @@ func main() {
 		ts := &tsnet.Server{
 			Hostname: "idp",
 		}
-		if !*flagVerbose {
-			ts.Logf = logger.Discard
+		if *flagVerbose {
+			ts.Logf = log.Printf
 		}
 		st, err = ts.Up(ctx)
 		if err != nil {

control/controlclient/auto.go

@@ -12,7 +12,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"tailscale.com/health"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/sockstats"
 	"tailscale.com/tailcfg"
@@ -27,9 +26,8 @@ import (
 
 type LoginGoal struct {
 	_     structs.Incomparable
-	token *tailcfg.Oauth2Token // oauth token to use when logging in
-	flags LoginFlags           // flags to use when logging in
-	url   string               // auth url that needs to be visited
+	flags LoginFlags // flags to use when logging in
+	url   string     // auth url that needs to be visited
 }
 
 var _ Client = (*Auto)(nil)
@@ -195,7 +193,7 @@ func NewNoStart(opts Options) (_ *Auto, err error) {
 	c.mapCtx, c.mapCancel = context.WithCancel(context.Background())
 	c.mapCtx = sockstats.WithSockStats(c.mapCtx, sockstats.LabelControlClientAuto, opts.Logf)
 
-	c.unregisterHealthWatch = health.RegisterWatcher(direct.ReportHealthChange)
+	c.unregisterHealthWatch = opts.HealthTracker.RegisterWatcher(direct.ReportHealthChange)
 	return c, nil
 
 }
@@ -316,7 +314,7 @@ func (c *Auto) authRoutine() {
 		}
 
 		if goal == nil {
-			health.SetAuthRoutineInError(nil)
+			c.direct.health.SetAuthRoutineInError(nil)
 			// Wait for user to Login or Logout.
 			<-ctx.Done()
 			c.logf("[v1] authRoutine: context done.")
@@ -339,19 +337,24 @@ func (c *Auto) authRoutine() {
 			url, err = c.direct.WaitLoginURL(ctx, goal.url)
 			f = "WaitLoginURL"
 		} else {
-			url, err = c.direct.TryLogin(ctx, goal.token, goal.flags)
+			url, err = c.direct.TryLogin(ctx, goal.flags)
 			f = "TryLogin"
 		}
 		if err != nil {
-			health.SetAuthRoutineInError(err)
+			c.direct.health.SetAuthRoutineInError(err)
 			report(err, f)
 			bo.BackOff(ctx, err)
 			continue
 		}
 		if url != "" {
-			// goal.url ought to be empty here.
-			// However, not all control servers get this right,
-			// and logging about it here just generates noise.
+			// goal.url ought to be empty here. However, not all control servers
+			// get this right, and logging about it here just generates noise.
+			//
+			// TODO(bradfitz): I don't follow that comment. Our own testcontrol
+			// used by tstest/integration hits this path, in fact.
+			if c.direct.panicOnUse {
+				panic("tainted client")
+			}
 			c.mu.Lock()
 			c.urlToVisit = url
 			c.loginGoal = &LoginGoal{
@@ -373,7 +376,7 @@ func (c *Auto) authRoutine() {
 		}
 
 		// success
-		health.SetAuthRoutineInError(nil)
+		c.direct.health.SetAuthRoutineInError(nil)
 		c.mu.Lock()
 		c.urlToVisit = ""
 		c.loggedIn = true
@@ -503,11 +506,11 @@ func (c *Auto) mapRoutine() {
 			c.logf("[v1] mapRoutine: context done.")
 			continue
 		}
-		health.SetOutOfPollNetMap()
+		c.direct.health.SetOutOfPollNetMap()
 
 		err := c.direct.PollNetMap(ctx, mrs)
 
-		health.SetOutOfPollNetMap()
+		c.direct.health.SetOutOfPollNetMap()
 		c.mu.Lock()
 		c.inMapPoll = false
 		if c.state == StateSynchronized {
@@ -608,17 +611,19 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM
 	})
 }
 
-func (c *Auto) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
-	c.logf("client.Login(%v, %v)", t != nil, flags)
+func (c *Auto) Login(flags LoginFlags) {
+	c.logf("client.Login(%v)", flags)
 
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.closed {
 		return
 	}
+	if c.direct != nil && c.direct.panicOnUse {
+		panic("tainted client")
+	}
 	c.wantLoggedIn = true
 	c.loginGoal = &LoginGoal{
-		token: t,
 		flags: flags,
 	}
 	c.cancelMapCtxLocked()
@@ -633,6 +638,9 @@ func (c *Auto) Logout(ctx context.Context) error {
 	c.wantLoggedIn = false
 	c.loginGoal = nil
 	closed := c.closed
+	if c.direct != nil && c.direct.panicOnUse {
+		panic("tainted client")
+	}
 	c.mu.Unlock()
 
 	if closed {
@@ -669,37 +677,35 @@ func (c *Auto) UpdateEndpoints(endpoints []tailcfg.Endpoint) {
 }
 
 func (c *Auto) Shutdown() {
-	c.logf("client.Shutdown()")
-
 	c.mu.Lock()
-	closed := c.closed
-	direct := c.direct
-	if !closed {
-		c.closed = true
-		c.observerQueue.Shutdown()
-		c.cancelAuthCtxLocked()
-		c.cancelMapCtxLocked()
-		for _, w := range c.unpauseWaiters {
-			w <- false
-		}
-		c.unpauseWaiters = nil
+	if c.closed {
+		c.mu.Unlock()
+		return
 	}
+	c.logf("client.Shutdown ...")
+
+	direct := c.direct
+	c.closed = true
+	c.observerQueue.Shutdown()
+	c.cancelAuthCtxLocked()
+	c.cancelMapCtxLocked()
+	for _, w := range c.unpauseWaiters {
+		w <- false
+	}
+	c.unpauseWaiters = nil
 	c.mu.Unlock()
 
-	c.logf("client.Shutdown")
-	if !closed {
-		c.unregisterHealthWatch()
-		<-c.authDone
-		<-c.mapDone
-		<-c.updateDone
-		if direct != nil {
-			direct.Close()
-		}
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancel()
-		c.observerQueue.Wait(ctx)
-		c.logf("Client.Shutdown done.")
+	c.unregisterHealthWatch()
+	<-c.authDone
+	<-c.mapDone
+	<-c.updateDone
+	if direct != nil {
+		direct.Close()
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	c.observerQueue.Wait(ctx)
+	c.logf("Client.Shutdown done.")
 }
 
 // NodePublicKey returns the node public key currently in use. This is

control/controlclient/client.go

@@ -43,8 +43,9 @@ type Client interface {
 	// Login begins an interactive or non-interactive login process.
 	// Client will eventually call the Status callback with either a
 	// LoginFinished flag (on success) or an auth URL (if further
-	// interaction is needed).
-	Login(*tailcfg.Oauth2Token, LoginFlags)
+	// interaction is needed). It merely sets the process in motion,
+	// and doesn't wait for it to complete.
+	Login(LoginFlags)
 	// Logout starts a synchronous logout process. It doesn't return
 	// until the logout operation has been completed.
 	Logout(context.Context) error

control/controlclient/direct.go

@@ -36,7 +36,6 @@ import (
 	"tailscale.com/logtail"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
-	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tlsdial"
@@ -56,6 +55,7 @@ import (
 	"tailscale.com/util/singleflight"
 	"tailscale.com/util/syspolicy"
 	"tailscale.com/util/systemd"
+	"tailscale.com/util/testenv"
 	"tailscale.com/util/zstdframe"
 )
 
@@ -68,7 +68,8 @@ type Direct struct {
 	serverURL                  string              // URL of the tailcontrol server
 	clock                      tstime.Clock
 	logf                       logger.Logf
-	netMon                     *netmon.Monitor // or nil
+	netMon                     *netmon.Monitor // non-nil
+	health                     *health.Tracker
 	discoPubKey                key.DiscoPublic
 	getMachinePrivKey          func() (key.MachinePrivate, error)
 	debugFlags                 []string
@@ -79,6 +80,7 @@ type Direct struct {
 	onClientVersion            func(*tailcfg.ClientVersion) // or nil
 	onControlTime              func(time.Time)              // or nil
 	onTailnetDefaultAutoUpdate func(bool)                   // or nil
+	panicOnUse                 bool                         // if true, panic if client is used (for testing)
 
 	dialPlan ControlDialPlanner // can be nil
 
@@ -119,10 +121,10 @@ type Options struct {
 	Hostinfo                   *tailcfg.Hostinfo // non-nil passes ownership, nil means to use default using os.Hostname, etc
 	DiscoPublicKey             key.DiscoPublic
 	Logf                       logger.Logf
-	HTTPTestClient             *http.Client                 // optional HTTP client to use (for tests only)
-	NoiseTestClient            *http.Client                 // optional HTTP client to use for noise RPCs (tests only)
-	DebugFlags                 []string                     // debug settings to send to control
-	NetMon                     *netmon.Monitor              // optional network monitor
+	HTTPTestClient             *http.Client // optional HTTP client to use (for tests only)
+	NoiseTestClient            *http.Client // optional HTTP client to use for noise RPCs (tests only)
+	DebugFlags                 []string     // debug settings to send to control
+	HealthTracker              *health.Tracker
 	PopBrowserURL              func(url string)             // optional func to open browser
 	OnClientVersion            func(*tailcfg.ClientVersion) // optional func to inform GUI of client version status
 	OnControlTime              func(time.Time)              // optional func to notify callers of new time from control
@@ -211,6 +213,19 @@ func NewDirect(opts Options) (*Direct, error) {
 	if opts.GetMachinePrivateKey == nil {
 		return nil, errors.New("controlclient.New: no GetMachinePrivateKey specified")
 	}
+	if opts.Dialer == nil {
+		if testenv.InTest() {
+			panic("no Dialer")
+		}
+		return nil, errors.New("controlclient.New: no Dialer specified")
+	}
+	netMon := opts.Dialer.NetMon()
+	if netMon == nil {
+		if testenv.InTest() {
+			panic("no NetMon in Dialer")
+		}
+		return nil, errors.New("controlclient.New: Dialer has nil NetMon")
+	}
 	if opts.ControlKnobs == nil {
 		opts.ControlKnobs = &controlknobs.Knobs{}
 	}
@@ -231,9 +246,8 @@ func NewDirect(opts Options) (*Direct, error) {
 	dnsCache := &dnscache.Resolver{
 		Forward:          dnscache.Get().Forward, // use default cache's forwarder
 		UseLastGood:      true,
-		LookupIPFallback: dnsfallback.MakeLookupFunc(opts.Logf, opts.NetMon),
+		LookupIPFallback: dnsfallback.MakeLookupFunc(opts.Logf, netMon),
 		Logf:             opts.Logf,
-		NetMon:           opts.NetMon,
 	}
 
 	httpc := opts.HTTPTestClient
@@ -248,7 +262,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
-		tr.TLSClientConfig = tlsdial.Config(serverURL.Hostname(), tr.TLSClientConfig)
+		tr.TLSClientConfig = tlsdial.Config(serverURL.Hostname(), opts.HealthTracker, tr.TLSClientConfig)
 		tr.DialContext = dnscache.Dialer(opts.Dialer.SystemDial, dnsCache)
 		tr.DialTLSContext = dnscache.TLSDialer(opts.Dialer.SystemDial, dnsCache, tr.TLSClientConfig)
 		tr.ForceAttemptHTTP2 = true
@@ -270,7 +284,8 @@ func NewDirect(opts Options) (*Direct, error) {
 		authKey:                    opts.AuthKey,
 		discoPubKey:                opts.DiscoPublicKey,
 		debugFlags:                 opts.DebugFlags,
-		netMon:                     opts.NetMon,
+		netMon:                     netMon,
+		health:                     opts.HealthTracker,
 		skipIPForwardingCheck:      opts.SkipIPForwardingCheck,
 		pinger:                     opts.Pinger,
 		popBrowser:                 opts.PopBrowserURL,
@@ -296,6 +311,9 @@ func NewDirect(opts Options) (*Direct, error) {
 		}
 		c.serverNoiseKey = key.NewMachine().Public() // prevent early error before hitting test client
 	}
+	if strings.Contains(opts.ServerURL, "controlplane.tailscale.com") && envknob.Bool("TS_PANIC_IF_HIT_MAIN_CONTROL") {
+		c.panicOnUse = true
+	}
 	return c, nil
 }
 
@@ -383,9 +401,12 @@ func (c *Direct) TryLogout(ctx context.Context) error {
 	return err
 }
 
-func (c *Direct) TryLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags) (url string, err error) {
-	c.logf("[v1] direct.TryLogin(token=%v, flags=%v)", t != nil, flags)
-	return c.doLoginOrRegen(ctx, loginOpt{Token: t, Flags: flags})
+func (c *Direct) TryLogin(ctx context.Context, flags LoginFlags) (url string, err error) {
+	if strings.Contains(c.serverURL, "controlplane.tailscale.com") && envknob.Bool("TS_PANIC_IF_HIT_MAIN_CONTROL") {
+		panic(fmt.Sprintf("[unexpected] controlclient: TryLogin called on %s; tainted=%v", c.serverURL, c.panicOnUse))
+	}
+	c.logf("[v1] direct.TryLogin(flags=%v)", flags)
+	return c.doLoginOrRegen(ctx, loginOpt{Flags: flags})
 }
 
 // WaitLoginURL sits in a long poll waiting for the user to authenticate at url.
@@ -420,7 +441,6 @@ func (c *Direct) SetExpirySooner(ctx context.Context, expiry time.Time) error {
 }
 
 type loginOpt struct {
-	Token  *tailcfg.Oauth2Token
 	Flags  LoginFlags
 	Regen  bool // generate a new nodekey, can be overridden in doLogin
 	URL    string
@@ -445,6 +465,9 @@ func (c *Direct) hostInfoLocked() *tailcfg.Hostinfo {
 }
 
 func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, newURL string, nks tkatype.MarshaledSignature, err error) {
+	if c.panicOnUse {
+		panic("tainted client")
+	}
 	c.mu.Lock()
 	persist := c.persist.AsStruct()
 	tryingNewKey := c.tryingNewKey
@@ -535,7 +558,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 
 	var nodeKeySignature tkatype.MarshaledSignature
 	if !oldNodeKey.IsZero() && opt.OldNodeKeySignature != nil {
-		if nodeKeySignature, err = resignNKS(persist.NetworkLockKey, tryingNewKey.Public(), opt.OldNodeKeySignature); err != nil {
+		if nodeKeySignature, err = tka.ResignNKS(persist.NetworkLockKey, tryingNewKey.Public(), opt.OldNodeKeySignature); err != nil {
 			c.logf("Failed re-signing node-key signature: %v", err)
 		}
 	} else if isWrapped {
@@ -586,10 +609,9 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	c.logf("RegisterReq: onode=%v node=%v fup=%v nks=%v",
 		request.OldNodeKey.ShortString(),
 		request.NodeKey.ShortString(), opt.URL != "", len(nodeKeySignature) > 0)
-	if opt.Token != nil || authKey != "" {
+	if authKey != "" {
 		request.Auth = &tailcfg.RegisterResponseAuth{
-			Oauth2Token: opt.Token,
-			AuthKey:     authKey,
+			AuthKey: authKey,
 		}
 	}
 	err = signRegisterRequest(&request, c.serverURL, c.serverLegacyKey, machinePrivKey.Public())
@@ -707,45 +729,6 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	return false, resp.AuthURL, nil, nil
 }
 
-// resignNKS re-signs a node-key signature for a new node-key.
-//
-// This only matters on network-locked tailnets, because node-key signatures are
-// how other nodes know that a node-key is authentic. When the node-key is
-// rotated then the existing signature becomes invalid, so this function is
-// responsible for generating a new wrapping signature to certify the new node-key.
-//
-// The signature itself is a SigRotation signature, which embeds the old signature
-// and certifies the new node-key as a replacement for the old by signing the new
-// signature with RotationPubkey (which is the node's own network-lock key).
-func resignNKS(priv key.NLPrivate, nodeKey key.NodePublic, oldNKS tkatype.MarshaledSignature) (tkatype.MarshaledSignature, error) {
-	var oldSig tka.NodeKeySignature
-	if err := oldSig.Unserialize(oldNKS); err != nil {
-		return nil, fmt.Errorf("decoding NKS: %w", err)
-	}
-
-	nk, err := nodeKey.MarshalBinary()
-	if err != nil {
-		return nil, fmt.Errorf("marshalling node-key: %w", err)
-	}
-
-	if bytes.Equal(nk, oldSig.Pubkey) {
-		// The old signature is valid for the node-key we are using, so just
-		// use it verbatim.
-		return oldNKS, nil
-	}
-
-	newSig := tka.NodeKeySignature{
-		SigKind: tka.SigRotation,
-		Pubkey:  nk,
-		Nested:  &oldSig,
-	}
-	if newSig.Signature, err = priv.SignNKS(newSig.SigHash()); err != nil {
-		return nil, fmt.Errorf("signing NKS: %w", err)
-	}
-
-	return newSig.Serialize(), nil
-}
-
 // newEndpoints acquires c.mu and sets the local port and endpoints and reports
 // whether they've changed.
 //
@@ -818,6 +801,9 @@ const watchdogTimeout = 120 * time.Second
 //
 // If nu is nil, OmitPeers will be set to true.
 func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu NetmapUpdater) error {
+	if c.panicOnUse {
+		panic("tainted client")
+	}
 	if isStreaming && nu == nil {
 		panic("cb must be non-nil if isStreaming is true")
 	}
@@ -894,10 +880,10 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 		ipForwardingBroken(hi.RoutableIPs, c.netMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
-	if health.RouterHealth() != nil {
+	if c.health.RouterHealth() != nil {
 		extraDebugFlags = append(extraDebugFlags, "warn-router-unhealthy")
 	}
-	extraDebugFlags = health.AppendWarnableDebugFlags(extraDebugFlags)
+	extraDebugFlags = c.health.AppendWarnableDebugFlags(extraDebugFlags)
 	if hostinfo.DisabledEtcAptSource() {
 		extraDebugFlags = append(extraDebugFlags, "warn-etc-apt-source-disabled")
 	}
@@ -970,7 +956,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 	}
 	defer res.Body.Close()
 
-	health.NoteMapRequestHeard(request)
+	c.health.NoteMapRequestHeard(request)
 	watchdogTimer.Reset(watchdogTimeout)
 
 	if nu == nil {
@@ -1041,7 +1027,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 		metricMapResponseMessages.Add(1)
 
 		if isStreaming {
-			health.GotStreamedMapResponse()
+			c.health.GotStreamedMapResponse()
 		}
 
 		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
@@ -1269,7 +1255,7 @@ var clock tstime.Clock = tstime.StdClock{}
 //
 // TODO(bradfitz): Change controlclient.Options.SkipIPForwardingCheck into a
 // func([]netip.Prefix) error signature instead.
-func ipForwardingBroken(routes []netip.Prefix, state *interfaces.State) bool {
+func ipForwardingBroken(routes []netip.Prefix, state *netmon.State) bool {
 	warn, err := netutil.CheckIPForwarding(routes, state)
 	if err != nil {
 		// Oh well, we tried. This is just for debugging.
@@ -1450,14 +1436,15 @@ func (c *Direct) getNoiseClient() (*NoiseClient, error) {
 		}
 		c.logf("[v1] creating new noise client")
 		nc, err := NewNoiseClient(NoiseOpts{
-			PrivKey:      k,
-			ServerPubKey: serverNoiseKey,
-			ServerURL:    c.serverURL,
-			Dialer:       c.dialer,
-			DNSCache:     c.dnsCache,
-			Logf:         c.logf,
-			NetMon:       c.netMon,
-			DialPlan:     dp,
+			PrivKey:       k,
+			ServerPubKey:  serverNoiseKey,
+			ServerURL:     c.serverURL,
+			Dialer:        c.dialer,
+			DNSCache:      c.dnsCache,
+			Logf:          c.logf,
+			NetMon:        c.netMon,
+			HealthTracker: c.health,
+			DialPlan:      dp,
 		})
 		if err != nil {
 			return nil, err
@@ -1513,6 +1500,9 @@ func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) (err er
 }
 
 func (c *Direct) DoNoiseRequest(req *http.Request) (*http.Response, error) {
+	if c.panicOnUse {
+		panic("tainted client")
+	}
 	nc, err := c.getNoiseClient()
 	if err != nil {
 		return nil, err
@@ -1609,6 +1599,9 @@ func (c *Direct) ReportHealthChange(sys health.Subsystem, sysErr error) {
 	if !ok {
 		return
 	}
+	if c.panicOnUse {
+		panic("tainted client")
+	}
 	req := &tailcfg.HealthChangeRequest{
 		Subsys:  string(sys),
 		NodeKey: nodeKey,

control/controlclient/direct_test.go

@@ -14,6 +14,7 @@ import (
 
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
@@ -31,7 +32,7 @@ func TestNewDirect(t *testing.T) {
 		GetMachinePrivateKey: func() (key.MachinePrivate, error) {
 			return k, nil
 		},
-		Dialer: new(tsdial.Dialer),
+		Dialer: tsdial.NewDialer(netmon.NewStatic()),
 	}
 	c, err := NewDirect(opts)
 	if err != nil {
@@ -107,7 +108,7 @@ func TestTsmpPing(t *testing.T) {
 		GetMachinePrivateKey: func() (key.MachinePrivate, error) {
 			return k, nil
 		},
-		Dialer: new(tsdial.Dialer),
+		Dialer: tsdial.NewDialer(netmon.NewStatic()),
 	}
 
 	c, err := NewDirect(opts)

control/controlclient/map.go

@@ -736,6 +736,10 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang
 			if was.IsWireGuardOnly() != n.IsWireGuardOnly {
 				return nil, false
 			}
+		case "IsJailed":
+			if was.IsJailed() != n.IsJailed {
+				return nil, false
+			}
 		case "Expired":
 			if was.Expired() != n.Expired {
 				return nil, false

control/controlclient/noise.go

@@ -19,6 +19,7 @@ import (
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlbase"
 	"tailscale.com/control/controlhttp"
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
@@ -174,6 +175,7 @@ type NoiseClient struct {
 
 	logf   logger.Logf
 	netMon *netmon.Monitor
+	health *health.Tracker
 
 	// mu only protects the following variables.
 	mu       sync.Mutex
@@ -204,6 +206,8 @@ type NoiseOpts struct {
 	// network interface state. This field can be nil; if so, the current
 	// state will be looked up dynamically.
 	NetMon *netmon.Monitor
+	// HealthTracker, if non-nil, is the health tracker to use.
+	HealthTracker *health.Tracker
 	// DialPlan, if set, is a function that should return an explicit plan
 	// on how to connect to the server.
 	DialPlan func() *tailcfg.ControlDialPlan
@@ -247,6 +251,7 @@ func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) {
 		dialPlan:     opts.DialPlan,
 		logf:         opts.Logf,
 		netMon:       opts.NetMon,
+		health:       opts.HealthTracker,
 	}
 
 	// Create the HTTP/2 Transport using a net/http.Transport
@@ -453,6 +458,7 @@ func (nc *NoiseClient) dial(ctx context.Context) (*noiseConn, error) {
 		DialPlan:        dialPlan,
 		Logf:            nc.logf,
 		NetMon:          nc.netMon,
+		HealthTracker:   nc.health,
 		Clock:           tstime.StdClock{},
 	}).Dial(ctx)
 	if err != nil {

control/controlclient/noise_test.go

@@ -16,6 +16,7 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlhttp"
+	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
@@ -73,7 +74,7 @@ func (tt noiseClientTest) run(t *testing.T) {
 	})
 	defer hs.Close()
 
-	dialer := new(tsdial.Dialer)
+	dialer := tsdial.NewDialer(netmon.NewStatic())
 	nc, err := NewNoiseClient(NoiseOpts{
 		PrivKey:      clientPrivate,
 		ServerPubKey: serverPrivate.Public(),

control/controlhttp/client.go

@@ -393,7 +393,6 @@ func (a *Dialer) resolver() *dnscache.Resolver {
 		LookupIPFallback: dnsfallback.MakeLookupFunc(a.logf, a.NetMon),
 		UseLastGood:      true,
 		Logf:             a.Logf, // not a.logf method; we want to propagate nil-ness
-		NetMon:           a.NetMon,
 	}
 }
 
@@ -412,7 +411,6 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr,
 			SingleHostStaticResult: []netip.Addr{addr},
 			SingleHost:             u.Hostname(),
 			Logf:                   a.Logf, // not a.logf method; we want to propagate nil-ness
-			NetMon:                 a.NetMon,
 		}
 	} else {
 		dns = a.resolver()
@@ -433,7 +431,7 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr,
 	// Disable HTTP2, since h2 can't do protocol switching.
 	tr.TLSClientConfig.NextProtos = []string{}
 	tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
-	tr.TLSClientConfig = tlsdial.Config(a.Hostname, tr.TLSClientConfig)
+	tr.TLSClientConfig = tlsdial.Config(a.Hostname, a.HealthTracker, tr.TLSClientConfig)
 	if !tr.TLSClientConfig.InsecureSkipVerify {
 		panic("unexpected") // should be set by tlsdial.Config
 	}

control/controlhttp/constants.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"time"
 
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/tailcfg"
@@ -79,6 +80,9 @@ type Dialer struct {
 
 	NetMon *netmon.Monitor
 
+	// HealthTracker, if non-nil, is the health tracker to use.
+	HealthTracker *health.Tracker
+
 	// DialPlan, if set, contains instructions from the control server on
 	// how to connect to it. If present, we will try the methods in this
 	// plan before falling back to DNS.

control/controlhttp/http_test.go

@@ -22,6 +22,7 @@ import (
 
 	"tailscale.com/control/controlbase"
 	"tailscale.com/net/dnscache"
+	"tailscale.com/net/netmon"
 	"tailscale.com/net/socks5"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
@@ -199,14 +200,17 @@ func testControlHTTP(t *testing.T, param httpTestParam) {
 		defer cancel()
 	}
 
+	netMon := netmon.NewStatic()
+	dialer := tsdial.NewDialer(netMon)
 	a := &Dialer{
 		Hostname:             "localhost",
 		HTTPPort:             strconv.Itoa(httpLn.Addr().(*net.TCPAddr).Port),
 		HTTPSPort:            strconv.Itoa(httpsLn.Addr().(*net.TCPAddr).Port),
 		MachineKey:           client,
 		ControlKey:           server.Public(),
+		NetMon:               netMon,
 		ProtocolVersion:      testProtocolVersion,
-		Dialer:               new(tsdial.Dialer).SystemDial,
+		Dialer:               dialer.SystemDial,
 		Logf:                 t.Logf,
 		omitCertErrorLogging: true,
 		testFallbackDelay:    fallbackDelay,
@@ -643,7 +647,7 @@ func TestDialPlan(t *testing.T) {
 
 			dialer := closeTrackDialer{
 				t:     t,
-				inner: new(tsdial.Dialer).SystemDial,
+				inner: tsdial.NewDialer(netmon.NewStatic()).SystemDial,
 				conns: make(map[*closeTrackConn]bool),
 			}
 			defer dialer.Done()

control/controlknobs/controlknobs.go

@@ -72,6 +72,15 @@ type Knobs struct {
 	// ProbeUDPLifetime is whether the node should probe UDP path lifetime on
 	// the tail end of an active direct connection in magicsock.
 	ProbeUDPLifetime atomic.Bool
+
+	// AppCStoreRoutes is whether the node should store RouteInfo to StateStore
+	// if it's an app connector.
+	AppCStoreRoutes atomic.Bool
+
+	// UserDialUseRoutes is whether tsdial.Dialer.UserDial should use routes to determine
+	// how to dial the destination address. When true, it also makes the DNS forwarder
+	// use UserDial instead of SystemDial when dialing resolvers.
+	UserDialUseRoutes atomic.Bool
 }
 
 // UpdateFromNodeAttributes updates k (if non-nil) based on the provided self
@@ -96,6 +105,8 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 		forceNfTables                 = has(tailcfg.NodeAttrLinuxMustUseNfTables)
 		seamlessKeyRenewal            = has(tailcfg.NodeAttrSeamlessKeyRenewal)
 		probeUDPLifetime              = has(tailcfg.NodeAttrProbeUDPLifetime)
+		appCStoreRoutes               = has(tailcfg.NodeAttrStoreAppCRoutes)
+		userDialUseRoutes             = has(tailcfg.NodeAttrUserDialUseRoutes)
 	)
 
 	if has(tailcfg.NodeAttrOneCGNATEnable) {
@@ -118,6 +129,8 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 	k.LinuxForceNfTables.Store(forceNfTables)
 	k.SeamlessKeyRenewal.Store(seamlessKeyRenewal)
 	k.ProbeUDPLifetime.Store(probeUDPLifetime)
+	k.AppCStoreRoutes.Store(appCStoreRoutes)
+	k.UserDialUseRoutes.Store(userDialUseRoutes)
 }
 
 // AsDebugJSON returns k as something that can be marshalled with json.Marshal
@@ -141,5 +154,7 @@ func (k *Knobs) AsDebugJSON() map[string]any {
 		"LinuxForceNfTables":            k.LinuxForceNfTables.Load(),
 		"SeamlessKeyRenewal":            k.SeamlessKeyRenewal.Load(),
 		"ProbeUDPLifetime":              k.ProbeUDPLifetime.Load(),
+		"AppCStoreRoutes":               k.AppCStoreRoutes.Load(),
+		"UserDialUseRoutes":             k.UserDialUseRoutes.Load(),
 	}
 }

derp/derp_server.go

@@ -69,10 +69,6 @@ func init() {
 	}
 }
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
 const (
 	perClientSendQueueDepth = 32 // packets buffered for sending
 	writeTimeout            = 2 * time.Second
@@ -333,20 +329,36 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
 	s.packetsRecvOther = s.packetsRecvByKind.Get("other")
-	s.packetsDroppedReasonCounters = []*expvar.Int{
-		s.packetsDroppedReason.Get("unknown_dest"),
-		s.packetsDroppedReason.Get("unknown_dest_on_fwd"),
-		s.packetsDroppedReason.Get("gone_disconnected"),
-		s.packetsDroppedReason.Get("gone_not_here"),
-		s.packetsDroppedReason.Get("queue_head"),
-		s.packetsDroppedReason.Get("queue_tail"),
-		s.packetsDroppedReason.Get("write_error"),
-	}
+
+	s.packetsDroppedReasonCounters = s.genPacketsDroppedReasonCounters()
+
 	s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
 	s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
 	return s
 }
 
+func (s *Server) genPacketsDroppedReasonCounters() []*expvar.Int {
+	getMetric := s.packetsDroppedReason.Get
+	ret := []*expvar.Int{
+		dropReasonUnknownDest:      getMetric("unknown_dest"),
+		dropReasonUnknownDestOnFwd: getMetric("unknown_dest_on_fwd"),
+		dropReasonGoneDisconnected: getMetric("gone_disconnected"),
+		dropReasonQueueHead:        getMetric("queue_head"),
+		dropReasonQueueTail:        getMetric("queue_tail"),
+		dropReasonWriteError:       getMetric("write_error"),
+		dropReasonDupClient:        getMetric("dup_client"),
+	}
+	if len(ret) != int(numDropReasons) {
+		panic("dropReason metrics out of sync")
+	}
+	for i := range numDropReasons {
+		if ret[i] == nil {
+			panic("dropReason metrics out of sync")
+		}
+	}
+	return ret
+}
+
 // SetMesh sets the pre-shared key that regional DERP servers used to mesh
 // amongst themselves.
 //
@@ -780,7 +792,6 @@ func (c *sclient) run(ctx context.Context) error {
 	var grp errgroup.Group
 	sendCtx, cancelSender := context.WithCancel(ctx)
 	grp.Go(func() error { return c.sendLoop(sendCtx) })
-	grp.Go(func() error { return c.statsLoop(sendCtx) })
 	defer func() {
 		cancelSender()
 		if err := grp.Wait(); err != nil && !c.s.isClosed() {
@@ -792,6 +803,8 @@ func (c *sclient) run(ctx context.Context) error {
 		}
 	}()
 
+	c.startStatsLoop(sendCtx)
+
 	for {
 		ft, fl, err := readFrameHeader(c.br)
 		c.debugLogf("read frame type %d len %d err %v", ft, fl, err)
@@ -1050,6 +1063,7 @@ const (
 	dropReasonQueueTail                          // destination queue is full, dropped packet at queue tail
 	dropReasonWriteError                         // OS write() failed
 	dropReasonDupClient                          // the public key is connected 2+ times (active/active, fighting)
+	numDropReasons                               // unused; keep last
 )
 
 func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) {

derp/derp_server_default.go

@@ -7,6 +7,7 @@ package derp
 
 import "context"
 
-func (c *sclient) statsLoop(ctx context.Context) error {
-	return nil
+func (c *sclient) startStatsLoop(ctx context.Context) {
+	// Nothing to do
+	return
 }

derp/derp_server_linux.go

@@ -12,40 +12,43 @@ import (
 	"tailscale.com/net/tcpinfo"
 )
 
-func (c *sclient) statsLoop(ctx context.Context) error {
+func (c *sclient) startStatsLoop(ctx context.Context) {
 	// Get the RTT initially to verify it's supported.
 	conn := c.tcpConn()
 	if conn == nil {
 		c.s.tcpRtt.Add("non-tcp", 1)
-		return nil
+		return
 	}
 	if _, err := tcpinfo.RTT(conn); err != nil {
 		c.logf("error fetching initial RTT: %v", err)
 		c.s.tcpRtt.Add("error", 1)
-		return nil
+		return
 	}
 
 	const statsInterval = 10 * time.Second
 
-	ticker, tickerChannel := c.s.clock.NewTicker(statsInterval)
-	defer ticker.Stop()
-
-statsLoop:
-	for {
-		select {
-		case <-tickerChannel:
-			rtt, err := tcpinfo.RTT(conn)
-			if err != nil {
-				continue statsLoop
-			}
-
-			// TODO(andrew): more metrics?
-			c.s.tcpRtt.Add(durationToLabel(rtt), 1)
-
-		case <-ctx.Done():
-			return ctx.Err()
+	// Don't launch a goroutine; use a timer instead.
+	var gatherStats func()
+	gatherStats = func() {
+		// Do nothing if the context is finished.
+		if ctx.Err() != nil {
+			return
 		}
+
+		// Reschedule ourselves when this stats gathering is finished.
+		defer c.s.clock.AfterFunc(statsInterval, gatherStats)
+
+		// Gather TCP RTT information.
+		rtt, err := tcpinfo.RTT(conn)
+		if err == nil {
+			c.s.tcpRtt.Add(durationToLabel(rtt), 1)
+		}
+
+		// TODO(andrew): more metrics?
 	}
+
+	// Kick off the initial timer.
+	c.s.clock.AfterFunc(statsInterval, gatherStats)
 }
 
 // tcpConn attempts to get the underlying *net.TCPConn from this client's

derp/derphttp/derphttp_client.go

@@ -31,6 +31,7 @@ import (
 	"go4.org/mem"
 	"tailscale.com/derp"
 	"tailscale.com/envknob"
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/netns"
@@ -51,10 +52,11 @@ import (
 // Send/Recv will completely re-establish the connection (unless Close
 // has been called).
 type Client struct {
-	TLSConfig *tls.Config        // optional; nil means default
-	DNSCache  *dnscache.Resolver // optional; nil means no caching
-	MeshKey   string             // optional; for trusted clients
-	IsProber  bool               // optional; for probers to optional declare themselves as such
+	TLSConfig     *tls.Config        // optional; nil means default
+	HealthTracker *health.Tracker    // optional; used if non-nil only
+	DNSCache      *dnscache.Resolver // optional; nil means no caching
+	MeshKey       string             // optional; for trusted clients
+	IsProber      bool               // optional; for probers to optional declare themselves as such
 
 	// WatchConnectionChanges is whether the client wishes to subscribe to
 	// notifications about clients connecting & disconnecting.
@@ -69,7 +71,7 @@ type Client struct {
 
 	privateKey key.NodePrivate
 	logf       logger.Logf
-	netMon     *netmon.Monitor // optional; nil means interfaces will be looked up on-demand
+	netMon     *netmon.Monitor // always non-nil
 	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
 	// Either url or getRegion is non-nil:
@@ -114,8 +116,11 @@ func (c *Client) String() string {
 
 // NewRegionClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
-// The netMon parameter is optional; if non-nil it's used to do faster interface lookups.
+// The healthTracker parameter is also optional.
 func NewRegionClient(privateKey key.NodePrivate, logf logger.Logf, netMon *netmon.Monitor, getRegion func() *tailcfg.DERPRegion) *Client {
+	if netMon == nil {
+		panic("nil netMon")
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	c := &Client{
 		privateKey: privateKey,
@@ -131,13 +136,23 @@ func NewRegionClient(privateKey key.NodePrivate, logf logger.Logf, netMon *netmo
 
 // NewNetcheckClient returns a Client that's only able to have its DialRegionTLS method called.
 // It's used by the netcheck package.
-func NewNetcheckClient(logf logger.Logf) *Client {
-	return &Client{logf: logf, clock: tstime.StdClock{}}
+func NewNetcheckClient(logf logger.Logf, netMon *netmon.Monitor) *Client {
+	if netMon == nil {
+		panic("nil netMon")
+	}
+	return &Client{
+		logf:   logf,
+		clock:  tstime.StdClock{},
+		netMon: netMon,
+	}
 }
 
 // NewClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
-func NewClient(privateKey key.NodePrivate, serverURL string, logf logger.Logf) (*Client, error) {
+func NewClient(privateKey key.NodePrivate, serverURL string, logf logger.Logf, netMon *netmon.Monitor) (*Client, error) {
+	if netMon == nil {
+		panic("nil netMon")
+	}
 	u, err := url.Parse(serverURL)
 	if err != nil {
 		return nil, fmt.Errorf("derphttp.NewClient: %v", err)
@@ -154,6 +169,7 @@ func NewClient(privateKey key.NodePrivate, serverURL string, logf logger.Logf) (
 		ctx:        ctx,
 		cancelCtx:  cancel,
 		clock:      tstime.StdClock{},
+		netMon:     netMon,
 	}
 	return c, nil
 }
@@ -612,7 +628,7 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 }
 
 func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn {
-	tlsConf := tlsdial.Config(c.tlsServerName(node), c.TLSConfig)
+	tlsConf := tlsdial.Config(c.tlsServerName(node), c.HealthTracker, c.TLSConfig)
 	if node != nil {
 		if node.InsecureForTests {
 			tlsConf.InsecureSkipVerify = true

derp/derphttp/derphttp_server.go

@@ -20,6 +20,16 @@ const fastStartHeader = "Derp-Fast-Start"
 
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// These are installed both here and in cmd/derper. The check here
+		// catches both cmd/derper run with DERP disabled (STUN only mode) as
+		// well as DERP being run in tests with derphttp.Handler directly,
+		// as netcheck still assumes this replies.
+		switch r.URL.Path {
+		case "/derp/probe", "/derp/latency-check":
+			ProbeHandler(w, r)
+			return
+		}
+
 		up := strings.ToLower(r.Header.Get("Upgrade"))
 		if up != "websocket" && up != "derp" {
 			if up != "" {
@@ -58,3 +68,14 @@ func Handler(s *derp.Server) http.Handler {
 		s.Accept(r.Context(), netConn, conn, netConn.RemoteAddr().String())
 	})
 }
+
+// ProbeHandler is the endpoint that clients without UDP access (including js/wasm) hit to measure
+// DERP latency, as a replacement for UDP STUN queries.
+func ProbeHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case "HEAD", "GET":
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+	default:
+		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
+	}
+}

derp/derphttp/derphttp_test.go

@@ -10,18 +10,22 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"net/netip"
 	"sync"
 	"testing"
 	"time"
 
 	"tailscale.com/derp"
+	"tailscale.com/net/netmon"
 	"tailscale.com/types/key"
 )
 
 func TestSendRecv(t *testing.T) {
 	serverPrivateKey := key.NewNode()
 
+	netMon := netmon.NewStatic()
+
 	const numClients = 3
 	var clientPrivateKeys []key.NodePrivate
 	var clientKeys []key.NodePublic
@@ -68,7 +72,7 @@ func TestSendRecv(t *testing.T) {
 	}()
 	for i := range numClients {
 		key := clientPrivateKeys[i]
-		c, err := NewClient(key, serverURL, t.Logf)
+		c, err := NewClient(key, serverURL, t.Logf, netMon)
 		if err != nil {
 			t.Fatalf("client %d: %v", i, err)
 		}
@@ -183,7 +187,7 @@ func TestPing(t *testing.T) {
 		}
 	}()
 
-	c, err := NewClient(key.NewNode(), serverURL, t.Logf)
+	c, err := NewClient(key.NewNode(), serverURL, t.Logf, netmon.NewStatic())
 	if err != nil {
 		t.Fatalf("NewClient: %v", err)
 	}
@@ -236,7 +240,7 @@ func newTestServer(t *testing.T, k key.NodePrivate) (serverURL string, s *derp.S
 }
 
 func newWatcherClient(t *testing.T, watcherPrivateKey key.NodePrivate, serverToWatchURL string) (c *Client) {
-	c, err := NewClient(watcherPrivateKey, serverToWatchURL, t.Logf)
+	c, err := NewClient(watcherPrivateKey, serverToWatchURL, t.Logf, netmon.NewStatic())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -461,3 +465,24 @@ func TestLocalAddrNoMutex(t *testing.T) {
 		t.Errorf("got error %q; want %q", got, want)
 	}
 }
+
+func TestProbe(t *testing.T) {
+	h := Handler(nil)
+
+	tests := []struct {
+		path string
+		want int
+	}{
+		{"/derp/probe", 200},
+		{"/derp/latency-check", 200},
+		{"/derp/sdf", http.StatusUpgradeRequired},
+	}
+
+	for _, tt := range tests {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest("GET", tt.path, nil))
+		if got := rec.Result().StatusCode; got != tt.want {
+			t.Errorf("for path %q got HTTP status %v; want %v", tt.path, got, tt.want)
+		}
+	}
+}

derp/dropreason_string.go

@@ -18,11 +18,12 @@ func _() {
 	_ = x[dropReasonQueueTail-4]
 	_ = x[dropReasonWriteError-5]
 	_ = x[dropReasonDupClient-6]
+	_ = x[numDropReasons-7]
 }
 
-const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneDisconnectedQueueHeadQueueTailWriteErrorDupClient"
+const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneDisconnectedQueueHeadQueueTailWriteErrorDupClientnumDropReasons"
 
-var _dropReason_index = [...]uint8{0, 11, 27, 43, 52, 61, 71, 80}
+var _dropReason_index = [...]uint8{0, 11, 27, 43, 52, 61, 71, 80, 94}
 
 func (i dropReason) String() string {
 	if i < 0 || i >= dropReason(len(_dropReason_index)-1) {

docs/k8s/subnet.yaml

@@ -29,5 +29,6 @@ spec:
     - name: TS_ROUTES
       value: "{{TS_ROUTES}}"
     securityContext:
-      runAsUser: 1000
-      runAsGroup: 1000
+      capabilities:
+        add:
+        - NET_ADMIN

doctor/ethtool/ethtool_linux.go

@@ -8,7 +8,7 @@ import (
 	"sort"
 
 	"github.com/safchain/ethtool"
-	"tailscale.com/net/interfaces"
+	"tailscale.com/net/netmon"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/set"
 )
@@ -21,7 +21,7 @@ func ethtoolImpl(logf logger.Logf) error {
 	}
 	defer et.Close()
 
-	interfaces.ForeachInterface(func(iface interfaces.Interface, _ []netip.Prefix) {
+	netmon.ForeachInterface(func(iface netmon.Interface, _ []netip.Prefix) {
 		ilogf := logger.WithPrefix(logf, iface.Name+": ")
 		features, err := et.Features(iface.Name)
 		if err == nil {

drive/driveimpl/compositedav/compositedav.go

@@ -27,10 +27,10 @@ import (
 type Child struct {
 	*dirfs.Child
 
-	// BaseURL is the base URL of the WebDAV service to which we'll proxy
+	// BaseURL returns the base URL of the WebDAV service to which we'll proxy
 	// requests for this Child. We will append the filename from the original
 	// URL to this.
-	BaseURL string
+	BaseURL func() (string, error)
 
 	// Transport (if specified) is the http transport to use when communicating
 	// with this Child's WebDAV service.
@@ -81,24 +81,39 @@ type Handler struct {
 	staticRoot string
 }
 
+var cacheInvalidatingMethods = map[string]bool{
+	"PUT":       true,
+	"POST":      true,
+	"COPY":      true,
+	"MKCOL":     true,
+	"MOVE":      true,
+	"PROPPATCH": true,
+	"DELETE":    true,
+}
+
 // ServeHTTP implements http.Handler.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == "PROPFIND" {
-		h.handlePROPFIND(w, r)
+	pathComponents := shared.CleanAndSplit(r.URL.Path)
+	mpl := h.maxPathLength(r)
+
+	switch r.Method {
+	case "PROPFIND":
+		h.handlePROPFIND(w, r, pathComponents, mpl)
+		return
+	case "LOCK":
+		h.handleLOCK(w, r, pathComponents, mpl)
 		return
 	}
 
-	if r.Method != "GET" {
-		// If the user is performing a modification (e.g. PUT, MKDIR, etc),
+	_, shouldInvalidate := cacheInvalidatingMethods[r.Method]
+	if shouldInvalidate {
+		// If the user is performing a modification (e.g. PUT, MKDIR, etc.),
 		// we need to invalidate the StatCache to make sure we're not knowingly
 		// showing stale stats.
-		// TODO(oxtoacart): maybe be more selective about invalidating cache
+		// TODO(oxtoacart): maybe only invalidate specific paths
 		h.StatCache.invalidate()
 	}
 
-	mpl := h.maxPathLength(r)
-	pathComponents := shared.CleanAndSplit(r.URL.Path)
-
 	if len(pathComponents) >= mpl {
 		h.delegate(mpl, pathComponents[mpl-1:], w, r)
 		return
@@ -130,6 +145,8 @@ func (h *Handler) handle(w http.ResponseWriter, r *http.Request) {
 
 // delegate sends the request to the Child WebDAV server.
 func (h *Handler) delegate(mpl int, pathComponents []string, w http.ResponseWriter, r *http.Request) {
+	rewriteIfHeader(r, pathComponents, mpl)
+
 	dest := r.Header.Get("Destination")
 	if dest != "" {
 		// Rewrite destination header
@@ -154,9 +171,15 @@ func (h *Handler) delegate(mpl int, pathComponents []string, w http.ResponseWrit
 		return
 	}
 
-	u, err := url.Parse(child.BaseURL)
+	baseURL, err := child.BaseURL()
 	if err != nil {
-		h.logf("warning: parse base URL %s failed: %s", child.BaseURL, err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		h.logf("warning: parse base URL %s failed: %s", baseURL, err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}

drive/driveimpl/compositedav/propfind.go

@@ -1,84 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package compositedav
-
-import (
-	"bytes"
-	"fmt"
-	"math"
-	"net/http"
-	"regexp"
-
-	"tailscale.com/drive/driveimpl/shared"
-)
-
-var (
-	hrefRegex = regexp.MustCompile(`(?s)<D:href>/?([^<]*)/?</D:href>`)
-)
-
-func (h *Handler) handlePROPFIND(w http.ResponseWriter, r *http.Request) {
-	pathComponents := shared.CleanAndSplit(r.URL.Path)
-	mpl := h.maxPathLength(r)
-	if !shared.IsRoot(r.URL.Path) && len(pathComponents)+getDepth(r) > mpl {
-		// Delegate to a Child.
-		depth := getDepth(r)
-
-		cached := h.StatCache.get(r.URL.Path, depth)
-		if cached != nil {
-			w.Header().Del("Content-Length")
-			w.WriteHeader(http.StatusMultiStatus)
-			w.Write(cached)
-			return
-		}
-
-		// Use a buffering ResponseWriter so that we can manipulate the result.
-		// The only thing we use from the original ResponseWriter is Header().
-		bw := &bufferingResponseWriter{ResponseWriter: w}
-
-		mpl := h.maxPathLength(r)
-		h.delegate(mpl, pathComponents[mpl-1:], bw, r)
-
-		// Fixup paths to add the requested path as a prefix.
-		pathPrefix := shared.Join(pathComponents[0:mpl]...)
-		b := hrefRegex.ReplaceAll(bw.buf.Bytes(), []byte(fmt.Sprintf("<D:href>%s/$1</D:href>", pathPrefix)))
-
-		if h.StatCache != nil && bw.status == http.StatusMultiStatus && b != nil {
-			h.StatCache.set(r.URL.Path, depth, b)
-		}
-
-		w.Header().Del("Content-Length")
-		w.WriteHeader(bw.status)
-		w.Write(b)
-
-		return
-	}
-
-	h.handle(w, r)
-}
-
-func getDepth(r *http.Request) int {
-	switch r.Header.Get("Depth") {
-	case "0":
-		return 0
-	case "1":
-		return 1
-	case "infinity":
-		return math.MaxInt
-	}
-	return 0
-}
-
-type bufferingResponseWriter struct {
-	http.ResponseWriter
-	status int
-	buf    bytes.Buffer
-}
-
-func (bw *bufferingResponseWriter) WriteHeader(statusCode int) {
-	bw.status = statusCode
-}
-
-func (bw *bufferingResponseWriter) Write(p []byte) (int, error) {
-	return bw.buf.Write(p)
-}

drive/driveimpl/compositedav/rewriting.go

@@ -0,0 +1,122 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package compositedav
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"tailscale.com/drive/driveimpl/shared"
+)
+
+var (
+	responseHrefRegex = regexp.MustCompile(`(?s)(<D:(response|lockroot)>)<D:href>/?([^<]*)/?</D:href>`)
+	ifHrefRegex       = regexp.MustCompile(`^<(https?://[^/]+)?([^>]+)>`)
+)
+
+func (h *Handler) handlePROPFIND(w http.ResponseWriter, r *http.Request, pathComponents []string, mpl int) {
+	if shouldDelegateToChild(r, pathComponents, mpl) {
+		// Delegate to a Child.
+		depth := getDepth(r)
+
+		status, result := h.StatCache.getOr(r.URL.Path, depth, func() (int, []byte) {
+			return h.delegateRewriting(w, r, pathComponents, mpl)
+		})
+
+		respondRewritten(w, status, result)
+		return
+	}
+
+	h.handle(w, r)
+}
+
+func (h *Handler) handleLOCK(w http.ResponseWriter, r *http.Request, pathComponents []string, mpl int) {
+	if shouldDelegateToChild(r, pathComponents, mpl) {
+		// Delegate to a Child.
+		status, result := h.delegateRewriting(w, r, pathComponents, mpl)
+		respondRewritten(w, status, result)
+		return
+	}
+
+	http.Error(w, "locking of top level directories is not allowed", http.StatusMethodNotAllowed)
+}
+
+// shouldDelegateToChild decides whether a request should be delegated to a
+// child filesystem, as opposed to being handled by this filesystem. It checks
+// the depth of the requested path, and if it's deeper than the portion of the
+// tree that's handled by the parent, returns true.
+func shouldDelegateToChild(r *http.Request, pathComponents []string, mpl int) bool {
+	return !shared.IsRoot(r.URL.Path) && len(pathComponents)+getDepth(r) > mpl
+}
+
+func (h *Handler) delegateRewriting(w http.ResponseWriter, r *http.Request, pathComponents []string, mpl int) (int, []byte) {
+	// Use a buffering ResponseWriter so that we can manipulate the result.
+	// The only thing we use from the original ResponseWriter is Header().
+	bw := &bufferingResponseWriter{ResponseWriter: w}
+
+	h.delegate(mpl, pathComponents[mpl-1:], bw, r)
+
+	// Fixup paths to add the requested path as a prefix, escaped for inclusion in XML.
+	pp := shared.EscapeForXML(shared.Join(pathComponents[0:mpl]...))
+	b := responseHrefRegex.ReplaceAll(bw.buf.Bytes(), []byte(fmt.Sprintf("$1<D:href>%s/$3</D:href>", pp)))
+	return bw.status, b
+}
+
+func respondRewritten(w http.ResponseWriter, status int, result []byte) {
+	w.Header().Del("Content-Length")
+	w.WriteHeader(status)
+	if result != nil {
+		w.Write(result)
+	}
+}
+
+func getDepth(r *http.Request) int {
+	switch r.Header.Get("Depth") {
+	case "0":
+		return 0
+	case "1":
+		return 1
+	case "infinity":
+		return math.MaxInt16 // a really large number, but not infinity (avoids wrapping when we do arithmetic with this)
+	}
+	return 0
+}
+
+type bufferingResponseWriter struct {
+	http.ResponseWriter
+	status int
+	buf    bytes.Buffer
+}
+
+func (bw *bufferingResponseWriter) WriteHeader(statusCode int) {
+	bw.status = statusCode
+}
+
+func (bw *bufferingResponseWriter) Write(p []byte) (int, error) {
+	return bw.buf.Write(p)
+}
+
+// rewriteIfHeader rewrites URLs in the If header by removing the host and the
+// portion of the path that corresponds to this composite filesystem. This way,
+// when we delegate requests to child filesystems, the If header will reference
+// a path that makes sense on those filesystems.
+//
+// See http://www.webdav.org/specs/rfc4918.html#HEADER_If
+func rewriteIfHeader(r *http.Request, pathComponents []string, mpl int) {
+	ih := r.Header.Get("If")
+	if ih == "" {
+		return
+	}
+	matches := ifHrefRegex.FindStringSubmatch(ih)
+	if len(matches) == 3 {
+		pp := shared.JoinEscaped(pathComponents[0:mpl]...)
+		p := strings.Replace(shared.JoinEscaped(pathComponents...), pp, "", 1)
+		nih := ifHrefRegex.ReplaceAllString(ih, fmt.Sprintf("<%s>", p))
+		r.Header.Set("If", nih)
+	}
+}

drive/driveimpl/compositedav/stat_cache.go

@@ -4,10 +4,19 @@
 package compositedav
 
 import (
+	"bytes"
+	"encoding/xml"
+	"log"
+	"net/http"
 	"sync"
 	"time"
 
 	"github.com/jellydator/ttlcache/v3"
+	"tailscale.com/drive/driveimpl/shared"
+)
+
+var (
+	notFound = newCacheEntry(http.StatusNotFound, nil)
 )
 
 // StatCache provides a cache for directory listings and file metadata.
@@ -17,22 +26,97 @@ import (
 // This is similar to the DirectoryCacheLifetime setting of Windows' built-in
 // SMB client, see
 // https://learn.microsoft.com/en-us/previous-versions/windows/it-pro/windows-7/ff686200(v=ws.10)
+//
+// StatCache is built specifically to cache the results of PROPFIND requests,
+// which come back as MultiStatus XML responses. Typical clients will issue two
+// kinds of PROPFIND:
+//
+// The first kind of PROPFIND is a directory listing performed to depth 1. At
+// this depth, the resulting XML will contain stats for the requested folder as
+// well as for all children of that folder.
+//
+// The second kind of PROPFIND is a file listing performed to depth 0. At this
+// depth, the resulting XML will contain stats only for the requested file.
+//
+// In order to avoid round-trips, when a PROPFIND at depth 0 is attempted, and
+// the requested file is not in the cache, StatCache will check to see if the
+// parent folder of that file is cached. If so, StatCache infers the correct
+// MultiStatus for the file according to the following logic:
+//
+//  1. If the parent folder is NotFound (404), treat the file itself as NotFound
+//  2. If the parent folder's XML doesn't contain the file, treat it as
+//     NotFound.
+//  3. If the parent folder's XML contains the file, build a MultiStatus for the
+//     file based on the parent's XML.
+//
+// To avoid inconsistencies from the perspective of the client, any operations
+// that modify the filesystem (e.g. PUT, MKDIR, etc.) should call invalidate()
+// to invalidate the cache.
 type StatCache struct {
 	TTL time.Duration
 
 	// mu guards the below values.
 	mu                   sync.Mutex
-	cachesByDepthAndPath map[int]*ttlcache.Cache[string, []byte]
+	cachesByDepthAndPath map[int]*ttlcache.Cache[string, *cacheEntry]
 }
 
-func (c *StatCache) get(name string, depth int) []byte {
+// getOr checks the cache for the named value at the given depth. If a cached
+// value was found, it returns http.StatusMultiStatus along with the cached
+// value. Otherwise, it executes the given function and returns the resulting
+// status and value. If the function returned http.StatusMultiStatus, getOr
+// caches the resulting value at the given name and depth before returning.
+func (c *StatCache) getOr(name string, depth int, or func() (int, []byte)) (int, []byte) {
+	ce := c.get(name, depth)
+	if ce == nil {
+		// Not cached, fetch value.
+		status, raw := or()
+		ce = newCacheEntry(status, raw)
+		if status == http.StatusMultiStatus || status == http.StatusNotFound {
+			// Got a legit status, cache value
+			c.set(name, depth, ce)
+		}
+	}
+	return ce.Status, ce.Raw
+}
+
+// get retrieves the entry for the named file at the given depth. If no entry
+// is found, and depth == 0, get will check to see if the parent path of name
+// is present in the cache at depth 1. If so, it will infer that the child does
+// not exist and return notFound (404).
+func (c *StatCache) get(name string, depth int) *cacheEntry {
 	if c == nil {
 		return nil
 	}
 
+	name = shared.Normalize(name)
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	ce := c.tryGetLocked(name, depth)
+	if ce != nil {
+		// Cache hit.
+		return ce
+	}
+
+	if depth > 0 {
+		// Cache miss.
+		return nil
+	}
+
+	// At depth 0, if child's parent is in the cache, and the child isn't
+	// cached, we can infer that the child is notFound.
+	p := c.tryGetLocked(shared.Parent(name), 1)
+	if p != nil {
+		return notFound
+	}
+
+	// No parent in cache, cache miss.
+	return nil
+}
+
+// tryGetLocked requires that c.mu be held.
+func (c *StatCache) tryGetLocked(name string, depth int) *cacheEntry {
 	if c.cachesByDepthAndPath == nil {
 		return nil
 	}
@@ -47,28 +131,80 @@ func (c *StatCache) get(name string, depth int) []byte {
 	return item.Value()
 }
 
-func (c *StatCache) set(name string, depth int, value []byte) {
+// set stores the given cacheEntry in the cache at the given name and depth. If
+// the depth is 1, set also populates depth 0 entries in the cache for the bare
+// name. If status is StatusMultiStatus, set will parse the PROPFIND result and
+// store depth 0 entries for all children. If parsing the result fails, nothing
+// is cached.
+func (c *StatCache) set(name string, depth int, ce *cacheEntry) {
 	if c == nil {
 		return
 	}
 
+	name = shared.Normalize(name)
+
+	var self *cacheEntry
+	var children map[string]*cacheEntry
+	if depth == 1 {
+		switch ce.Status {
+		case http.StatusNotFound:
+			// Record notFound as the self entry.
+			self = ce
+		case http.StatusMultiStatus:
+			// Parse the raw MultiStatus and extract specific responses
+			// corresponding to the self entry (e.g. the directory, but at depth 0)
+			// and children (e.g. files within the directory) so that subsequent
+			// requests for these can be satisfied from the cache.
+			var ms multiStatus
+			err := xml.Unmarshal(ce.Raw, &ms)
+			if err != nil {
+				// unparseable MultiStatus response, don't cache
+				log.Printf("statcache.set error: %s", err)
+				return
+			}
+			children = make(map[string]*cacheEntry, len(ms.Responses)-1)
+			for i := 0; i < len(ms.Responses); i++ {
+				response := ms.Responses[i]
+				name := shared.Normalize(response.Href)
+				raw := marshalMultiStatus(response)
+				entry := newCacheEntry(ce.Status, raw)
+				if i == 0 {
+					self = entry
+				} else {
+					children[name] = entry
+				}
+			}
+		}
+	}
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	c.setLocked(name, depth, ce)
+	if self != nil {
+		c.setLocked(name, 0, self)
+	}
+	for childName, child := range children {
+		c.setLocked(childName, 0, child)
+	}
+}
 
+// setLocked requires that c.mu be held.
+func (c *StatCache) setLocked(name string, depth int, ce *cacheEntry) {
 	if c.cachesByDepthAndPath == nil {
-		c.cachesByDepthAndPath = make(map[int]*ttlcache.Cache[string, []byte])
+		c.cachesByDepthAndPath = make(map[int]*ttlcache.Cache[string, *cacheEntry])
 	}
 	cache := c.cachesByDepthAndPath[depth]
 	if cache == nil {
 		cache = ttlcache.New(
-			ttlcache.WithTTL[string, []byte](c.TTL),
+			ttlcache.WithTTL[string, *cacheEntry](c.TTL),
 		)
 		go cache.Start()
 		c.cachesByDepthAndPath[depth] = cache
 	}
-	cache.Set(name, value, ttlcache.DefaultTTL)
+	cache.Set(name, ce, ttlcache.DefaultTTL)
 }
 
+// invalidate invalidates the entire cache.
 func (c *StatCache) invalidate() {
 	if c == nil {
 		return
@@ -90,3 +226,54 @@ func (c *StatCache) stop() {
 		cache.Stop()
 	}
 }
+
+type cacheEntry struct {
+	Status int
+	Raw    []byte
+}
+
+func newCacheEntry(status int, raw []byte) *cacheEntry {
+	return &cacheEntry{Status: status, Raw: raw}
+}
+
+type propStat struct {
+	InnerXML []byte `xml:",innerxml"`
+}
+
+type response struct {
+	XMLName   xml.Name    `xml:"response"`
+	Href      string      `xml:"href"`
+	PropStats []*propStat `xml:"propstat"`
+}
+
+type multiStatus struct {
+	XMLName   xml.Name    `xml:"multistatus"`
+	Responses []*response `xml:"response"`
+}
+
+// marshalMultiStatus performs custom marshalling of a MultiStatus to preserve
+// the original formatting, namespacing, etc. Doing this with Go's XML encoder
+// is somewhere between difficult and impossible, which is why we use this more
+// manual approach.
+func marshalMultiStatus(response *response) []byte {
+	// TODO(percy): maybe pool these buffers
+	var buf bytes.Buffer
+	buf.WriteString(multistatusTemplateStart)
+	buf.WriteString(response.Href)
+	buf.WriteString(hrefEnd)
+	for _, propStat := range response.PropStats {
+		buf.WriteString(propstatStart)
+		buf.Write(propStat.InnerXML)
+		buf.WriteString(propstatEnd)
+	}
+	buf.WriteString(multistatusTemplateEnd)
+	return buf.Bytes()
+}
+
+const (
+	multistatusTemplateStart = `<?xml version="1.0" encoding="UTF-8"?><D:multistatus xmlns:D="DAV:"><D:response><D:href>`
+	hrefEnd                  = `</D:href>`
+	propstatStart            = `<D:propstat>`
+	propstatEnd              = `</D:propstat>`
+	multistatusTemplateEnd   = `</D:response></D:multistatus>`
+)