ipn/ipnlocal: empty allowed exit nodes syspolicy should be treated as allow all

Updates tailscale/corp#19681

If the syspolicy returns an empty list of allowed exit nodes,
this should be treated as "allow all" rather than "allow none"

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

Makefile

@@ -115,10 +115,7 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
-	echo "Testing on fedora:38" && docker build --build-arg="BASE=dokken/fedora-38" -t ssh-fedora-38 ssh/tailssh/testcontainers && \
-	echo "Testing on fedora:39" && docker build --build-arg="BASE=dokken/fedora-39" -t ssh-fedora-39 ssh/tailssh/testcontainers && \
-	echo "Testing on fedora:40" && docker build --build-arg="BASE=dokken/fedora-40" -t ssh-fedora-40 ssh/tailssh/testcontainers
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers
 
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"

client/tailscale/localclient.go

@@ -778,6 +778,17 @@ func (lc *LocalClient) SetDNS(ctx context.Context, name, value string) error {
 //
 // The ctx is only used for the duration of the call, not the lifetime of the net.Conn.
 func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) {
+	return lc.UserDial(ctx, "tcp", host, port)
+}
+
+// UserDial connects to the host's port via Tailscale for the given network.
+//
+// The host may be a base DNS name (resolved from the netmap inside tailscaled),
+// a FQDN, or an IP address.
+//
+// The ctx is only used for the duration of the call, not the lifetime of the
+// net.Conn.
+func (lc *LocalClient) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) {
 	connCh := make(chan net.Conn, 1)
 	trace := httptrace.ClientTrace{
 		GotConn: func(info httptrace.GotConnInfo) {
@@ -790,10 +801,11 @@ func (lc *LocalClient) DialTCP(ctx context.Context, host string, port uint16) (n
 		return nil, err
 	}
 	req.Header = http.Header{
-		"Upgrade":    []string{"ts-dial"},
-		"Connection": []string{"upgrade"},
-		"Dial-Host":  []string{host},
-		"Dial-Port":  []string{fmt.Sprint(port)},
+		"Upgrade":      []string{"ts-dial"},
+		"Connection":   []string{"upgrade"},
+		"Dial-Host":    []string{host},
+		"Dial-Port":    []string{fmt.Sprint(port)},
+		"Dial-Network": []string{network},
 	}
 	res, err := lc.DoLocalRequest(req)
 	if err != nil {

client/tailscale/localclient_test.go

@@ -35,6 +35,7 @@ func TestDeps(t *testing.T) {
 		BadDeps: map[string]string{
 			// Make sure we don't again accidentally bring in a dependency on
 			// drive or its transitive dependencies
+			"testing":                        "do not use testing package in production code",
 			"tailscale.com/drive/driveimpl":  "https://github.com/tailscale/tailscale/pull/10631",
 			"github.com/studio-b12/gowebdav": "https://github.com/tailscale/tailscale/pull/10631",
 		},

clientupdate/clientupdate.go

@@ -653,6 +653,9 @@ func (up *Updater) updateAlpineLike() (err error) {
 		return fmt.Errorf(`failed to parse latest version from "apk info tailscale": %w`, err)
 	}
 	if !up.confirm(ver) {
+		if err := checkOutdatedAlpineRepo(up.Logf, ver, up.Track); err != nil {
+			up.Logf("failed to check whether Alpine release is outdated: %v", err)
+		}
 		return nil
 	}
 
@@ -690,6 +693,37 @@ func parseAlpinePackageVersion(out []byte) (string, error) {
 	return "", errors.New("tailscale version not found in output")
 }
 
+var apkRepoVersionRE = regexp.MustCompile(`v[0-9]+\.[0-9]+`)
+
+func checkOutdatedAlpineRepo(logf logger.Logf, apkVer, track string) error {
+	latest, err := LatestTailscaleVersion(track)
+	if err != nil {
+		return err
+	}
+	if latest == apkVer {
+		// Actually on latest release.
+		return nil
+	}
+	f, err := os.Open("/etc/apk/repositories")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	// Read the first repo line. Typically, there are multiple repos that all
+	// contain the same version in the path, like:
+	//   https://dl-cdn.alpinelinux.org/alpine/v3.20/main
+	//   https://dl-cdn.alpinelinux.org/alpine/v3.20/community
+	s := bufio.NewScanner(f)
+	if !s.Scan() {
+		return s.Err()
+	}
+	alpineVer := apkRepoVersionRE.FindString(s.Text())
+	if alpineVer != "" {
+		logf("The latest Tailscale release for Linux is %q, but your apk repository only provides %q.\nYour Alpine version is %q, you may need to upgrade the system to get the latest Tailscale version: https://wiki.alpinelinux.org/wiki/Upgrading_Alpine", latest, apkVer, alpineVer)
+	}
+	return nil
+}
+
 func (up *Updater) updateMacSys() error {
 	return errors.New("NOTREACHED: On MacSys builds, `tailscale update` is handled in Swift to launch the GUI updater")
 }

cmd/containerboot/kube.go

@@ -138,9 +138,9 @@ func initKubeClient(root string) {
 	if err != nil {
 		log.Fatalf("Error creating kube client: %v", err)
 	}
-	if root != "/" {
-		// If we are running in a test, we need to set the URL to the
-		// httptest server.
+	if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		// Used to set http server in tests, or optionally enabled by flag
 		kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
 	}
 }

cmd/derper/bootstrap_dns.go

@@ -5,35 +5,45 @@ package main
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"expvar"
 	"log"
+	"math/rand/v2"
 	"net"
 	"net/http"
+	"net/netip"
+	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/syncs"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/slicesx"
 )
 
 const refreshTimeout = time.Minute
 
-type dnsEntryMap map[string][]net.IP
+type dnsEntryMap struct {
+	IPs     map[string][]net.IP
+	Percent map[string]float64 // "foo.com" => 0.5 for 50%
+}
 
 var (
-	dnsCache            syncs.AtomicValue[dnsEntryMap]
+	dnsCache            atomic.Pointer[dnsEntryMap]
 	dnsCacheBytes       syncs.AtomicValue[[]byte] // of JSON
-	unpublishedDNSCache syncs.AtomicValue[dnsEntryMap]
+	unpublishedDNSCache atomic.Pointer[dnsEntryMap]
 	bootstrapLookupMap  syncs.Map[string, bool]
 )
 
 var (
-	bootstrapDNSRequests = expvar.NewInt("counter_bootstrap_dns_requests")
-	publishedDNSHits     = expvar.NewInt("counter_bootstrap_dns_published_hits")
-	publishedDNSMisses   = expvar.NewInt("counter_bootstrap_dns_published_misses")
-	unpublishedDNSHits   = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
-	unpublishedDNSMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+	bootstrapDNSRequests        = expvar.NewInt("counter_bootstrap_dns_requests")
+	publishedDNSHits            = expvar.NewInt("counter_bootstrap_dns_published_hits")
+	publishedDNSMisses          = expvar.NewInt("counter_bootstrap_dns_published_misses")
+	unpublishedDNSHits          = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
+	unpublishedDNSMisses        = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+	unpublishedDNSPercentMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_percent_misses")
 )
 
 func init() {
@@ -59,15 +69,13 @@ func refreshBootstrapDNS() {
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-	dnsEntries := resolveList(ctx, strings.Split(*bootstrapDNS, ","))
+	dnsEntries := resolveList(ctx, *bootstrapDNS)
 	// Randomize the order of the IPs for each name to avoid the client biasing
 	// to IPv6
-	for k := range dnsEntries {
-		ips := dnsEntries[k]
-		slicesx.Shuffle(ips)
-		dnsEntries[k] = ips
+	for _, vv := range dnsEntries.IPs {
+		slicesx.Shuffle(vv)
 	}
-	j, err := json.MarshalIndent(dnsEntries, "", "\t")
+	j, err := json.MarshalIndent(dnsEntries.IPs, "", "\t")
 	if err != nil {
 		// leave the old values in place
 		return
@@ -81,27 +89,50 @@ func refreshUnpublishedDNS() {
 	if *unpublishedDNS == "" {
 		return
 	}
-
 	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-
-	dnsEntries := resolveList(ctx, strings.Split(*unpublishedDNS, ","))
+	dnsEntries := resolveList(ctx, *unpublishedDNS)
 	unpublishedDNSCache.Store(dnsEntries)
 }
 
-func resolveList(ctx context.Context, names []string) dnsEntryMap {
-	dnsEntries := make(dnsEntryMap)
+// resolveList takes a comma-separated list of DNS names to resolve.
+//
+// If an entry contains a slash, it's two DNS names: the first is the one to
+// resolve and the second is that of a TXT recording containing the rollout
+// percentage in range "0".."100". If the TXT record doesn't exist or is
+// malformed, the percentage is 0. If the TXT record is not provided (there's no
+// slash), then the percentage is 100.
+func resolveList(ctx context.Context, list string) *dnsEntryMap {
+	ents := strings.Split(list, ",")
+
+	ret := &dnsEntryMap{}
 
 	var r net.Resolver
-	for _, name := range names {
+	for _, ent := range ents {
+		name, txtName, _ := strings.Cut(ent, "/")
 		addrs, err := r.LookupIP(ctx, "ip", name)
 		if err != nil {
 			log.Printf("bootstrap DNS lookup %q: %v", name, err)
 			continue
 		}
-		dnsEntries[name] = addrs
+		mak.Set(&ret.IPs, name, addrs)
+
+		if txtName == "" {
+			mak.Set(&ret.Percent, name, 1.0)
+			continue
+		}
+		vals, err := r.LookupTXT(ctx, txtName)
+		if err != nil {
+			log.Printf("bootstrap DNS lookup %q: %v", txtName, err)
+			continue
+		}
+		for _, v := range vals {
+			if v, err := strconv.Atoi(v); err == nil && v >= 0 && v <= 100 {
+				mak.Set(&ret.Percent, name, float64(v)/100)
+			}
+		}
 	}
-	return dnsEntries
+	return ret
 }
 
 func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
@@ -115,22 +146,36 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	// Try answering a query from our hidden map first
 	if q := r.URL.Query().Get("q"); q != "" {
 		bootstrapLookupMap.Store(q, true)
-		if ips, ok := unpublishedDNSCache.Load()[q]; ok && len(ips) > 0 {
+		if bootstrapLookupMap.Len() > 500 { // defensive
+			bootstrapLookupMap.Clear()
+		}
+		if m := unpublishedDNSCache.Load(); m != nil && len(m.IPs[q]) > 0 {
 			unpublishedDNSHits.Add(1)
 
-			// Only return the specific query, not everything.
-			m := dnsEntryMap{q: ips}
-			j, err := json.MarshalIndent(m, "", "\t")
-			if err == nil {
-				w.Write(j)
-				return
+			percent := m.Percent[q]
+			if remoteAddrMatchesPercent(r.RemoteAddr, percent) {
+				// Only return the specific query, not everything.
+				m := map[string][]net.IP{q: m.IPs[q]}
+				j, err := json.MarshalIndent(m, "", "\t")
+				if err == nil {
+					w.Write(j)
+					return
+				}
+			} else {
+				unpublishedDNSPercentMisses.Add(1)
 			}
 		}
 
 		// If we have a "q" query for a name in the published cache
 		// list, then track whether that's a hit/miss.
-		if m, ok := dnsCache.Load()[q]; ok {
-			if len(m) > 0 {
+		m := dnsCache.Load()
+		var inPub bool
+		var ips []net.IP
+		if m != nil {
+			ips, inPub = m.IPs[q]
+		}
+		if inPub {
+			if len(ips) > 0 {
 				publishedDNSHits.Add(1)
 			} else {
 				publishedDNSMisses.Add(1)
@@ -146,3 +191,29 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	j := dnsCacheBytes.Load()
 	w.Write(j)
 }
+
+// percent is [0.0, 1.0].
+func remoteAddrMatchesPercent(remoteAddr string, percent float64) bool {
+	if percent == 0 {
+		return false
+	}
+	if percent == 1 {
+		return true
+	}
+	reqIPStr, _, err := net.SplitHostPort(remoteAddr)
+	if err != nil {
+		return false
+	}
+	reqIP, err := netip.ParseAddr(reqIPStr)
+	if err != nil {
+		return false
+	}
+	if reqIP.IsLoopback() {
+		// For local testing.
+		return rand.Float64() < 0.5
+	}
+	reqIP16 := reqIP.As16()
+	rndSrc := rand.NewPCG(binary.LittleEndian.Uint64(reqIP16[:8]), binary.LittleEndian.Uint64(reqIP16[8:]))
+	rnd := rand.New(rndSrc)
+	return percent > rnd.Float64()
+}

cmd/derper/bootstrap_dns_test.go

@@ -4,10 +4,13 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/netip"
 	"net/url"
 	"reflect"
 	"testing"
@@ -38,7 +41,7 @@ func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p),
 
 func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {}
 
-func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
+func getBootstrapDNS(t *testing.T, q string) map[string][]net.IP {
 	t.Helper()
 	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape(q), nil)
 	w := httptest.NewRecorder()
@@ -48,11 +51,12 @@ func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
 	if res.StatusCode != 200 {
 		t.Fatalf("got status=%d; want %d", res.StatusCode, 200)
 	}
-	var ips dnsEntryMap
-	if err := json.NewDecoder(res.Body).Decode(&ips); err != nil {
-		t.Fatalf("error decoding response body: %v", err)
+	var m map[string][]net.IP
+	var buf bytes.Buffer
+	if err := json.NewDecoder(io.TeeReader(res.Body, &buf)).Decode(&m); err != nil {
+		t.Fatalf("error decoding response body %q: %v", buf.Bytes(), err)
 	}
-	return ips
+	return m
 }
 
 func TestUnpublishedDNS(t *testing.T) {
@@ -107,15 +111,21 @@ func resetMetrics() {
 // Verify that we don't count an empty list in the unpublishedDNSCache as a
 // cache hit in our metrics.
 func TestUnpublishedDNSEmptyList(t *testing.T) {
-	pub := dnsEntryMap{
-		"tailscale.com": {net.IPv4(10, 10, 10, 10)},
+	pub := &dnsEntryMap{
+		IPs: map[string][]net.IP{"tailscale.com": {net.IPv4(10, 10, 10, 10)}},
 	}
 	dnsCache.Store(pub)
 	dnsCacheBytes.Store([]byte(`{"tailscale.com":["10.10.10.10"]}`))
 
-	unpublishedDNSCache.Store(dnsEntryMap{
-		"log.tailscale.io":           {},
-		"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+	unpublishedDNSCache.Store(&dnsEntryMap{
+		IPs: map[string][]net.IP{
+			"log.tailscale.io":           {},
+			"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+		},
+		Percent: map[string]float64{
+			"log.tailscale.io":           1.0,
+			"controlplane.tailscale.com": 1.0,
+		},
 	})
 
 	t.Run("CacheMiss", func(t *testing.T) {
@@ -125,8 +135,8 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 			ips := getBootstrapDNS(t, q)
 
 			// Expected our public map to be returned on a cache miss
-			if !reflect.DeepEqual(ips, pub) {
-				t.Errorf("got ips=%+v; want %+v", ips, pub)
+			if !reflect.DeepEqual(ips, pub.IPs) {
+				t.Errorf("got ips=%+v; want %+v", ips, pub.IPs)
 			}
 			if v := unpublishedDNSHits.Value(); v != 0 {
 				t.Errorf("got hits=%d; want 0", v)
@@ -141,7 +151,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 	t.Run("CacheHit", func(t *testing.T) {
 		resetMetrics()
 		ips := getBootstrapDNS(t, "controlplane.tailscale.com")
-		want := dnsEntryMap{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
+		want := map[string][]net.IP{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
 		if !reflect.DeepEqual(ips, want) {
 			t.Errorf("got ips=%+v; want %+v", ips, want)
 		}
@@ -166,3 +176,54 @@ func TestLookupMetric(t *testing.T) {
 		t.Errorf("bootstrapLookupMap.Len() want=5, got %v", bootstrapLookupMap.Len())
 	}
 }
+
+func TestRemoteAddrMatchesPercent(t *testing.T) {
+	tests := []struct {
+		remoteAddr string
+		percent    float64
+		want       bool
+	}{
+		// 0% and 100%.
+		{"10.0.0.1:1234", 0.0, false},
+		{"10.0.0.1:1234", 1.0, true},
+
+		// Invalid IP.
+		{"", 1.0, true},
+		{"", 0.0, false},
+		{"", 0.5, false},
+
+		// Small manual sample at 50%. The func uses a deterministic PRNG seed.
+		{"1.2.3.4:567", 0.5, true},
+		{"1.2.3.5:567", 0.5, true},
+		{"1.2.3.6:567", 0.5, false},
+		{"1.2.3.7:567", 0.5, true},
+		{"1.2.3.8:567", 0.5, false},
+		{"1.2.3.9:567", 0.5, true},
+		{"1.2.3.10:567", 0.5, true},
+	}
+	for _, tt := range tests {
+		got := remoteAddrMatchesPercent(tt.remoteAddr, tt.percent)
+		if got != tt.want {
+			t.Errorf("remoteAddrMatchesPercent(%q, %v) = %v; want %v", tt.remoteAddr, tt.percent, got, tt.want)
+		}
+	}
+
+	var match, all int
+	const wantPercent = 0.5
+	for a := range 256 {
+		for b := range 256 {
+			all++
+			if remoteAddrMatchesPercent(
+				netip.AddrPortFrom(netip.AddrFrom4([4]byte{1, 2, byte(a), byte(b)}), 12345).String(),
+				wantPercent) {
+				match++
+			}
+		}
+	}
+	gotPercent := float64(match) / float64(all)
+	const tolerance = 0.005
+	t.Logf("got percent %v (goal %v)", gotPercent, wantPercent)
+	if gotPercent < wantPercent-tolerance || gotPercent > wantPercent+tolerance {
+		t.Errorf("got %v; want %v ± %v", gotPercent, wantPercent, tolerance)
+	}
+}

cmd/derper/depaware.txt

@@ -235,7 +235,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         encoding/pem                                                 from crypto/tls+
         errors                                                       from bufio+
         expvar                                                       from github.com/prometheus/client_golang/prometheus+
-        flag                                                         from tailscale.com/cmd/derper+
+        flag                                                         from tailscale.com/cmd/derper
         fmt                                                          from compress/flate+
         go/token                                                     from google.golang.org/protobuf/internal/strs
         hash                                                         from crypto+
@@ -253,7 +253,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid
+        math/rand/v2                                                 from tailscale.com/util/fastuuid+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart
@@ -277,7 +277,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         runtime/debug                                                from github.com/prometheus/client_golang/prometheus+
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof
-        runtime/trace                                                from net/http/pprof+
+        runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/ipn/ipnstate+
         sort                                                         from compress/flate+
         strconv                                                      from compress/flate+
@@ -285,7 +285,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         sync                                                         from compress/flate+
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from runtime/pprof
         time                                                         from compress/gzip+
         unicode                                                      from bytes+

cmd/derper/derper.go

@@ -55,7 +55,7 @@ var (
 	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 	bootstrapDNS    = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
-	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list")
+	unpublishedDNS  = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
 	verifyClients   = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 	verifyClientURL = flag.String("verify-client-url", "", "if non-empty, an admission controller URL for permitting client connections; see tailcfg.DERPAdmitClientRequest")
 	verifyFailOpen  = flag.Bool("verify-client-url-fail-open", true, "whether we fail open if --verify-client-url is unreachable")

cmd/derper/derper_test.go

@@ -99,6 +99,7 @@ func TestNoContent(t *testing.T) {
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
+			"testing":                            "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/buffer":       "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",

cmd/k8s-operator/deploy/chart/values.yaml

@@ -51,6 +51,10 @@ operatorConfig:
 # proxies created by the operator.
 # https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress
 # https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress
+# Note that this section contains only a few global configuration options and
+# will not be updated with more configuration options in the future.
+# If you need more configuration options, take a look at ProxyClass:
+# https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
     repo: tailscale/tailscale

cmd/k8s-operator/operator.go

@@ -45,12 +45,12 @@ import (
 	"tailscale.com/version"
 )
 
-// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
-//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
-
 // Generate Connector and ProxyClass CustomResourceDefinition yamls from their Go types.
 //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen crd schemapatch:manifests=./deploy/crds output:dir=./deploy/crds paths=../../k8s-operator/apis/...
 
+// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
+//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
+
 // Generate CRD docs from the yamls
 //go:generate go run fybrik.io/crdoc --resources=./deploy/crds --output=../../k8s-operator/api.md
 

cmd/k8s-operator/svc.go

@@ -161,7 +161,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	}
 	if violations := validateService(svc); len(violations) > 0 {
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", "))
-		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVCICE", msg)
+		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
 		a.logger.Error(msg)
 		return nil
 	}

cmd/tailscale/cli/cli_test.go

@@ -24,6 +24,7 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 	"tailscale.com/version/distro"
@@ -176,9 +177,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "bare_up_means_up",
 			flags: []string{},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         false,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -186,11 +188,12 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "losing_hostname",
 			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				WantRunning:   false,
-				Hostname:      "foo",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				WantRunning:         false,
+				Hostname:            "foo",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
 		},
@@ -198,10 +201,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "hostname_changing_explicitly",
 			flags: []string{"--hostname=bar"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
-				Hostname:      "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -209,10 +213,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "hostname_changing_empty_explicitly",
 			flags: []string{"--hostname="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
-				Hostname:      "foo",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -228,10 +233,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "implicit_operator_change",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				OperatorUser:  "alice",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				OperatorUser:        "alice",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
 			want:    accidentalUpPrefix + " --hostname=foo --operator=alice",
@@ -240,10 +246,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "implicit_operator_matches_shell_user",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
-				OperatorUser:  "alice",
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "alice",
 			want:    "",
@@ -260,6 +267,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
 		},
@@ -275,6 +283,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -290,6 +299,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -297,9 +307,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "advertise_exit_node", // Issue 1859
 			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -314,6 +325,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -329,6 +341,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -340,7 +353,8 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				CorpDNS:       true,
 				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeID: "fooID",
+				ExitNodeID:          "fooID",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "",
 		},
@@ -362,8 +376,9 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
 			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
@@ -384,8 +399,9 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				AdvertiseRoutes: []netip.Prefix{
 					netip.MustParsePrefix("10.0.0.0/16"),
 				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:       preftype.NetfilterNoDivert,
+				OperatorUser:        "alice",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			curUser: "eve",
 			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --netfilter-mode=nodivert --operator=alice --shields-up",
@@ -394,10 +410,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "loggedout_is_implicit",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				LoggedOut:     true,
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				LoggedOut:           true,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "", // not an error. LoggedOut is implicit.
 		},
@@ -440,6 +457,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --operator=expbits --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
@@ -455,6 +473,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 					netip.MustParsePrefix("::/0"),
 					netip.MustParsePrefix("1.2.0.0/16"),
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --advertise-routes=1.2.0.0/16 --operator=expbits --advertise-exit-node",
 		},
@@ -467,7 +486,8 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				CorpDNS:       true,
 				NetfilterMode: preftype.NetfilterOn,
 
-				Hostname: "foo",
+				Hostname:            "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --auth-key=secretrand --force-reauth=false --reset --hostname=foo",
 		},
@@ -479,7 +499,8 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				CorpDNS:       true,
 				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeIP: netip.MustParseAddr("100.64.5.4"),
+				ExitNodeIP:          netip.MustParseAddr("100.64.5.4"),
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.4",
 		},
@@ -492,7 +513,8 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				CorpDNS:       true,
 				NetfilterMode: preftype.NetfilterOn,
 
-				ExitNodeID: "some_stable_id",
+				ExitNodeID:          "some_stable_id",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
@@ -507,6 +529,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 
 				ExitNodeAllowLANAccess: true,
 				ExitNodeID:             "some_stable_id",
+				NoStatefulFiltering:    opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node-allow-lan-access --exit-node=100.2.3.4",
 		},
@@ -514,9 +537,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_login_server_synonym",
 			flags: []string{"--login-server=https://controlplane.tailscale.com"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: "", // not an error
 		},
@@ -524,9 +548,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "ignore_login_server_synonym_on_other_change",
 			flags: []string{"--netfilter-mode=off"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       false,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             false,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
 		},
@@ -536,10 +561,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "synology_permit_omit_accept_routes",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				RouteAll:      true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				RouteAll:            true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos:   "linux",
 			distro: distro.Synology,
@@ -551,10 +577,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "not_synology_dont_permit_omit_accept_routes",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				RouteAll:      true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				RouteAll:            true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos:   "linux",
 			distro: "", // not Synology
@@ -564,10 +591,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			name:  "profile_name_ignored_in_up",
 			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
-				ProfileName:   "foo",
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				ProfileName:         "foo",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			goos: "linux",
 			want: "",
@@ -630,7 +658,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				ControlURL:          ipn.DefaultControlURL,
 				WantRunning:         true,
 				NoSNAT:              false,
-				NoStatefulFiltering: "false",
+				NoStatefulFiltering: "true",
 				NetfilterMode:       preftype.NetfilterOn,
 				CorpDNS:             true,
 				AutoUpdate: ipn.AutoUpdatePrefs{
@@ -648,7 +676,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				CorpDNS:             true,
 				RouteAll:            true,
 				NoSNAT:              false,
-				NoStatefulFiltering: "false",
+				NoStatefulFiltering: "true",
 				NetfilterMode:       preftype.NetfilterOn,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
@@ -666,7 +694,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 					netip.MustParsePrefix("0.0.0.0/0"),
 					netip.MustParsePrefix("::/0"),
 				},
-				NoStatefulFiltering: "false",
+				NoStatefulFiltering: "true",
 				NetfilterMode:       preftype.NetfilterOn,
 				AutoUpdate: ipn.AutoUpdatePrefs{
 					Check: true,
@@ -1033,10 +1061,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "change_login_server",
 			flags: []string{"--login-server=https://localhost:1000"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env:            upCheckEnv{backendState: "Running"},
 			wantSimpleUp:   true,
@@ -1047,10 +1076,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "change_tags",
 			flags: []string{"--advertise-tags=tag:foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env: upCheckEnv{backendState: "Running"},
 		},
@@ -1059,10 +1089,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "explicit_empty_operator",
 			flags: []string{"--operator="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
-				OperatorUser:  "somebody",
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				OperatorUser:        "somebody",
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env: upCheckEnv{user: "somebody", backendState: "Running"},
 			wantJustEditMP: &ipn.MaskedPrefs{
@@ -1079,10 +1110,11 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "enable_ssh",
 			flags: []string{"--ssh"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1099,11 +1131,12 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "disable_ssh",
 			flags: []string{"--ssh=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				RunSSH:        true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				RunSSH:              true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1123,11 +1156,12 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=false"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
-				RunSSH:        true,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				RunSSH:              true,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1146,10 +1180,11 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=true"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1168,10 +1203,11 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=true", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1189,11 +1225,12 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--ssh=false", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				Persist:       &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
-				CorpDNS:       true,
-				RunSSH:        true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				Persist:             &persist.Persist{UserProfile: tailcfg.UserProfile{LoginName: "crawshaw.github"}},
+				CorpDNS:             true,
+				RunSSH:              true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				RunSSHSet:      true,
@@ -1211,9 +1248,10 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--force-reauth"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			env:          upCheckEnv{backendState: "Running"},
 			wantErrSubtr: "aborted, no changes made",
@@ -1223,9 +1261,10 @@ func TestUpdatePrefs(t *testing.T) {
 			flags:            []string{"--force-reauth", "--accept-risk=lose-ssh"},
 			sshOverTailscale: true,
 			curPrefs: &ipn.Prefs{
-				ControlURL:    "https://login.tailscale.com",
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          "https://login.tailscale.com",
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: nil,
 			env:            upCheckEnv{backendState: "Running"},
@@ -1234,9 +1273,10 @@ func TestUpdatePrefs(t *testing.T) {
 			name:  "advertise_connector",
 			flags: []string{"--advertise-connector"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
-				CorpDNS:       true,
-				NetfilterMode: preftype.NetfilterOn,
+				ControlURL:          ipn.DefaultControlURL,
+				CorpDNS:             true,
+				NetfilterMode:       preftype.NetfilterOn,
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AppConnectorSet: true,
@@ -1259,6 +1299,7 @@ func TestUpdatePrefs(t *testing.T) {
 				AppConnector: ipn.AppConnectorPrefs{
 					Advertise: true,
 				},
+				NoStatefulFiltering: opt.NewBool(true),
 			},
 			wantJustEditMP: &ipn.MaskedPrefs{
 				AppConnectorSet: true,

cmd/tailscale/cli/netcheck.go

@@ -142,7 +142,6 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 		printf("\t* IPv6: no, unavailable in OS\n")
 	}
 	printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
-	printf("\t* HairPinning: %v\n", report.HairPinning)
 	printf("\t* PortMapping: %v\n", portMapping(report))
 	if report.CaptivePortal != "" {
 		printf("\t* CaptivePortal: %v\n", report.CaptivePortal)

cmd/tailscale/cli/network-lock.go

@@ -222,7 +222,8 @@ func runNetworkLockStatus(ctx context.Context, args []string) error {
 
 	if st.Enabled && st.NodeKey != nil && !st.PublicKey.IsZero() {
 		if st.NodeKeySigned {
-			fmt.Println("This node is accessible under tailnet lock.")
+			fmt.Println("This node is accessible under tailnet lock. Node signature:")
+			fmt.Println(st.NodeKeySignature.String())
 		} else {
 			fmt.Println("This node is LOCKED OUT by tailnet-lock, and action is required to establish connectivity.")
 			fmt.Printf("Run the following command on a node with a trusted key:\n\ttailscale lock sign %v %s\n", st.NodeKey, st.PublicKey.CLIString())

cmd/tailscale/cli/set.go

@@ -103,7 +103,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet {
 	switch goos {
 	case "linux":
 		setf.BoolVar(&setArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
-		setf.BoolVar(&setArgs.statefulFiltering, "stateful-filtering", true, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)")
+		setf.BoolVar(&setArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)")
 		setf.StringVar(&setArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
 	case "windows":
 		setf.BoolVar(&setArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")

cmd/tailscale/cli/up.go

@@ -121,7 +121,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet {
 	switch goos {
 	case "linux":
 		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
-		upf.BoolVar(&upArgs.statefulFiltering, "stateful-filtering", true, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)")
+		upf.BoolVar(&upArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)")
 		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
 	case "windows":
 		upf.BoolVar(&upArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")
@@ -885,11 +885,26 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck
 			// Issue 6811. Ignore on Synology.
 			continue
 		}
+		if flagName == "stateful-filtering" && valCur == true && valNew == false && env.goos == "linux" {
+			// See https://github.com/tailscale/tailscale/issues/12307
+			// Stateful filtering was on by default in tailscale 1.66.0-1.66.3, then off in 1.66.4.
+			// This broke Tailscale installations in containerized
+			// environments that use the default containerboot
+			// configuration that configures tailscale using
+			// 'tailscale up' command, which requires that all
+			// previously set flags are explicitly provided on
+			// subsequent restarts.
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
 		return nil
 	}
+
+	// Some previously provided flags are missing. This run of 'tailscale
+	// up' will error out.
+
 	sort.Strings(missing)
 
 	// Compute the stringification of the explicitly provided args in flagSet

cmd/tailscale/depaware.txt

@@ -299,7 +299,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
         runtime/debug                                                from nhooyr.io/websocket/internal/xsync+
-        runtime/trace                                                from testing
         slices                                                       from tailscale.com/client/web+
         sort                                                         from archive/tar+
         strconv                                                      from archive/tar+
@@ -307,7 +306,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         sync                                                         from archive/tar+
         sync/atomic                                                  from context+
         syscall                                                      from archive/tar+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from github.com/peterbourgon/ff/v3/ffcli+
         text/template                                                from html/template
         text/template/parse                                          from html/template+

cmd/tailscale/tailscale_test.go

@@ -12,6 +12,7 @@ import (
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
+			"testing":                            "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/buffer":       "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",

cmd/tailscaled/depaware.txt

@@ -320,6 +320,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/clientupdate/distsign+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
         tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
+        tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/client/tailscale+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
@@ -553,7 +554,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+
         runtime/pprof                                                from net/http/pprof+
-        runtime/trace                                                from net/http/pprof+
+        runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/appc+
         sort                                                         from archive/tar+
         strconv                                                      from archive/tar+
@@ -561,7 +562,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         sync                                                         from archive/tar+
         sync/atomic                                                  from context+
         syscall                                                      from archive/tar+
-        testing                                                      from tailscale.com/util/syspolicy
         text/tabwriter                                               from runtime/pprof
         text/template                                                from html/template
         text/template/parse                                          from html/template+

cmd/tailscaled/tailscaled.go

@@ -118,7 +118,7 @@ var args struct {
 	tunname string
 
 	cleanUp        bool
-	confFile       string
+	confFile       string // empty, file path, or "vm:user-data"
 	debug          string
 	port           uint16
 	statepath      string
@@ -169,7 +169,7 @@ func main() {
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
 	flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support")
-	flag.StringVar(&args.confFile, "config", "", "path to config file")
+	flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2)")
 
 	if len(os.Args) > 0 && filepath.Base(os.Args[0]) == "tailscale" && beCLI != nil {
 		beCLI()
@@ -548,14 +548,25 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 			return ok
 		}
 		dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
-			// Note: don't just return ns.DialContextTCP or we'll
-			// return an interface containing a nil pointer.
+			// Note: don't just return ns.DialContextTCP or we'll return
+			// *gonet.TCPConn(nil) instead of a nil interface which trips up
+			// callers.
 			tcpConn, err := ns.DialContextTCP(ctx, dst)
 			if err != nil {
 				return nil, err
 			}
 			return tcpConn, nil
 		}
+		dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+			// Note: don't just return ns.DialContextUDP or we'll return
+			// *gonet.UDPConn(nil) instead of a nil interface which trips up
+			// callers.
+			udpConn, err := ns.DialContextUDP(ctx, dst)
+			if err != nil {
+				return nil, err
+			}
+			return udpConn, nil
+		}
 	}
 	if socksListener != nil || httpProxyListener != nil {
 		var addrs []string

cmd/tailscaled/tailscaled_test.go

@@ -20,6 +20,7 @@ func TestDeps(t *testing.T) {
 		GOOS:   "darwin",
 		GOARCH: "arm64",
 		BadDeps: map[string]string{
+			"testing":                        "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658",
 		},
 	}.Check(t)
@@ -28,6 +29,7 @@ func TestDeps(t *testing.T) {
 		GOOS:   "linux",
 		GOARCH: "arm64",
 		BadDeps: map[string]string{
+			"testing":                        "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/hostarch": "will crash on non-4K page sizes; see https://github.com/tailscale/tailscale/issues/8658",
 		},
 	}.Check(t)

control/controlclient/direct.go

@@ -558,7 +558,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 
 	var nodeKeySignature tkatype.MarshaledSignature
 	if !oldNodeKey.IsZero() && opt.OldNodeKeySignature != nil {
-		if nodeKeySignature, err = resignNKS(persist.NetworkLockKey, tryingNewKey.Public(), opt.OldNodeKeySignature); err != nil {
+		if nodeKeySignature, err = tka.ResignNKS(persist.NetworkLockKey, tryingNewKey.Public(), opt.OldNodeKeySignature); err != nil {
 			c.logf("Failed re-signing node-key signature: %v", err)
 		}
 	} else if isWrapped {
@@ -729,45 +729,6 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	return false, resp.AuthURL, nil, nil
 }
 
-// resignNKS re-signs a node-key signature for a new node-key.
-//
-// This only matters on network-locked tailnets, because node-key signatures are
-// how other nodes know that a node-key is authentic. When the node-key is
-// rotated then the existing signature becomes invalid, so this function is
-// responsible for generating a new wrapping signature to certify the new node-key.
-//
-// The signature itself is a SigRotation signature, which embeds the old signature
-// and certifies the new node-key as a replacement for the old by signing the new
-// signature with RotationPubkey (which is the node's own network-lock key).
-func resignNKS(priv key.NLPrivate, nodeKey key.NodePublic, oldNKS tkatype.MarshaledSignature) (tkatype.MarshaledSignature, error) {
-	var oldSig tka.NodeKeySignature
-	if err := oldSig.Unserialize(oldNKS); err != nil {
-		return nil, fmt.Errorf("decoding NKS: %w", err)
-	}
-
-	nk, err := nodeKey.MarshalBinary()
-	if err != nil {
-		return nil, fmt.Errorf("marshalling node-key: %w", err)
-	}
-
-	if bytes.Equal(nk, oldSig.Pubkey) {
-		// The old signature is valid for the node-key we are using, so just
-		// use it verbatim.
-		return oldNKS, nil
-	}
-
-	newSig := tka.NodeKeySignature{
-		SigKind: tka.SigRotation,
-		Pubkey:  nk,
-		Nested:  &oldSig,
-	}
-	if newSig.Signature, err = priv.SignNKS(newSig.SigHash()); err != nil {
-		return nil, fmt.Errorf("signing NKS: %w", err)
-	}
-
-	return newSig.Serialize(), nil
-}
-
 // newEndpoints acquires c.mu and sets the local port and endpoints and reports
 // whether they've changed.
 //

derp/derp_server.go

@@ -329,20 +329,36 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
 	s.packetsRecvOther = s.packetsRecvByKind.Get("other")
-	s.packetsDroppedReasonCounters = []*expvar.Int{
-		s.packetsDroppedReason.Get("unknown_dest"),
-		s.packetsDroppedReason.Get("unknown_dest_on_fwd"),
-		s.packetsDroppedReason.Get("gone_disconnected"),
-		s.packetsDroppedReason.Get("gone_not_here"),
-		s.packetsDroppedReason.Get("queue_head"),
-		s.packetsDroppedReason.Get("queue_tail"),
-		s.packetsDroppedReason.Get("write_error"),
-	}
+
+	s.packetsDroppedReasonCounters = s.genPacketsDroppedReasonCounters()
+
 	s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
 	s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
 	return s
 }
 
+func (s *Server) genPacketsDroppedReasonCounters() []*expvar.Int {
+	getMetric := s.packetsDroppedReason.Get
+	ret := []*expvar.Int{
+		dropReasonUnknownDest:      getMetric("unknown_dest"),
+		dropReasonUnknownDestOnFwd: getMetric("unknown_dest_on_fwd"),
+		dropReasonGoneDisconnected: getMetric("gone_disconnected"),
+		dropReasonQueueHead:        getMetric("queue_head"),
+		dropReasonQueueTail:        getMetric("queue_tail"),
+		dropReasonWriteError:       getMetric("write_error"),
+		dropReasonDupClient:        getMetric("dup_client"),
+	}
+	if len(ret) != int(numDropReasons) {
+		panic("dropReason metrics out of sync")
+	}
+	for i := range numDropReasons {
+		if ret[i] == nil {
+			panic("dropReason metrics out of sync")
+		}
+	}
+	return ret
+}
+
 // SetMesh sets the pre-shared key that regional DERP servers used to mesh
 // amongst themselves.
 //
@@ -1047,6 +1063,7 @@ const (
 	dropReasonQueueTail                          // destination queue is full, dropped packet at queue tail
 	dropReasonWriteError                         // OS write() failed
 	dropReasonDupClient                          // the public key is connected 2+ times (active/active, fighting)
+	numDropReasons                               // unused; keep last
 )
 
 func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) {

derp/dropreason_string.go

@@ -18,11 +18,12 @@ func _() {
 	_ = x[dropReasonQueueTail-4]
 	_ = x[dropReasonWriteError-5]
 	_ = x[dropReasonDupClient-6]
+	_ = x[numDropReasons-7]
 }
 
-const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneDisconnectedQueueHeadQueueTailWriteErrorDupClient"
+const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneDisconnectedQueueHeadQueueTailWriteErrorDupClientnumDropReasons"
 
-var _dropReason_index = [...]uint8{0, 11, 27, 43, 52, 61, 71, 80}
+var _dropReason_index = [...]uint8{0, 11, 27, 43, 52, 61, 71, 80, 94}
 
 func (i dropReason) String() string {
 	if i < 0 || i >= dropReason(len(_dropReason_index)-1) {

docs/k8s/subnet.yaml

@@ -29,5 +29,6 @@ spec:
     - name: TS_ROUTES
       value: "{{TS_ROUTES}}"
     securityContext:
-      runAsUser: 1000
-      runAsGroup: 1000
+      capabilities:
+        add:
+        - NET_ADMIN

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.64
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.33.0
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7
+	github.com/bramvdbogaerde/go-scp v1.4.0
 	github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/creack/pty v1.1.21

go.sum

@@ -177,6 +177,8 @@ github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ
 github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k=
 github.com/bombsimon/wsl/v3 v3.4.0 h1:RkSxjT3tmlptwfgEgTgU+KYKLI35p/tviNXNXiL2aNU=
 github.com/bombsimon/wsl/v3 v3.4.0/go.mod h1:KkIB+TXkqy6MvK9BDZVbZxKNYsE1/oLRJbIFtf14qqo=
+github.com/bramvdbogaerde/go-scp v1.4.0 h1:jKMwpwCbcX1KyvDbm/PDJuXcMuNVlLGi0Q0reuzjyKY=
+github.com/bramvdbogaerde/go-scp v1.4.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
 github.com/breml/bidichk v0.2.4 h1:i3yedFWWQ7YzjdZJHnPo9d/xURinSq3OM+gyM43K4/8=
 github.com/breml/bidichk v0.2.4/go.mod h1:7Zk0kRFt1LIZxtQdl9W9JwGAcLTTkOs+tN7wuEYGJ3s=
 github.com/breml/errchkjson v0.3.1 h1:hlIeXuspTyt8Y/UmP5qy1JocGNR00KQHgfaNtRAjoxQ=

ipn/conffile/cloudconf.go

@@ -0,0 +1,59 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package conffile
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"tailscale.com/omit"
+)
+
+func getEC2MetadataToken() (string, error) {
+	if omit.AWS {
+		return "", omit.Err
+	}
+	req, _ := http.NewRequest("PUT", "http://169.254.169.254/latest/api/token", nil)
+	req.Header.Add("X-aws-ec2-metadata-token-ttl-seconds", "300")
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to get metadata token: %w", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return "", fmt.Errorf("failed to get metadata token: %v", res.Status)
+	}
+	all, err := io.ReadAll(res.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read metadata token: %w", err)
+	}
+	return strings.TrimSpace(string(all)), nil
+}
+
+func readVMUserData() ([]byte, error) {
+	// TODO(bradfitz): support GCP, Azure, Proxmox/cloud-init
+	// (NoCloud/ConfigDrive ISO), etc.
+
+	if omit.AWS {
+		return nil, omit.Err
+	}
+	token, tokErr := getEC2MetadataToken()
+	req, _ := http.NewRequest("GET", "http://169.254.169.254/latest/user-data", nil)
+	req.Header.Add("X-aws-ec2-metadata-token", token)
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		if tokErr != nil {
+			return nil, fmt.Errorf("failed to get VM user data: %v; also failed to get metadata token: %v", res.Status, tokErr)
+		}
+		return nil, errors.New(res.Status)
+	}
+	return io.ReadAll(res.Body)
+}

ipn/conffile/conffile.go

@@ -17,7 +17,7 @@ import (
 
 // Config describes a config file.
 type Config struct {
-	Path    string // disk path of HuJSON
+	Path    string // disk path of HuJSON, or VMUserDataPath
 	Raw     []byte // raw bytes from disk, in HuJSON form
 	Std     []byte // standardized JSON form
 	Version string // "alpha0" for now
@@ -35,13 +35,22 @@ func (c *Config) WantRunning() bool {
 	return c != nil && !c.Parsed.Enabled.EqualBool(false)
 }
 
+// VMUserDataPath is a sentinel value for Load to use to get the data
+// from the VM's metadata service's user-data field.
+const VMUserDataPath = "vm:user-data"
+
 // Load reads and parses the config file at the provided path on disk.
 func Load(path string) (*Config, error) {
 	var c Config
 	c.Path = path
-
 	var err error
-	c.Raw, err = os.ReadFile(path)
+
+	switch path {
+	case VMUserDataPath:
+		c.Raw, err = readVMUserData()
+	default:
+		c.Raw, err = os.ReadFile(path)
+	}
 	if err != nil {
 		return nil, err
 	}

ipn/ipn_test.go

@@ -12,6 +12,7 @@ import (
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{
+			"testing":                            "do not use testing package in production code",
 			"gvisor.dev/gvisor/pkg/buffer":       "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",

ipn/ipnlocal/c2n.go

@@ -478,17 +478,44 @@ func findCmdTailscale() (string, error) {
 }
 
 func tailscaleUpdateCmd(cmdTS string) *exec.Cmd {
+	defaultCmd := exec.Command(cmdTS, "update", "--yes")
 	if runtime.GOOS != "linux" {
-		return exec.Command(cmdTS, "update", "--yes")
+		return defaultCmd
 	}
 	if _, err := exec.LookPath("systemd-run"); err != nil {
-		return exec.Command(cmdTS, "update", "--yes")
+		return defaultCmd
 	}
+
 	// When systemd-run is available, use it to run the update command. This
 	// creates a new temporary unit separate from the tailscaled unit. When
 	// tailscaled is restarted during the update, systemd won't kill this
 	// temporary update unit, which could cause unexpected breakage.
-	return exec.Command("systemd-run", "--wait", "--pipe", "--collect", cmdTS, "update", "--yes")
+	//
+	// We want to use the --wait flag for systemd-run, to block the update
+	// command until completion and collect output. But this flag was added in
+	// systemd 232, so we need to check the version first.
+	//
+	// The output will look like:
+	//
+	//   systemd 255 (255.7-1-arch)
+	//   +PAM +AUDIT ... other feature flags ...
+	systemdVerOut, err := exec.Command("systemd-run", "--version").Output()
+	if err != nil {
+		return defaultCmd
+	}
+	parts := strings.Fields(string(systemdVerOut))
+	if len(parts) < 2 || parts[0] != "systemd" {
+		return defaultCmd
+	}
+	systemdVer, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return defaultCmd
+	}
+	if systemdVer < 232 {
+		return exec.Command("systemd-run", "--pipe", "--collect", cmdTS, "update", "--yes")
+	} else {
+		return exec.Command("systemd-run", "--wait", "--pipe", "--collect", cmdTS, "update", "--yes")
+	}
 }
 
 func regularFileExists(path string) bool {

ipn/ipnlocal/local.go

@@ -4186,18 +4186,7 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs ipn.PrefsView, oneC
 	}
 
 	var doStatefulFiltering bool
-	if v, ok := prefs.NoStatefulFiltering().Get(); !ok {
-		// The stateful filtering preference isn't explicitly set; this is
-		// unexpected since we expect it to be set during the profile
-		// backfill, but to be safe let's enable stateful filtering
-		// absent further information.
-		doStatefulFiltering = true
-		b.logf("[unexpected] NoStatefulFiltering preference not set; enabling stateful filtering")
-	} else if v {
-		// The preferences explicitly say "no stateful filtering", so
-		// we don't do it.
-		doStatefulFiltering = false
-	} else {
+	if v, ok := prefs.NoStatefulFiltering().Get(); ok && !v {
 		// The preferences explicitly "do stateful filtering" is turned
 		// off, or to expand the double negative, to do stateful
 		// filtering. Do so.
@@ -6464,8 +6453,17 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, r *rand
 	if report.PreferredDERP == 0 {
 		return res, ErrNoPreferredDERP
 	}
+	var allowedCandidates set.Set[string]
+	if allowed, err := syspolicy.GetStringArray(syspolicy.AllowedSuggestedExitNodes, nil); err != nil {
+		return res, fmt.Errorf("unable to read %s policy: %w", syspolicy.AllowedSuggestedExitNodes, err)
+	} else if allowed != nil && len(allowed) > 0 {
+		allowedCandidates = set.SetOf(allowed)
+	}
 	candidates := make([]tailcfg.NodeView, 0, len(netMap.Peers))
 	for _, peer := range netMap.Peers {
+		if allowedCandidates != nil && !allowedCandidates.Contains(string(peer.StableID())) {
+			continue
+		}
 		if peer.CapMap().Has(tailcfg.NodeAttrSuggestExitNode) && tsaddr.ContainsExitRoutes(peer.AllowedIPs()) {
 			candidates = append(candidates, peer)
 		}

ipn/ipnlocal/local_test.go

@@ -1595,6 +1595,9 @@ type mockSyspolicyHandler struct {
 	// queried by the current test. If the policy is expected but unset, then
 	// use nil, otherwise use a string equal to the policy's desired value.
 	stringPolicies map[syspolicy.Key]*string
+	// stringArrayPolicies is the collection of policies that we expected to see
+	// queries by the current test, that return policy string arrays.
+	stringArrayPolicies map[syspolicy.Key][]string
 	// failUnknownPolicies is set if policies other than those in stringPolicies
 	// (uint64 or bool policies are not supported by mockSyspolicyHandler yet)
 	// should be considered a test failure if they are queried.
@@ -1632,6 +1635,12 @@ func (h *mockSyspolicyHandler) ReadStringArray(key string) ([]string, error) {
 	if h.failUnknownPolicies {
 		h.t.Errorf("ReadStringArray(%q) unexpectedly called", key)
 	}
+	if s, ok := h.stringArrayPolicies[syspolicy.Key(key)]; ok {
+		if s == nil {
+			return []string{}, syspolicy.ErrNoSuchKey
+		}
+		return s, nil
+	}
 	return nil, syspolicy.ErrNoSuchKey
 }
 
@@ -3474,6 +3483,7 @@ func TestLocalBackendSuggestExitNode(t *testing.T) {
 		lastSuggestedExitNode     lastSuggestedExitNode
 		report                    *netcheck.Report
 		netMap                    netmap.NetworkMap
+		allowedSuggestedExitNodes []string
 		wantID                    tailcfg.StableNodeID
 		wantName                  string
 		wantErr                   error
@@ -3766,10 +3776,138 @@ func TestLocalBackendSuggestExitNode(t *testing.T) {
 			},
 			wantErr: ErrCannotSuggestExitNode,
 		},
+		{
+			name:                  "only pick from allowed suggested exit nodes",
+			lastSuggestedExitNode: lastSuggestedExitNode{name: "test", id: "test"},
+			report: &netcheck.Report{
+				RegionLatency: map[int]time.Duration{
+					1: 10,
+					2: 10,
+					3: 5,
+				},
+				PreferredDERP: 1,
+			},
+			netMap: netmap.NetworkMap{
+				SelfNode: (&tailcfg.Node{
+					Addresses: []netip.Prefix{
+						netip.MustParsePrefix("100.64.1.1/32"),
+						netip.MustParsePrefix("fe70::1/128"),
+					},
+				}).View(),
+				DERPMap: &tailcfg.DERPMap{
+					Regions: map[int]*tailcfg.DERPRegion{
+						1: {},
+						2: {},
+						3: {},
+					},
+				},
+				Peers: []tailcfg.NodeView{
+					(&tailcfg.Node{
+						ID:       2,
+						StableID: "test",
+						Name:     "test",
+						DERP:     "127.3.3.40:1",
+						AllowedIPs: []netip.Prefix{
+							netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
+						},
+						CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+							tailcfg.NodeAttrSuggestExitNode: {},
+							tailcfg.NodeAttrAutoExitNode:    {},
+						}),
+					}).View(),
+					(&tailcfg.Node{
+						ID:       3,
+						StableID: "foo",
+						Name:     "foo",
+						DERP:     "127.3.3.40:3",
+						AllowedIPs: []netip.Prefix{
+							netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
+						},
+						CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+							tailcfg.NodeAttrSuggestExitNode: {},
+							tailcfg.NodeAttrAutoExitNode:    {},
+						}),
+					}).View(),
+				},
+			},
+			allowedSuggestedExitNodes: []string{"test"},
+			wantID:                    "test",
+			wantName:                  "test",
+			wantLastSuggestedExitNode: lastSuggestedExitNode{name: "test", id: "test"},
+		},
+		{
+			name:                  "allowed suggested exit nodes not nil but length 0",
+			lastSuggestedExitNode: lastSuggestedExitNode{name: "test", id: "test"},
+			report: &netcheck.Report{
+				RegionLatency: map[int]time.Duration{
+					1: 10,
+					2: 10,
+					3: 5,
+				},
+				PreferredDERP: 1,
+			},
+			netMap: netmap.NetworkMap{
+				SelfNode: (&tailcfg.Node{
+					Addresses: []netip.Prefix{
+						netip.MustParsePrefix("100.64.1.1/32"),
+						netip.MustParsePrefix("fe70::1/128"),
+					},
+				}).View(),
+				DERPMap: &tailcfg.DERPMap{
+					Regions: map[int]*tailcfg.DERPRegion{
+						1: {},
+						2: {},
+						3: {},
+					},
+				},
+				Peers: []tailcfg.NodeView{
+					(&tailcfg.Node{
+						ID:       2,
+						StableID: "test",
+						Name:     "test",
+						DERP:     "127.3.3.40:1",
+						AllowedIPs: []netip.Prefix{
+							netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
+						},
+						CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+							tailcfg.NodeAttrSuggestExitNode: {},
+							tailcfg.NodeAttrAutoExitNode:    {},
+						}),
+					}).View(),
+					(&tailcfg.Node{
+						ID:       3,
+						StableID: "foo",
+						Name:     "foo",
+						DERP:     "127.3.3.40:3",
+						AllowedIPs: []netip.Prefix{
+							netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
+						},
+						CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+							tailcfg.NodeAttrSuggestExitNode: {},
+							tailcfg.NodeAttrAutoExitNode:    {},
+						}),
+					}).View(),
+				},
+			},
+			allowedSuggestedExitNodes: []string{},
+			wantID:                    "foo",
+			wantName:                  "foo",
+			wantLastSuggestedExitNode: lastSuggestedExitNode{name: "foo", id: "foo"},
+		},
 	}
 
 	for _, tt := range tests {
 		lb := newTestLocalBackend(t)
+		msh := &mockSyspolicyHandler{
+			t: t,
+			stringArrayPolicies: map[syspolicy.Key][]string{
+				syspolicy.AllowedSuggestedExitNodes: nil,
+			},
+		}
+		if len(tt.allowedSuggestedExitNodes) != 0 {
+			msh.stringArrayPolicies[syspolicy.AllowedSuggestedExitNodes] = tt.allowedSuggestedExitNodes
+		}
+		syspolicy.SetHandlerForTest(t, msh)
 		lb.lastSuggestedExitNode = tt.lastSuggestedExitNode
 		lb.netMap = &tt.netMap
 		lb.sys.MagicSock.Get().SetLastNetcheckReportForTest(context.Background(), tt.report)

ipn/ipnlocal/network-lock.go

@@ -18,6 +18,7 @@ import (
 	"net/netip"
 	"os"
 	"path/filepath"
+	"slices"
 	"time"
 
 	"tailscale.com/health/healthmsg"
@@ -27,10 +28,12 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/tkatype"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
 )
 
 // TODO(tom): RPC retry/backoff was broken and has been removed. Fix?
@@ -66,6 +69,7 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 		return // TKA not enabled.
 	}
 
+	tracker := rotationTracker{logf: b.logf}
 	var toDelete map[int]bool // peer index => true
 	for i, p := range nm.Peers {
 		if p.UnsignedPeerAPIOnly() {
@@ -76,21 +80,32 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 			b.logf("Network lock is dropping peer %v(%v) due to missing signature", p.ID(), p.StableID())
 			mak.Set(&toDelete, i, true)
 		} else {
-			if err := b.tka.authority.NodeKeyAuthorized(p.Key(), p.KeySignature().AsSlice()); err != nil {
+			details, err := b.tka.authority.NodeKeyAuthorizedWithDetails(p.Key(), p.KeySignature().AsSlice())
+			if err != nil {
 				b.logf("Network lock is dropping peer %v(%v) due to failed signature check: %v", p.ID(), p.StableID(), err)
 				mak.Set(&toDelete, i, true)
+				continue
+			}
+			if details != nil {
+				// Rotation details are returned when the node key is signed by a valid SigRotation signature.
+				tracker.addRotationDetails(p.Key(), details)
 			}
 		}
 	}
 
+	obsoleteByRotation := tracker.obsoleteKeys()
+
 	// nm.Peers is ordered, so deletion must be order-preserving.
-	if len(toDelete) > 0 {
+	if len(toDelete) > 0 || len(obsoleteByRotation) > 0 {
 		peers := make([]tailcfg.NodeView, 0, len(nm.Peers))
-		filtered := make([]ipnstate.TKAFilteredPeer, 0, len(toDelete))
+		filtered := make([]ipnstate.TKAFilteredPeer, 0, len(toDelete)+len(obsoleteByRotation))
 		for i, p := range nm.Peers {
-			if !toDelete[i] {
+			if !toDelete[i] && !obsoleteByRotation.Contains(p.Key()) {
 				peers = append(peers, p)
 			} else {
+				if obsoleteByRotation.Contains(p.Key()) {
+					b.logf("Network lock is dropping peer %v(%v) due to key rotation", p.ID(), p.StableID())
+				}
 				// Record information about the node we filtered out.
 				fp := ipnstate.TKAFilteredPeer{
 					Name:         p.Name(),
@@ -122,6 +137,84 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 	}
 }
 
+// rotationTracker determines the set of node keys that are made obsolete by key
+// rotation.
+//   - for each SigRotation signature, all previous node keys referenced by the
+//     nested signatures are marked as obsolete.
+//   - if there are multiple SigRotation signatures tracing back to the same
+//     wrapping pubkey (e.g. if a node is cloned with all its keys), we keep
+//     just one of them, marking the others as obsolete.
+type rotationTracker struct {
+	// obsolete is the set of node keys that are obsolete due to key rotation.
+	// users of rotationTracker should use the obsoleteKeys method for complete results.
+	obsolete set.Set[key.NodePublic]
+
+	// byWrappingKey keeps track of rotation details per wrapping pubkey.
+	byWrappingKey map[string][]sigRotationDetails
+
+	logf logger.Logf
+}
+
+// sigRotationDetails holds information about a node key signed by a SigRotation.
+type sigRotationDetails struct {
+	np          key.NodePublic
+	numPrevKeys int
+}
+
+// addRotationDetails records the rotation signature details for a node key.
+func (r *rotationTracker) addRotationDetails(np key.NodePublic, d *tka.RotationDetails) {
+	r.obsolete.Make()
+	r.obsolete.AddSlice(d.PrevNodeKeys)
+	rd := sigRotationDetails{
+		np:          np,
+		numPrevKeys: len(d.PrevNodeKeys),
+	}
+	if r.byWrappingKey == nil {
+		r.byWrappingKey = make(map[string][]sigRotationDetails)
+	}
+	wp := string(d.WrappingPubkey)
+	r.byWrappingKey[wp] = append(r.byWrappingKey[wp], rd)
+}
+
+// obsoleteKeys returns the set of node keys that are obsolete due to key rotation.
+func (r *rotationTracker) obsoleteKeys() set.Set[key.NodePublic] {
+	for _, v := range r.byWrappingKey {
+		// If there are multiple rotation signatures with the same wrapping
+		// pubkey, we need to decide which one is the "latest", and keep it.
+		// The signature with the largest number of previous keys is likely to
+		// be the latest, unless it has been marked as obsolete (rotated out) by
+		// another signature (which might happen in the future if we start
+		// compacting long rotated signature chains).
+		slices.SortStableFunc(v, func(a, b sigRotationDetails) int {
+			// Group all obsolete keys after non-obsolete keys.
+			if ao, bo := r.obsolete.Contains(a.np), r.obsolete.Contains(b.np); ao != bo {
+				if ao {
+					return 1
+				}
+				return -1
+			}
+			// Sort by decreasing number of previous keys.
+			return b.numPrevKeys - a.numPrevKeys
+		})
+		// If there are several signatures with the same number of previous
+		// keys, we cannot determine which one is the latest, so all of them are
+		// rejected for safety.
+		if len(v) >= 2 && v[0].numPrevKeys == v[1].numPrevKeys {
+			r.logf("at least two nodes (%s and %s) have equally valid rotation signatures with the same wrapping pubkey, rejecting", v[0].np, v[1].np)
+			for _, rd := range v {
+				r.obsolete.Add(rd.np)
+			}
+		} else {
+			// The first key in v is the one with the longest chain of previous
+			// keys, so it must be the newest one. Mark all older keys as obsolete.
+			for _, rd := range v[1:] {
+				r.obsolete.Add(rd.np)
+			}
+		}
+	}
+	return r.obsolete
+}
+
 // tkaSyncIfNeeded examines TKA info reported from the control plane,
 // performing the steps necessary to synchronize local tka state.
 //
@@ -423,8 +516,12 @@ func (b *LocalBackend) NetworkLockStatus() *ipnstate.NetworkLockStatus {
 	copy(head[:], h[:])
 
 	var selfAuthorized bool
+	nodeKeySignature := &tka.NodeKeySignature{}
 	if b.netMap != nil {
 		selfAuthorized = b.tka.authority.NodeKeyAuthorized(b.netMap.SelfNode.Key(), b.netMap.SelfNode.KeySignature().AsSlice()) == nil
+		if err := nodeKeySignature.Unserialize(b.netMap.SelfNode.KeySignature().AsSlice()); err != nil {
+			b.logf("failed to decode self node key signature: %v", err)
+		}
 	}
 
 	keys := b.tka.authority.Keys()
@@ -445,14 +542,15 @@ func (b *LocalBackend) NetworkLockStatus() *ipnstate.NetworkLockStatus {
 	stateID1, _ := b.tka.authority.StateIDs()
 
 	return &ipnstate.NetworkLockStatus{
-		Enabled:       true,
-		Head:          &head,
-		PublicKey:     nlPriv.Public(),
-		NodeKey:       nodeKey,
-		NodeKeySigned: selfAuthorized,
-		TrustedKeys:   outKeys,
-		FilteredPeers: filtered,
-		StateID:       stateID1,
+		Enabled:          true,
+		Head:             &head,
+		PublicKey:        nlPriv.Public(),
+		NodeKey:          nodeKey,
+		NodeKeySigned:    selfAuthorized,
+		NodeKeySignature: nodeKeySignature,
+		TrustedKeys:      outKeys,
+		FilteredPeers:    filtered,
+		StateID:          stateID1,
 	}
 }
 

ipn/ipnlocal/network-lock_test.go

@@ -13,8 +13,11 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 
+	go4mem "go4.org/mem"
+
 	"github.com/google/go-cmp/cmp"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
@@ -30,6 +33,7 @@ import (
 	"tailscale.com/types/persist"
 	"tailscale.com/types/tkatype"
 	"tailscale.com/util/must"
+	"tailscale.com/util/set"
 )
 
 type observerFunc func(controlclient.Status)
@@ -563,18 +567,32 @@ func TestTKAFilterNetmap(t *testing.T) {
 	}
 	n4Sig.Signature[3] = 42 // mess up the signature
 	n4Sig.Signature[4] = 42 // mess up the signature
-	n5GoodSig, err := signNodeKey(tailcfg.TKASignInfo{NodePublic: n5.Public()}, nlPriv)
+
+	n5nl := key.NewNLPrivate()
+	n5InitialSig, err := signNodeKey(tailcfg.TKASignInfo{NodePublic: n5.Public(), RotationPubkey: n5nl.Public().Verifier()}, nlPriv)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	resign := func(nl key.NLPrivate, currentSig tkatype.MarshaledSignature) (key.NodePrivate, tkatype.MarshaledSignature) {
+		nk := key.NewNode()
+		sig, err := tka.ResignNKS(nl, nk.Public(), currentSig)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return nk, sig
+	}
+
+	n5Rotated, n5RotatedSig := resign(n5nl, n5InitialSig.Serialize())
+
 	nm := &netmap.NetworkMap{
 		Peers: nodeViews([]*tailcfg.Node{
 			{ID: 1, Key: n1.Public(), KeySignature: n1GoodSig.Serialize()},
-			{ID: 2, Key: n2.Public(), KeySignature: nil},                   // missing sig
-			{ID: 3, Key: n3.Public(), KeySignature: n1GoodSig.Serialize()}, // someone elses sig
-			{ID: 4, Key: n4.Public(), KeySignature: n4Sig.Serialize()},     // messed-up signature
-			{ID: 5, Key: n5.Public(), KeySignature: n5GoodSig.Serialize()},
+			{ID: 2, Key: n2.Public(), KeySignature: nil},                       // missing sig
+			{ID: 3, Key: n3.Public(), KeySignature: n1GoodSig.Serialize()},     // someone elses sig
+			{ID: 4, Key: n4.Public(), KeySignature: n4Sig.Serialize()},         // messed-up signature
+			{ID: 50, Key: n5.Public(), KeySignature: n5InitialSig.Serialize()}, // rotated
+			{ID: 51, Key: n5Rotated.Public(), KeySignature: n5RotatedSig},
 		}),
 	}
 
@@ -586,12 +604,39 @@ func TestTKAFilterNetmap(t *testing.T) {
 
 	want := nodeViews([]*tailcfg.Node{
 		{ID: 1, Key: n1.Public(), KeySignature: n1GoodSig.Serialize()},
-		{ID: 5, Key: n5.Public(), KeySignature: n5GoodSig.Serialize()},
+		{ID: 51, Key: n5Rotated.Public(), KeySignature: n5RotatedSig},
 	})
 	nodePubComparer := cmp.Comparer(func(x, y key.NodePublic) bool {
 		return x.Raw32() == y.Raw32()
 	})
-	if diff := cmp.Diff(nm.Peers, want, nodePubComparer); diff != "" {
+	if diff := cmp.Diff(want, nm.Peers, nodePubComparer); diff != "" {
+		t.Errorf("filtered netmap differs (-want, +got):\n%s", diff)
+	}
+
+	// Create two more node signatures using the same wrapping key as n5.
+	// Since they have the same rotation chain, both will be filtered out.
+	n7, n7Sig := resign(n5nl, n5RotatedSig)
+	n8, n8Sig := resign(n5nl, n5RotatedSig)
+
+	nm = &netmap.NetworkMap{
+		Peers: nodeViews([]*tailcfg.Node{
+			{ID: 1, Key: n1.Public(), KeySignature: n1GoodSig.Serialize()},
+			{ID: 2, Key: n2.Public(), KeySignature: nil},                       // missing sig
+			{ID: 3, Key: n3.Public(), KeySignature: n1GoodSig.Serialize()},     // someone elses sig
+			{ID: 4, Key: n4.Public(), KeySignature: n4Sig.Serialize()},         // messed-up signature
+			{ID: 50, Key: n5.Public(), KeySignature: n5InitialSig.Serialize()}, // rotated
+			{ID: 51, Key: n5Rotated.Public(), KeySignature: n5RotatedSig},      // rotated
+			{ID: 7, Key: n7.Public(), KeySignature: n7Sig},                     // same rotation chain as n8
+			{ID: 8, Key: n8.Public(), KeySignature: n8Sig},                     // same rotation chain as n7
+		}),
+	}
+
+	b.tkaFilterNetmapLocked(nm)
+
+	want = nodeViews([]*tailcfg.Node{
+		{ID: 1, Key: n1.Public(), KeySignature: n1GoodSig.Serialize()},
+	})
+	if diff := cmp.Diff(want, nm.Peers, nodePubComparer); diff != "" {
 		t.Errorf("filtered netmap differs (-want, +got):\n%s", diff)
 	}
 }
@@ -1130,3 +1175,85 @@ func TestTKARecoverCompromisedKeyFlow(t *testing.T) {
 		t.Errorf("NetworkLockSubmitRecoveryAUM() failed: %v", err)
 	}
 }
+
+func TestRotationTracker(t *testing.T) {
+	newNK := func(idx byte) key.NodePublic {
+		// single-byte public key to make it human-readable in tests.
+		raw32 := [32]byte{idx}
+		return key.NodePublicFromRaw32(go4mem.B(raw32[:]))
+	}
+	n1, n2, n3, n4, n5 := newNK(1), newNK(2), newNK(3), newNK(4), newNK(5)
+
+	pk1, pk2, pk3 := []byte{1}, []byte{2}, []byte{3}
+	type addDetails struct {
+		np      key.NodePublic
+		details *tka.RotationDetails
+	}
+	tests := []struct {
+		name       string
+		addDetails []addDetails
+		want       set.Set[key.NodePublic]
+	}{
+		{
+			name: "empty",
+			want: nil,
+		},
+		{
+			name: "single_prev_key",
+			addDetails: []addDetails{
+				{np: n1, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n2}, WrappingPubkey: pk1}},
+			},
+			want: set.SetOf([]key.NodePublic{n2}),
+		},
+		{
+			name: "several_prev_keys",
+			addDetails: []addDetails{
+				{np: n1, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n2}, WrappingPubkey: pk1}},
+				{np: n3, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n4}, WrappingPubkey: pk2}},
+				{np: n2, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n3, n4}, WrappingPubkey: pk1}},
+			},
+			want: set.SetOf([]key.NodePublic{n2, n3, n4}),
+		},
+		{
+			name: "several_per_pubkey_latest_wins",
+			addDetails: []addDetails{
+				{np: n2, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1}, WrappingPubkey: pk3}},
+				{np: n3, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2}, WrappingPubkey: pk3}},
+				{np: n4, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2, n3}, WrappingPubkey: pk3}},
+				{np: n5, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n4}, WrappingPubkey: pk3}},
+			},
+			want: set.SetOf([]key.NodePublic{n1, n2, n3, n4}),
+		},
+		{
+			name: "several_per_pubkey_same_chain_length_all_rejected",
+			addDetails: []addDetails{
+				{np: n2, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1}, WrappingPubkey: pk3}},
+				{np: n3, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2}, WrappingPubkey: pk3}},
+				{np: n4, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2}, WrappingPubkey: pk3}},
+				{np: n5, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2}, WrappingPubkey: pk3}},
+			},
+			want: set.SetOf([]key.NodePublic{n1, n2, n3, n4, n5}),
+		},
+		{
+			name: "several_per_pubkey_longest_wins",
+			addDetails: []addDetails{
+				{np: n2, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1}, WrappingPubkey: pk3}},
+				{np: n3, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2}, WrappingPubkey: pk3}},
+				{np: n4, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2}, WrappingPubkey: pk3}},
+				{np: n5, details: &tka.RotationDetails{PrevNodeKeys: []key.NodePublic{n1, n2, n3}, WrappingPubkey: pk3}},
+			},
+			want: set.SetOf([]key.NodePublic{n1, n2, n3, n4}),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &rotationTracker{logf: t.Logf}
+			for _, ad := range tt.addDetails {
+				r.addRotationDetails(ad.np, ad.details)
+			}
+			if got := r.obsoleteKeys(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("rotationTracker.obsoleteKeys() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

ipn/ipnlocal/profiles.go

@@ -354,10 +354,6 @@ func (pm *profileManager) loadSavedPrefs(key ipn.StateKey) (ipn.PrefsView, error
 		return ipn.PrefsView{}, err
 	}
 	savedPrefs := ipn.NewPrefs()
-	// NewPrefs sets a default NoStatefulFiltering, but we want to actually see
-	// if the saved state had an empty value. The empty value gets migrated
-	// based on NoSNAT, while a default "false" does not.
-	savedPrefs.NoStatefulFiltering = ""
 	if err := ipn.PrefsFromBytes(bs, savedPrefs); err != nil {
 		return ipn.PrefsView{}, fmt.Errorf("parsing saved prefs: %v", err)
 	}
@@ -382,32 +378,6 @@ func (pm *profileManager) loadSavedPrefs(key ipn.StateKey) (ipn.PrefsView, error
 		savedPrefs.AutoUpdate.Apply.Clear()
 	}
 
-	// Backfill a missing NoStatefulFiltering field based on the value of
-	// the NoSNAT field; we want to apply stateful filtering in all cases
-	// *except* where the user has disabled SNAT.
-	//
-	// Only backfill if the user hasn't set a value for
-	// NoStatefulFiltering, however.
-	_, haveNoStateful := savedPrefs.NoStatefulFiltering.Get()
-	if !haveNoStateful {
-		if savedPrefs.NoSNAT {
-			pm.logf("backfilling NoStatefulFiltering field to true because NoSNAT is set")
-
-			// No SNAT: no stateful filtering
-			savedPrefs.NoStatefulFiltering.Set(true)
-		} else {
-			pm.logf("backfilling NoStatefulFiltering field to false because NoSNAT is not set")
-
-			// SNAT (default): apply stateful filtering
-			savedPrefs.NoStatefulFiltering.Set(false)
-		}
-
-		// Write back to the preferences store now that we've updated it.
-		if err := pm.writePrefsToStore(key, savedPrefs.View()); err != nil {
-			return ipn.PrefsView{}, err
-		}
-	}
-
 	return savedPrefs.View(), nil
 }
 

ipn/ipnlocal/profiles_test.go

@@ -4,7 +4,6 @@
 package ipnlocal
 
 import (
-	"encoding/json"
 	"fmt"
 	"os/user"
 	"strconv"
@@ -13,14 +12,12 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"tailscale.com/clientupdate"
-	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
 	"tailscale.com/util/must"
 )
@@ -604,89 +601,6 @@ func TestProfileManagementWindows(t *testing.T) {
 	}
 }
 
-func TestProfileBackfillStatefulFiltering(t *testing.T) {
-	envknob.Setenv("TS_DEBUG_PROFILES", "true")
-
-	tests := []struct {
-		noSNAT     bool
-		noStateful opt.Bool
-		want       bool
-	}{
-		// Default: NoSNAT is false, NoStatefulFiltering is false, so
-		// we want it to stay false.
-		{false, "false", false},
-
-		// NoSNAT being set to true and NoStatefulFiltering being false
-		// should result in NoStatefulFiltering still being false,
-		// since it was explicitly set.
-		{true, "false", false},
-
-		// If NoSNAT is false, and NoStatefulFiltering is unset, we
-		// backfill it to 'false'.
-		{false, "", false},
-
-		// If NoSNAT is true, and NoStatefulFiltering is unset, we
-		// backfill to 'true' to not break users of NoSNAT.
-		//
-		// In other words: if the user is not using SNAT, they almost
-		// certainly also don't want to use stateful filtering.
-		{true, "", true},
-
-		// However, if the user specifies both NoSNAT and stateful
-		// filtering, don't change that.
-		{true, "true", true},
-		{false, "true", true},
-	}
-
-	for _, tt := range tests {
-		t.Run(fmt.Sprintf("noSNAT=%v,noStateful=%q", tt.noSNAT, tt.noStateful), func(t *testing.T) {
-			prefs := ipn.NewPrefs()
-			prefs.Persist = &persist.Persist{
-				NodeID: tailcfg.StableNodeID("node1"),
-				UserProfile: tailcfg.UserProfile{
-					ID:        tailcfg.UserID(1),
-					LoginName: "user1@example.com",
-				},
-			}
-
-			prefs.NoSNAT = tt.noSNAT
-			prefs.NoStatefulFiltering = tt.noStateful
-
-			// Make enough of a state store to load the prefs.
-			const profileName = "profile1"
-			bn := must.Get(json.Marshal(map[string]any{
-				string(ipn.CurrentProfileStateKey): []byte(profileName),
-				string(ipn.KnownProfilesStateKey): must.Get(json.Marshal(map[ipn.ProfileID]*ipn.LoginProfile{
-					profileName: {
-						ID:  "profile1-id",
-						Key: profileName,
-					},
-				})),
-				profileName: prefs.ToBytes(),
-			}))
-
-			store := new(mem.Store)
-			err := store.LoadFromJSON([]byte(bn))
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			ht := new(health.Tracker)
-			pm, err := newProfileManagerWithGOOS(store, t.Logf, ht, "linux")
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			// Get the current profile and verify that we backfilled our
-			// StatefulFiltering boolean.
-			pf := pm.CurrentPrefs()
-			if !pf.NoStatefulFiltering().EqualBool(tt.want) {
-				t.Fatalf("got NoStatefulFiltering=%q, want %v", pf.NoStatefulFiltering(), tt.want)
-			}
-		})
-	}
-}
-
 // TestDefaultPrefs tests that defaultPrefs is just NewPrefs with
 // LoggedOut=true (the Prefs we use before connecting to control). We shouldn't
 // be putting any defaulting there, and instead put all defaults in NewPrefs.

ipn/ipnstate/ipnstate.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"tailscale.com/tailcfg"
+	"tailscale.com/tka"
 	"tailscale.com/types/key"
 	"tailscale.com/types/ptr"
 	"tailscale.com/types/views"
@@ -126,6 +127,9 @@ type NetworkLockStatus struct {
 	// NodeKeySigned is true if our node is authorized by network-lock.
 	NodeKeySigned bool
 
+	// NodeKeySignature is the current signature of this node's key.
+	NodeKeySignature *tka.NodeKeySignature
+
 	// TrustedKeys describes the keys currently trusted to make changes
 	// to network-lock.
 	TrustedKeys []TKAKey

ipn/localapi/localapi.go

@@ -6,6 +6,7 @@ package localapi
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -1939,8 +1940,10 @@ func (h *Handler) serveDial(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	network := cmp.Or(r.Header.Get("Dial-Network"), "tcp")
+
 	addr := net.JoinHostPort(hostStr, portStr)
-	outConn, err := h.b.Dialer().UserDial(r.Context(), "tcp", addr)
+	outConn, err := h.b.Dialer().UserDial(r.Context(), network, addr)
 	if err != nil {
 		http.Error(w, "dial failure: "+err.Error(), http.StatusBadGateway)
 		return

ipn/prefs.go

@@ -191,17 +191,16 @@ type Prefs struct {
 	// Linux-only.
 	NoSNAT bool
 
-	// NoStatefulFiltering specifies whether to apply stateful filtering
-	// when advertising routes in AdvertiseRoutes. The default is to apply
+	// NoStatefulFiltering specifies whether to apply stateful filtering when
+	// advertising routes in AdvertiseRoutes. The default is to not apply
 	// stateful filtering.
 	//
 	// To allow inbound connections from advertised routes, both NoSNAT and
 	// NoStatefulFiltering must be true.
 	//
-	// This is an opt.Bool because it was added after NoSNAT, but is backfilled
-	// based on the value of that parameter. We need to treat it as a tristate:
-	// true, false, or unset, and backfill based on that value. See
-	// ipn/ipnlocal for more details on the backfill.
+	// This is an opt.Bool because it was first added after NoSNAT, with a
+	// backfill based on the value of that parameter. The backfill has been
+	// removed since then, but the field remains an opt.Bool.
 	//
 	// Linux-only.
 	NoStatefulFiltering opt.Bool `json:",omitempty"`
@@ -666,7 +665,7 @@ func NewPrefs() *Prefs {
 		CorpDNS:             true,
 		WantRunning:         false,
 		NetfilterMode:       preftype.NetfilterOn,
-		NoStatefulFiltering: opt.NewBool(false),
+		NoStatefulFiltering: opt.NewBool(true),
 		AutoUpdate: AutoUpdatePrefs{
 			Check: true,
 			Apply: opt.Bool("unset"),

ipn/serve.go

@@ -626,7 +626,7 @@ func (v ServeConfigView) HasAllowFunnel() bool {
 	}()
 }
 
-// FindFunnel reports whether target exists in in either the background AllowFunnel
+// FindFunnel reports whether target exists in either the background AllowFunnel
 // or any of the foreground configs.
 func (v ServeConfigView) HasFunnelForTarget(target HostPort) bool {
 	if v.AllowFunnel().Get(target) {

ipn/store/kubestore/store_kube.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 	"time"
 
@@ -30,6 +31,10 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 	if err != nil {
 		return nil, err
 	}
+	if os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
+		// Derive the API server address from the environment variables
+		c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	}
 	canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
 	if err != nil {
 		return nil, err

licenses/android.md

@@ -35,7 +35,7 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/golang/groupcache/lru](https://pkg.go.dev/github.com/golang/groupcache/lru) ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
  - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.1.2/LICENSE))
  - [github.com/google/nftables](https://pkg.go.dev/github.com/google/nftables) ([Apache-2.0](https://github.com/google/nftables/blob/5e242ec57806/LICENSE))
- - [github.com/google/uuid](https://pkg.go.dev/github.com/google/uuid) ([BSD-3-Clause](https://github.com/google/uuid/blob/v1.5.0/LICENSE))
+ - [github.com/google/uuid](https://pkg.go.dev/github.com/google/uuid) ([BSD-3-Clause](https://github.com/google/uuid/blob/v1.6.0/LICENSE))
  - [github.com/hdevalence/ed25519consensus](https://pkg.go.dev/github.com/hdevalence/ed25519consensus) ([BSD-3-Clause](https://github.com/hdevalence/ed25519consensus/blob/v0.2.0/LICENSE))
  - [github.com/illarion/gonotify](https://pkg.go.dev/github.com/illarion/gonotify) ([MIT](https://github.com/illarion/gonotify/blob/v1.0.1/LICENSE))
  - [github.com/insomniacslk/dhcp](https://pkg.go.dev/github.com/insomniacslk/dhcp) ([BSD-3-Clause](https://github.com/insomniacslk/dhcp/blob/8c70d406f6d2/LICENSE))

licenses/apple.md

@@ -58,6 +58,7 @@ See also the dependencies in the [Tailscale CLI][].
  - [github.com/miekg/dns](https://pkg.go.dev/github.com/miekg/dns) ([BSD-3-Clause](https://github.com/miekg/dns/blob/v1.1.58/LICENSE))
  - [github.com/mitchellh/go-ps](https://pkg.go.dev/github.com/mitchellh/go-ps) ([MIT](https://github.com/mitchellh/go-ps/blob/v1.0.0/LICENSE.md))
  - [github.com/pierrec/lz4/v4](https://pkg.go.dev/github.com/pierrec/lz4/v4) ([BSD-3-Clause](https://github.com/pierrec/lz4/blob/v4.1.21/LICENSE))
+ - [github.com/prometheus-community/pro-bing](https://pkg.go.dev/github.com/prometheus-community/pro-bing) ([MIT](https://github.com/prometheus-community/pro-bing/blob/v0.4.0/LICENSE))
  - [github.com/safchain/ethtool](https://pkg.go.dev/github.com/safchain/ethtool) ([Apache-2.0](https://github.com/safchain/ethtool/blob/v0.3.0/LICENSE))
  - [github.com/tailscale/golang-x-crypto](https://pkg.go.dev/github.com/tailscale/golang-x-crypto) ([BSD-3-Clause](https://github.com/tailscale/golang-x-crypto/blob/7ce1f622c780/LICENSE))
  - [github.com/tailscale/goupnp](https://pkg.go.dev/github.com/tailscale/goupnp) ([BSD-2-Clause](https://github.com/tailscale/goupnp/blob/c64d0f06ea05/LICENSE))

licenses/tailscale.md

@@ -47,7 +47,7 @@ Some packages may only be included on certain architectures or operating systems
  - [github.com/golang/groupcache/lru](https://pkg.go.dev/github.com/golang/groupcache/lru) ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
  - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.1.2/LICENSE))
  - [github.com/google/nftables](https://pkg.go.dev/github.com/google/nftables) ([Apache-2.0](https://github.com/google/nftables/blob/5e242ec57806/LICENSE))
- - [github.com/google/uuid](https://pkg.go.dev/github.com/google/uuid) ([BSD-3-Clause](https://github.com/google/uuid/blob/v1.5.0/LICENSE))
+ - [github.com/google/uuid](https://pkg.go.dev/github.com/google/uuid) ([BSD-3-Clause](https://github.com/google/uuid/blob/v1.6.0/LICENSE))
  - [github.com/gorilla/csrf](https://pkg.go.dev/github.com/gorilla/csrf) ([BSD-3-Clause](https://github.com/gorilla/csrf/blob/v1.7.2/LICENSE))
  - [github.com/gorilla/securecookie](https://pkg.go.dev/github.com/gorilla/securecookie) ([BSD-3-Clause](https://github.com/gorilla/securecookie/blob/v1.1.2/LICENSE))
  - [github.com/hdevalence/ed25519consensus](https://pkg.go.dev/github.com/hdevalence/ed25519consensus) ([BSD-3-Clause](https://github.com/hdevalence/ed25519consensus/blob/v0.2.0/LICENSE))
@@ -73,6 +73,7 @@ Some packages may only be included on certain architectures or operating systems
  - [github.com/peterbourgon/ff/v3](https://pkg.go.dev/github.com/peterbourgon/ff/v3) ([Apache-2.0](https://github.com/peterbourgon/ff/blob/v3.4.0/LICENSE))
  - [github.com/pierrec/lz4/v4](https://pkg.go.dev/github.com/pierrec/lz4/v4) ([BSD-3-Clause](https://github.com/pierrec/lz4/blob/v4.1.21/LICENSE))
  - [github.com/pkg/sftp](https://pkg.go.dev/github.com/pkg/sftp) ([BSD-2-Clause](https://github.com/pkg/sftp/blob/v1.13.6/LICENSE))
+ - [github.com/prometheus-community/pro-bing](https://pkg.go.dev/github.com/prometheus-community/pro-bing) ([MIT](https://github.com/prometheus-community/pro-bing/blob/v0.4.0/LICENSE))
  - [github.com/safchain/ethtool](https://pkg.go.dev/github.com/safchain/ethtool) ([Apache-2.0](https://github.com/safchain/ethtool/blob/v0.3.0/LICENSE))
  - [github.com/skip2/go-qrcode](https://pkg.go.dev/github.com/skip2/go-qrcode) ([MIT](https://github.com/skip2/go-qrcode/blob/da1b6568686e/LICENSE))
  - [github.com/tailscale/certstore](https://pkg.go.dev/github.com/tailscale/certstore) ([MIT](https://github.com/tailscale/certstore/blob/d3fa0460f47e/LICENSE.md))

logtail/backoff/backoff.go

@@ -44,9 +44,8 @@ func NewBackoff(name string, logf logger.Logf, maxBackoff time.Duration) *Backof
 	}
 }
 
-// Backoff sleeps an increasing amount of time if err is non-nil.
-// and the context is not a
-// It resets the backoff schedule once err is nil.
+// BackOff sleeps an increasing amount of time if err is non-nil while the
+// context is active. It resets the backoff schedule once err is nil.
 func (b *Backoff) BackOff(ctx context.Context, err error) {
 	if err == nil {
 		// No error. Reset number of consecutive failures.

net/dns/manager.go

@@ -262,6 +262,18 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 		// config is empty, then we need to fallback to SplitDNS mode.
 		ocfg.MatchDomains = cfg.matchDomains()
 	} else {
+		// On iOS only (for now), check if all route names point to resources inside the tailnet.
+		// If so, we can set those names as MatchDomains to enable a split DNS configuration
+		// which will help preserve battery life.
+		// Because on iOS MatchDomains must equal SearchDomains, we cannot do this when
+		// we have any Routes outside the tailnet. Otherwise when app connectors are enabled,
+		// a query for 'work-laptop' might lead to search domain expansion, resolving
+		// as 'work-laptop.aws.com' for example.
+		if runtime.GOOS == "ios" && rcfg.RoutesRequireNoCustomResolvers() {
+			for r := range rcfg.Routes {
+				ocfg.MatchDomains = append(ocfg.MatchDomains, r)
+			}
+		}
 		var defaultRoutes []*dnstype.Resolver
 		for _, ip := range baseCfg.Nameservers {
 			defaultRoutes = append(defaultRoutes, &dnstype.Resolver{Addr: ip.String()})

net/dns/resolver/tsdns.go

@@ -175,6 +175,25 @@ func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]*dnstype.Resolver) {
 	}
 }
 
+// RoutesRequireNoCustomResolvers returns true if this resolver.Config only contains routes
+// that do not specify a set of custom resolver(s), i.e. they can be resolved by the local
+// upstream DNS resolver.
+func (c *Config) RoutesRequireNoCustomResolvers() bool {
+	for route, resolvers := range c.Routes {
+		if route.WithoutTrailingDot() == "ts.net" {
+			// Ignore the "ts.net" route here. It always specifies the corp resolvers but
+			// its presence is not an issue, as ts.net will be a search domain.
+			continue
+		}
+		if len(resolvers) != 0 {
+			// Found a route with custom resolvers.
+			return false
+		}
+	}
+	// No routes other than ts.net have specified one or more resolvers.
+	return true
+}
+
 // Resolver is a DNS resolver for nodes on the Tailscale network,
 // associating them with domain names of the form <mynode>.<mydomain>.<root>.
 // If it is asked to resolve a domain that is not of that form,

net/dns/resolver/tsdns_test.go

@@ -243,6 +243,43 @@ func mustIP(str string) netip.Addr {
 	return ip
 }
 
+func TestRoutesRequireNoCustomResolvers(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   Config
+		expected bool
+	}{
+		{"noRoutes", Config{Routes: map[dnsname.FQDN][]*dnstype.Resolver{}}, true},
+		{"onlyDefault", Config{Routes: map[dnsname.FQDN][]*dnstype.Resolver{
+			"ts.net.": {
+				{},
+			},
+		}}, true},
+		{"oneOther", Config{Routes: map[dnsname.FQDN][]*dnstype.Resolver{
+			"example.com.": {
+				{},
+			},
+		}}, false},
+		{"defaultAndOneOther", Config{Routes: map[dnsname.FQDN][]*dnstype.Resolver{
+			"ts.net.": {
+				{},
+			},
+			"example.com.": {
+				{},
+			},
+		}}, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.config.RoutesRequireNoCustomResolvers()
+			if result != tt.expected {
+				t.Errorf("result = %v; want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
 func TestRDNSNameToIPv4(t *testing.T) {
 	tests := []struct {
 		name   string

net/netcheck/netcheck.go

@@ -64,9 +64,6 @@ const (
 	// icmpProbeTimeout is the maximum amount of time netcheck will spend
 	// probing with ICMP packets.
 	icmpProbeTimeout = 1 * time.Second
-	// hairpinCheckTimeout is the amount of time we wait for a
-	// hairpinned packet to come back.
-	hairpinCheckTimeout = 100 * time.Millisecond
 	// defaultActiveRetransmitTime is the retransmit interval we use
 	// for STUN probes when we're in steady state (not in start-up),
 	// but don't have previous latency information for a DERP
@@ -96,11 +93,6 @@ type Report struct {
 	// STUN server you're talking to (on IPv4).
 	MappingVariesByDestIP opt.Bool
 
-	// HairPinning is whether the router supports communicating
-	// between two local devices through the NATted public IP address
-	// (on IPv4).
-	HairPinning opt.Bool
-
 	// UPnP is whether UPnP appears present on the LAN.
 	// Empty means not checked.
 	UPnP opt.Bool
@@ -116,11 +108,11 @@ type Report struct {
 	RegionV4Latency map[int]time.Duration // keyed by DERP Region ID
 	RegionV6Latency map[int]time.Duration // keyed by DERP Region ID
 
-	GlobalV4Counters map[netip.AddrPort]int // keyed by IP:port, number of times observed
-	GlobalV6Counters map[netip.AddrPort]int // keyed by [IP]:port, number of times observed
+	GlobalV4Counters map[netip.AddrPort]int // number of times the endpoint was observed
+	GlobalV6Counters map[netip.AddrPort]int // number of times the endpoint was observed
 
-	GlobalV4 netip.AddrPort // ip:port of global IPv4
-	GlobalV6 netip.AddrPort // [ip]:port of global IPv6
+	GlobalV4 netip.AddrPort
+	GlobalV6 netip.AddrPort
 
 	// CaptivePortal is set when we think there's a captive portal that is
 	// intercepting HTTP traffic.
@@ -133,8 +125,7 @@ type Report struct {
 // netcheck, which includes the best latency endpoint first, followed by any
 // other endpoints that were observed repeatedly. It excludes singular endpoints
 // that are likely only the result of a hard NAT.
-func (r *Report) GetGlobalAddrs() ([]netip.AddrPort, []netip.AddrPort) {
-	var v4, v6 []netip.AddrPort
+func (r *Report) GetGlobalAddrs() (v4, v6 []netip.AddrPort) {
 	// Always add the best latency entries first.
 	if r.GlobalV4.IsValid() {
 		v4 = append(v4, r.GlobalV4)
@@ -287,23 +278,6 @@ func (c *Client) vlogf(format string, a ...any) {
 	}
 }
 
-// handleHairSTUN reports whether pkt (from src) was our magic hairpin
-// probe packet that we sent to ourselves.
-func (c *Client) handleHairSTUNLocked(pkt []byte, src netip.AddrPort) bool {
-	rs := c.curState
-	if rs == nil {
-		return false
-	}
-	if tx, err := stun.ParseBindingRequest(pkt); err == nil && tx == rs.hairTX {
-		select {
-		case rs.gotHairSTUN <- src:
-		default:
-		}
-		return true
-	}
-	return false
-}
-
 // MakeNextReportFull forces the next GetReport call to be a full
 // (non-incremental) probe of all DERP regions.
 func (c *Client) MakeNextReportFull() {
@@ -326,10 +300,6 @@ func (c *Client) ReceiveSTUNPacket(pkt []byte, src netip.AddrPort) {
 	}
 
 	c.mu.Lock()
-	if c.handleHairSTUNLocked(pkt, src) {
-		c.mu.Unlock()
-		return
-	}
 	rs := c.curState
 	c.mu.Unlock()
 
@@ -340,6 +310,8 @@ func (c *Client) ReceiveSTUNPacket(pkt []byte, src netip.AddrPort) {
 	tx, addrPort, err := stun.ParseResponse(pkt)
 	if err != nil {
 		if _, err := stun.ParseBindingRequest(pkt); err == nil {
+			// We no longer send hairpin checks, but perhaps we might catch a
+			// stray from earlier versions.
 			// This was probably our own netcheck hairpin
 			// check probe coming in late. Ignore.
 			return
@@ -565,20 +537,15 @@ type reportState struct {
 	c           *Client
 	start       time.Time
 	opts        *GetReportOpts
-	hairTX      stun.TxID
-	gotHairSTUN chan netip.AddrPort
-	hairTimeout chan struct{} // closed on timeout
-	pc4Hair     nettype.PacketConn
 	incremental bool // doing a lite, follow-up netcheck
 	stopProbeCh chan struct{}
 	waitPortMap sync.WaitGroup
 
-	mu            sync.Mutex
-	sentHairCheck bool
-	report        *Report                            // to be returned by GetReport
-	inFlight      map[stun.TxID]func(netip.AddrPort) // called without c.mu held
-	gotEP4        netip.AddrPort
-	timers        []*time.Timer
+	mu       sync.Mutex
+	report   *Report                            // to be returned by GetReport
+	inFlight map[stun.TxID]func(netip.AddrPort) // called without c.mu held
+	gotEP4   netip.AddrPort
+	timers   []*time.Timer
 }
 
 func (rs *reportState) anyUDP() bool {
@@ -628,50 +595,6 @@ func (rs *reportState) probeWouldHelp(probe probe, node *tailcfg.DERPNode) bool
 	return false
 }
 
-func (rs *reportState) startHairCheckLocked(dst netip.AddrPort) {
-	if rs.sentHairCheck || rs.incremental {
-		return
-	}
-	rs.sentHairCheck = true
-	rs.pc4Hair.WriteToUDPAddrPort(stun.Request(rs.hairTX), dst)
-	rs.c.vlogf("sent haircheck to %v", dst)
-	time.AfterFunc(hairpinCheckTimeout, func() { close(rs.hairTimeout) })
-}
-
-func (rs *reportState) waitHairCheck(ctx context.Context) {
-	rs.mu.Lock()
-	defer rs.mu.Unlock()
-	ret := rs.report
-	if rs.incremental {
-		if rs.c.last != nil {
-			ret.HairPinning = rs.c.last.HairPinning
-		}
-		return
-	}
-	if !rs.sentHairCheck {
-		return
-	}
-
-	// First, check whether we have a value before we check for timeouts.
-	select {
-	case <-rs.gotHairSTUN:
-		ret.HairPinning.Set(true)
-		return
-	default:
-	}
-
-	// Now, wait for a response or a timeout.
-	select {
-	case <-rs.gotHairSTUN:
-		ret.HairPinning.Set(true)
-	case <-rs.hairTimeout:
-		rs.c.vlogf("hairCheck timeout")
-		ret.HairPinning.Set(false)
-	case <-ctx.Done():
-		rs.c.vlogf("hairCheck context timeout")
-	}
-}
-
 func (rs *reportState) stopTimers() {
 	rs.mu.Lock()
 	defer rs.mu.Unlock()
@@ -720,7 +643,6 @@ func (rs *reportState) addNodeLatency(node *tailcfg.DERPNode, ipp netip.AddrPort
 		if !rs.gotEP4.IsValid() {
 			rs.gotEP4 = ipp
 			ret.GlobalV4 = ipp
-			rs.startHairCheckLocked(ipp)
 		} else {
 			if rs.gotEP4 != ipp {
 				ret.MappingVariesByDestIP.Set(true)
@@ -834,9 +756,6 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe
 		opts:        opts,
 		report:      newReport(),
 		inFlight:    map[stun.TxID]func(netip.AddrPort){},
-		hairTX:      stun.NewTxID(), // random payload
-		gotHairSTUN: make(chan netip.AddrPort, 1),
-		hairTimeout: make(chan struct{}),
 		stopProbeCh: make(chan struct{}, 1),
 	}
 	c.curState = rs
@@ -894,34 +813,11 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe
 		v6udp.Close()
 	}
 
-	// Create a UDP4 socket used for sending to our discovered IPv4 address.
-	rs.pc4Hair, err = nettype.MakePacketListenerWithNetIP(netns.Listener(c.logf, c.NetMon)).ListenPacket(ctx, "udp4", ":0")
-	if err != nil {
-		c.logf("udp4: %v", err)
-		return nil, err
-	}
-	defer rs.pc4Hair.Close()
-
 	if !c.SkipExternalNetwork && c.PortMapper != nil {
 		rs.waitPortMap.Add(1)
 		go rs.probePortMapServices()
 	}
 
-	// At least the Apple Airport Extreme doesn't allow hairpin
-	// sends from a private socket until it's seen traffic from
-	// that src IP:port to something else out on the internet.
-	//
-	// See https://github.com/tailscale/tailscale/issues/188#issuecomment-600728643
-	//
-	// And it seems that even sending to a likely-filtered RFC 5737
-	// documentation-only IPv4 range is enough to set up the mapping.
-	// So do that for now. In the future we might want to classify networks
-	// that do and don't require this separately. But for now help it.
-	const documentationIP = "203.0.113.1"
-	rs.pc4Hair.WriteToUDPAddrPort(
-		[]byte("tailscale netcheck; see https://github.com/tailscale/tailscale/issues/188"),
-		netip.AddrPortFrom(netip.MustParseAddr(documentationIP), 12345))
-
 	plan := makeProbePlan(dm, ifState, last)
 
 	// If we're doing a full probe, also check for a captive portal. We
@@ -999,8 +895,6 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe
 		captivePortalStop()
 	}
 
-	rs.waitHairCheck(ctx)
-	c.vlogf("hairCheck done")
 	if !c.SkipExternalNetwork && c.PortMapper != nil {
 		rs.waitPortMap.Wait()
 		c.vlogf("portMap done")
@@ -1369,7 +1263,6 @@ func (c *Client) logConciseReport(r *Report, dm *tailcfg.DERPMap) {
 			fmt.Fprintf(w, " v6os=%v", r.OSHasIPv6)
 		}
 		fmt.Fprintf(w, " mapvarydest=%v", r.MappingVariesByDestIP)
-		fmt.Fprintf(w, " hair=%v", r.HairPinning)
 		if r.AnyPortMappingChecked() {
 			fmt.Fprintf(w, " portmap=%v%v%v", conciseOptBool(r.UPnP, "U"), conciseOptBool(r.PMP, "M"), conciseOptBool(r.PCP, "C"))
 		} else {

net/netcheck/netcheck_test.go

@@ -20,142 +20,12 @@ import (
 	"time"
 
 	"tailscale.com/net/netmon"
-	"tailscale.com/net/stun"
 	"tailscale.com/net/stun/stuntest"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/nettest"
 )
 
-func TestHairpinSTUN(t *testing.T) {
-	tx := stun.NewTxID()
-	c := &Client{
-		curState: &reportState{
-			hairTX:      tx,
-			gotHairSTUN: make(chan netip.AddrPort, 1),
-		},
-	}
-	req := stun.Request(tx)
-	if !stun.Is(req) {
-		t.Fatal("expected STUN message")
-	}
-	if !c.handleHairSTUNLocked(req, netip.AddrPort{}) {
-		t.Fatal("expected true")
-	}
-	select {
-	case <-c.curState.gotHairSTUN:
-	default:
-		t.Fatal("expected value")
-	}
-}
-
-func TestHairpinWait(t *testing.T) {
-	makeClient := func(t *testing.T) (*Client, *reportState) {
-		tx := stun.NewTxID()
-		c := &Client{}
-		req := stun.Request(tx)
-		if !stun.Is(req) {
-			t.Fatal("expected STUN message")
-		}
-
-		var err error
-		rs := &reportState{
-			c:           c,
-			hairTX:      tx,
-			gotHairSTUN: make(chan netip.AddrPort, 1),
-			hairTimeout: make(chan struct{}),
-			report:      newReport(),
-		}
-		rs.pc4Hair, err = net.ListenUDP("udp4", &net.UDPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 0,
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		c.curState = rs
-		return c, rs
-	}
-
-	ll, err := net.ListenPacket("udp", "localhost:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ll.Close()
-	dstAddr := netip.MustParseAddrPort(ll.LocalAddr().String())
-
-	t.Run("Success", func(t *testing.T) {
-		c, rs := makeClient(t)
-		req := stun.Request(rs.hairTX)
-
-		// Start a hairpin check to ourselves.
-		rs.startHairCheckLocked(dstAddr)
-
-		// Fake receiving the stun check from ourselves after some period of time.
-		src := netip.MustParseAddrPort(rs.pc4Hair.LocalAddr().String())
-		c.handleHairSTUNLocked(req, src)
-
-		rs.waitHairCheck(context.Background())
-
-		// Verify that we set HairPinning
-		if got := rs.report.HairPinning; !got.EqualBool(true) {
-			t.Errorf("wanted HairPinning=true, got %v", got)
-		}
-	})
-
-	t.Run("LateReply", func(t *testing.T) {
-		c, rs := makeClient(t)
-		req := stun.Request(rs.hairTX)
-
-		// Start a hairpin check to ourselves.
-		rs.startHairCheckLocked(dstAddr)
-
-		// Wait until we've timed out, to mimic the race in #1795.
-		<-rs.hairTimeout
-
-		// Fake receiving the stun check from ourselves after some period of time.
-		src := netip.MustParseAddrPort(rs.pc4Hair.LocalAddr().String())
-		c.handleHairSTUNLocked(req, src)
-
-		// Wait for a hairpin response
-		rs.waitHairCheck(context.Background())
-
-		// Verify that we set HairPinning
-		if got := rs.report.HairPinning; !got.EqualBool(true) {
-			t.Errorf("wanted HairPinning=true, got %v", got)
-		}
-	})
-
-	t.Run("Timeout", func(t *testing.T) {
-		_, rs := makeClient(t)
-
-		// Start a hairpin check to ourselves.
-		rs.startHairCheckLocked(dstAddr)
-
-		ctx, cancel := context.WithTimeout(context.Background(), hairpinCheckTimeout*50)
-		defer cancel()
-
-		// Wait in the background
-		waitDone := make(chan struct{})
-		go func() {
-			rs.waitHairCheck(ctx)
-			close(waitDone)
-		}()
-
-		// If we do nothing, then we time out; confirm that we set
-		// HairPinning to false in this case.
-		select {
-		case <-waitDone:
-			if got := rs.report.HairPinning; !got.EqualBool(false) {
-				t.Errorf("wanted HairPinning=false, got %v", got)
-			}
-		case <-ctx.Done():
-			t.Fatalf("timed out waiting for hairpin channel")
-		}
-	})
-}
-
 func newTestClient(t testing.TB) *Client {
 	c := &Client{
 		NetMon: netmon.NewStatic(),
@@ -211,10 +81,9 @@ func TestMultiGlobalAddressMapping(t *testing.T) {
 	}
 
 	rs := &reportState{
-		c:             c,
-		start:         time.Now(),
-		report:        newReport(),
-		sentHairCheck: true, // prevent hair check start, not relevant here
+		c:      c,
+		start:  time.Now(),
+		report: newReport(),
 	}
 	derpNode := &tailcfg.DERPNode{}
 	port1 := netip.MustParseAddrPort("127.0.0.1:1234")
@@ -784,12 +653,12 @@ func TestLogConciseReport(t *testing.T) {
 		{
 			name: "no_udp",
 			r:    &Report{},
-			want: "udp=false v4=false icmpv4=false v6=false mapvarydest= hair= portmap=? derp=0",
+			want: "udp=false v4=false icmpv4=false v6=false mapvarydest= portmap=? derp=0",
 		},
 		{
 			name: "no_udp_icmp",
 			r:    &Report{ICMPv4: true, IPv4: true},
-			want: "udp=false icmpv4=true v6=false mapvarydest= hair= portmap=? derp=0",
+			want: "udp=false icmpv4=true v6=false mapvarydest= portmap=? derp=0",
 		},
 		{
 			name: "ipv4_one_region",
@@ -804,7 +673,7 @@ func TestLogConciseReport(t *testing.T) {
 					1: 10 * ms,
 				},
 			},
-			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms",
+			want: "udp=true v6=false mapvarydest= portmap=? derp=1 derpdist=1v4:10ms",
 		},
 		{
 			name: "ipv4_all_region",
@@ -823,7 +692,7 @@ func TestLogConciseReport(t *testing.T) {
 					3: 30 * ms,
 				},
 			},
-			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
+			want: "udp=true v6=false mapvarydest= portmap=? derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
 		},
 		{
 			name: "ipboth_all_region",
@@ -848,7 +717,7 @@ func TestLogConciseReport(t *testing.T) {
 					3: 30 * ms,
 				},
 			},
-			want: "udp=true v6=true mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+			want: "udp=true v6=true mapvarydest= portmap=? derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
 		},
 		{
 			name: "portmap_all",
@@ -858,7 +727,7 @@ func TestLogConciseReport(t *testing.T) {
 				PMP:  "true",
 				PCP:  "true",
 			},
-			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UMC derp=0",
+			want: "udp=true v4=false v6=false mapvarydest= portmap=UMC derp=0",
 		},
 		{
 			name: "portmap_some",
@@ -868,7 +737,7 @@ func TestLogConciseReport(t *testing.T) {
 				PMP:  "false",
 				PCP:  "true",
 			},
-			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UC derp=0",
+			want: "udp=true v4=false v6=false mapvarydest= portmap=UC derp=0",
 		},
 	}
 	for _, tt := range tests {

net/tsdial/tsdial.go

@@ -59,6 +59,10 @@ type Dialer struct {
 	// If nil, it's not used.
 	NetstackDialTCP func(context.Context, netip.AddrPort) (net.Conn, error)
 
+	// NetstackDialUDP dials the provided IPPort using netstack.
+	// If nil, it's not used.
+	NetstackDialUDP func(context.Context, netip.AddrPort) (net.Conn, error)
+
 	peerClientOnce sync.Once
 	peerClient     *http.Client
 
@@ -403,9 +407,12 @@ func (d *Dialer) UserDial(ctx context.Context, network, addr string) (net.Conn,
 		return nil, err
 	}
 	if d.UseNetstackForIP != nil && d.UseNetstackForIP(ipp.Addr()) {
-		if d.NetstackDialTCP == nil {
+		if d.NetstackDialTCP == nil || d.NetstackDialUDP == nil {
 			return nil, errors.New("Dialer not initialized correctly")
 		}
+		if strings.HasPrefix(network, "udp") {
+			return d.NetstackDialUDP(ctx, ipp)
+		}
 		return d.NetstackDialTCP(ctx, ipp)
 	}
 

omit/aws_def.go

@@ -0,0 +1,9 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !ts_omit_aws
+
+package omit
+
+// AWS is whether AWS support should be omitted from the build.
+const AWS = false

omit/aws_omit.go

@@ -0,0 +1,9 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build ts_omit_aws
+
+package omit
+
+// AWS is whether AWS support should be omitted from the build.
+const AWS = true

omit/omit.go

@@ -0,0 +1,12 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package omit provides consts to access Tailscale ts_omit_FOO build tags.
+// They're often more convenient to eliminate some away locally with a const
+// rather than using build tags.
+package omit
+
+import "errors"
+
+// Err is an error that can be returned by functions in this package.
+var Err = errors.New("feature not linked into binary per ts_omit build tag")

publicapi/device.md

@@ -236,8 +236,6 @@ The API exposes two methods for dealing with subnet routes:
 - Get routes: [`GET /api/v2/device/{deviceID}/routes`](#get-device-routes) to fetch lists of advertised and enabled routes for a device
 - Set routes: [`POST /api/v2/device/{deviceID}/routes`](#set-device-routes) to set enabled routes for a device
 
-<a name="device-get"></a>
-
 ## Get device
 
 ```http
@@ -295,8 +293,6 @@ curl "https://api.tailscale.com/api/v2/device/12345?fields=all" \
 }
 ```
 
-<a href="device-delete"></a>
-
 ## Delete device
 
 ```http
@@ -336,8 +332,6 @@ HTTP/1.1 501 Not Implemented
 {"message":"cannot delete devices outside of your tailnet"}
 ```
 
-<a href="expire-device-key"></a>
-
 ## Expire a device's key
 
 ```http
@@ -372,8 +366,6 @@ HTTP/1.1 200 OK
 
 ## Routes
 
-<a href="device-routes-get">
-
 ## Get device routes
 
 ```http
@@ -409,8 +401,6 @@ Returns the enabled and advertised subnet routes for a device.
 }
 ```
 
-<a href="device-routes-post"></a>
-
 ## Set device routes
 
 ```http
@@ -458,8 +448,6 @@ Returns the enabled and advertised subnet routes for a device.
 
 ## Authorize
 
-<a href="#device-authorized-post"></a>
-
 ## Authorize device
 
 ```http
@@ -502,8 +490,6 @@ The response is 2xx on success. The response body is currently an empty JSON obj
 
 ## Tags
 
-<a href="device-tags-post"></a>
-
 ## Update device tags
 
 ```http
@@ -562,8 +548,6 @@ If the tags supplied in the `POST` call do not exist in the tailnet policy file,
 
 ## Keys
 
-<a href="device-key-post"></a>
-
 ## Update device key
 
 ```http

publicapi/tailnet.md

@@ -64,8 +64,6 @@ The policy file is expressed using "[HuJSON](https://github.com/tailscale/hujson
 Most policy file API methods can also return regular JSON for compatibility with other tools.
 Learn more about [network access controls](https://tailscale.com/kb/1018/).
 
-<a href="tailnet-acl-get"></a>
-
 ## Get Policy File
 
 ```http
@@ -203,8 +201,6 @@ In addition, errors and warnings are returned.
 }
 ```
 
-<a href="tailnet-acl-post"></a>
-
 ## Update policy file
 
 ```http
@@ -325,8 +321,6 @@ A successful response returns an HTTP status of '200' and the modified tailnet p
 }
 ```
 
-<a href="tailnet-acl-preview-post"></a>
-
 ## Preview policy file rule matches
 
 ```http
@@ -418,8 +412,6 @@ The response also echoes the `type` and `previewFor` values supplied in the requ
 }
 ```
 
-<a href="tailnet-acl-validate-post"></a>
-
 ## Validate and test policy file
 
 ```http
@@ -526,8 +518,6 @@ any groups that are used in the policy file that are not being synced from SCIM.
 
 ## Devices
 
-<a href="tailnet-devices"></a>
-
 ## List tailnet devices
 
 ```http
@@ -643,8 +633,6 @@ The remaining three methods operate on auth keys and API access tokens.
 }
 ```
 
-<a href="tailnet-keys-get"></a>
-
 ## List tailnet keys
 
 ```http
@@ -684,8 +672,6 @@ Returns a JSON object with the IDs of all active keys.
 }
 ```
 
-<a href="tailnet-keys-post"></a>
-
 ## Create auth key
 
 ```http
@@ -783,8 +769,6 @@ It holds the capabilities specified in the request and can no longer be retrieve
 }
 ```
 
-<a href="tailnet-keys-key-get"></a>
-
 ## Get key
 
 ```http
@@ -845,8 +829,6 @@ Response for a revoked (deleted) or expired key will have an `invalid` field set
 }
 ```
 
-<a href="tailnet-keys-key-delete"></a>
-
 ## Delete key
 
 ```http
@@ -876,8 +858,6 @@ curl -X DELETE 'https://api.tailscale.com/api/v2/tailnet/example.com/keys/k12345
 
 This returns status 200 upon success.
 
-<a href="tailnet-dns"></a>
-
 ## DNS
 
 The tailnet DNS methods are provided for fetching and modifying various DNS settings for a tailnet.
@@ -886,8 +866,6 @@ Learn more about [DNS in Tailscale](https://tailscale.com/kb/1054/).
 
 ## Nameservers
 
-<a href="tailnet-dns-nameservers-get"></a>
-
 ## Get nameservers
 
 ```http
@@ -917,8 +895,6 @@ curl "https://api.tailscale.com/api/v2/tailnet/example.com/dns/nameservers" \
 }
 ```
 
-<a href="tailnet-dns-nameservers-post"></a>
-
 ## Set nameservers
 
 ```http
@@ -989,8 +965,6 @@ The response is a JSON object containing the new list of nameservers and the sta
 
 ## Preferences
 
-<a href="tailnet-dns-preferences-get"></a>
-
 ## Get DNS preferences
 
 ```http
@@ -1020,8 +994,6 @@ curl "https://api.tailscale.com/api/v2/tailnet/example.com/dns/preferences" \
 }
 ```
 
-<a href="tailnet-dns-preferences-post"></a>
-
 ## Set DNS preferences
 
 ```http
@@ -1085,8 +1057,6 @@ If there are DNS servers, this returns the MagicDNS status:
 
 ## Search Paths
 
-<a href="tailnet-dns-searchpaths-get"></a>
-
 ## Get search paths
 
 ```http
@@ -1116,8 +1086,6 @@ curl "https://api.tailscale.com/api/v2/tailnet/example.com/dns/searchpaths" \
 }
 ```
 
-<a href="tailnet-dns-searchpaths-post"></a>
-
 ## Set search paths
 
 ```http

ssh/tailssh/incubator.go

@@ -36,6 +36,7 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/cmd/tailscaled/childproc"
 	"tailscale.com/hostinfo"
+	"tailscale.com/tailcfg"
 	"tailscale.com/tempfork/gliderlabs/ssh"
 	"tailscale.com/types/logger"
 	"tailscale.com/version/distro"
@@ -43,18 +44,22 @@ import (
 
 func init() {
 	childproc.Add("ssh", beIncubator)
+	childproc.Add("sftp", beSFTP)
 }
 
 var ptyName = func(f *os.File) (string, error) {
 	return "", fmt.Errorf("unimplemented")
 }
 
-// maybeStartLoginSession starts a new login session for the specified UID.
-// On success, it may return a non-nil close func which must be closed to
+// maybeStartLoginSession informs the system that we are about to log someone
+// in. On success, it may return a non-nil close func which must be closed to
 // release the session.
+// We can only do this if we are running as root.
+// This is best effort to still allow running on machines where
+// we don't support starting sessions, e.g. darwin.
 // See maybeStartLoginSessionLinux.
-var maybeStartLoginSession = func(logf logger.Logf, ia incubatorArgs) (close func() error, err error) {
-	return nil, nil
+var maybeStartLoginSession = func(dlogf logger.Logf, ia incubatorArgs) (close func() error) {
+	return nil
 }
 
 // newIncubatorCommand returns a new exec.Cmd configured with
@@ -64,40 +69,39 @@ var maybeStartLoginSession = func(logf logger.Logf, ia incubatorArgs) (close fun
 // exec.CommandContext.
 //
 // The returned Cmd.Env is guaranteed to be nil; the caller populates it.
-func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) {
+func (ss *sshSession) newIncubatorCommand(logf logger.Logf) (cmd *exec.Cmd, err error) {
 	defer func() {
 		if cmd.Env != nil {
 			panic("internal error")
 		}
 	}()
-	var (
-		name    string
-		args    []string
-		isSFTP  bool
-		isShell bool
-	)
+
+	var isSFTP, isShell bool
 	switch ss.Subsystem() {
 	case "sftp":
 		isSFTP = true
 	case "":
-		name = ss.conn.localUser.LoginShell()
-		if rawCmd := ss.RawCommand(); rawCmd != "" {
-			args = append(args, "-c", rawCmd)
-		} else {
-			isShell = true
-			args = append(args, "-l") // login shell
-		}
+		isShell = ss.RawCommand() == ""
 	default:
 		panic(fmt.Sprintf("unexpected subsystem: %v", ss.Subsystem()))
 	}
 
 	if ss.conn.srv.tailscaledPath == "" {
-		// TODO(maisem): this doesn't work with sftp
-		return exec.CommandContext(ss.ctx, name, args...)
+		if isSFTP {
+			// SFTP relies on the embedded Go-based SFTP server in tailscaled,
+			// so without tailscaled, we can't serve SFTP.
+			return nil, errors.New("no tailscaled found on path, can't serve SFTP")
+		}
+
+		loginShell := ss.conn.localUser.LoginShell()
+		args := shellArgs(isShell, ss.RawCommand())
+		logf("directly running %s %q", loginShell, args)
+		return exec.CommandContext(ss.ctx, loginShell, args...), nil
 	}
+
 	lu := ss.conn.localUser
 	ci := ss.conn.info
-	gids := strings.Join(ss.conn.userGroupIDs, ",")
+	groups := strings.Join(ss.conn.userGroupIDs, ",")
 	remoteUser := ci.uprof.LoginName
 	if ci.node.IsTagged() {
 		remoteUser = strings.Join(ci.node.Tags().AsSlice(), ",")
@@ -106,9 +110,10 @@ func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) {
 	incubatorArgs := []string{
 		"be-child",
 		"ssh",
+		"--login-shell=" + lu.LoginShell(),
 		"--uid=" + lu.Uid,
 		"--gid=" + lu.Gid,
-		"--groups=" + gids,
+		"--groups=" + groups,
 		"--local-user=" + lu.Username,
 		"--remote-user=" + remoteUser,
 		"--remote-ip=" + ci.src.Addr().String(),
@@ -116,39 +121,31 @@ func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) {
 		"--tty-name=",     // updated in-place by startWithPTY
 	}
 
+	forceV1Behavior := ss.conn.srv.lb.NetMap().HasCap(tailcfg.NodeAttrSSHBehaviorV1)
+	if forceV1Behavior {
+		incubatorArgs = append(incubatorArgs, "--force-v1-behavior")
+	}
+
 	if debugTest.Load() {
 		incubatorArgs = append(incubatorArgs, "--debug-test")
 	}
 
-	if isSFTP {
-		incubatorArgs = append(incubatorArgs, "--sftp")
-	} else {
-		if isShell {
-			incubatorArgs = append(incubatorArgs, "--shell")
-		}
-		// Only the macOS version of the login command supports executing a
-		// command, all other versions only support launching a shell
-		// without taking any arguments.
-		shouldUseLoginCmd := isShell || runtime.GOOS == "darwin"
-		if hostinfo.IsSELinuxEnforcing() {
-			// If we're running on a SELinux-enabled system, the login
-			// command will be unable to set the correct context for the
-			// shell. Fall back to using the incubator to launch the shell.
-			// See http://github.com/tailscale/tailscale/issues/4908.
-			shouldUseLoginCmd = false
-		}
-		if shouldUseLoginCmd {
-			if lp, err := exec.LookPath("login"); err == nil {
-				incubatorArgs = append(incubatorArgs, "--login-cmd="+lp)
-			}
-		}
-		incubatorArgs = append(incubatorArgs, "--cmd="+name)
-		if len(args) > 0 {
-			incubatorArgs = append(incubatorArgs, "--")
-			incubatorArgs = append(incubatorArgs, args...)
-		}
+	switch {
+	case isSFTP:
+		// Note that we include both the `--sftp` flag and a command to launch
+		// tailscaled as `be-child sftp`. If login or su is available, and
+		// we're not running with tailcfg.NodeAttrSSHBehaviorV1, this will
+		// result in serving SFTP within a login shell, with full PAM
+		// integration. Otherwise, we'll serve SFTP in the incubator process
+		// with no PAM integration.
+		incubatorArgs = append(incubatorArgs, "--sftp", fmt.Sprintf("--cmd=%s be-child sftp", ss.conn.srv.tailscaledPath))
+	case isShell:
+		incubatorArgs = append(incubatorArgs, "--shell")
+	default:
+		incubatorArgs = append(incubatorArgs, "--cmd="+ss.RawCommand())
 	}
-	return exec.CommandContext(ss.ctx, ss.conn.srv.tailscaledPath, incubatorArgs...)
+
+	return exec.CommandContext(ss.ctx, ss.conn.srv.tailscaledPath, incubatorArgs...), nil
 }
 
 var debugIncubator bool
@@ -170,51 +167,60 @@ func (stdRWC) Close() error {
 }
 
 type incubatorArgs struct {
-	uid          int
-	gid          int
-	groups       string
-	localUser    string
-	remoteUser   string
-	remoteIP     string
-	ttyName      string
-	hasTTY       bool
-	cmdName      string
-	isSFTP       bool
-	isShell      bool
-	loginCmdPath string
-	cmdArgs      []string
-	debugTest    bool
+	loginShell      string
+	uid             int
+	gid             int
+	gids            []int
+	localUser       string
+	remoteUser      string
+	remoteIP        string
+	ttyName         string
+	hasTTY          bool
+	cmd             string
+	isSFTP          bool
+	isShell         bool
+	forceV1Behavior bool
+	debugTest       bool
 }
 
-func parseIncubatorArgs(args []string) (a incubatorArgs) {
+func parseIncubatorArgs(args []string) (incubatorArgs, error) {
+	var ia incubatorArgs
+	var groups string
+
 	flags := flag.NewFlagSet("", flag.ExitOnError)
-	flags.IntVar(&a.uid, "uid", 0, "the uid of local-user")
-	flags.IntVar(&a.gid, "gid", 0, "the gid of local-user")
-	flags.StringVar(&a.groups, "groups", "", "comma-separated list of gids of local-user")
-	flags.StringVar(&a.localUser, "local-user", "", "the user to run as")
-	flags.StringVar(&a.remoteUser, "remote-user", "", "the remote user/tags")
-	flags.StringVar(&a.remoteIP, "remote-ip", "", "the remote Tailscale IP")
-	flags.StringVar(&a.ttyName, "tty-name", "", "the tty name (pts/3)")
-	flags.BoolVar(&a.hasTTY, "has-tty", false, "is the output attached to a tty")
-	flags.StringVar(&a.cmdName, "cmd", "", "the cmd to launch (ignored in sftp mode)")
-	flags.BoolVar(&a.isShell, "shell", false, "is launching a shell (with no cmds)")
-	flags.BoolVar(&a.isSFTP, "sftp", false, "run sftp server (cmd is ignored)")
-	flags.StringVar(&a.loginCmdPath, "login-cmd", "", "the path to `login` cmd")
-	flags.BoolVar(&a.debugTest, "debug-test", false, "should debug in test mode")
+	flags.StringVar(&ia.loginShell, "login-shell", "", "path to the user's preferred login shell")
+	flags.IntVar(&ia.uid, "uid", 0, "the uid of local-user")
+	flags.IntVar(&ia.gid, "gid", 0, "the gid of local-user")
+	flags.StringVar(&groups, "groups", "", "comma-separated list of gids of local-user")
+	flags.StringVar(&ia.localUser, "local-user", "", "the user to run as")
+	flags.StringVar(&ia.remoteUser, "remote-user", "", "the remote user/tags")
+	flags.StringVar(&ia.remoteIP, "remote-ip", "", "the remote Tailscale IP")
+	flags.StringVar(&ia.ttyName, "tty-name", "", "the tty name (pts/3)")
+	flags.BoolVar(&ia.hasTTY, "has-tty", false, "is the output attached to a tty")
+	flags.StringVar(&ia.cmd, "cmd", "", "the cmd to launch, including all arguments (ignored in sftp mode)")
+	flags.BoolVar(&ia.isShell, "shell", false, "is launching a shell (with no cmds)")
+	flags.BoolVar(&ia.isSFTP, "sftp", false, "run sftp server (cmd is ignored)")
+	flags.BoolVar(&ia.forceV1Behavior, "force-v1-behavior", false, "allow falling back to the su command if login is unavailable")
+	flags.BoolVar(&ia.debugTest, "debug-test", false, "should debug in test mode")
 	flags.Parse(args)
-	a.cmdArgs = flags.Args()
-	return a
+
+	for _, g := range strings.Split(groups, ",") {
+		gid, err := strconv.Atoi(g)
+		if err != nil {
+			return ia, fmt.Errorf("unable to parse group id %q: %w", g, err)
+		}
+		ia.gids = append(ia.gids, gid)
+	}
+
+	return ia, nil
 }
 
 // beIncubator is the entrypoint to the `tailscaled be-child ssh` subcommand.
-// It is responsible for informing the system of a new login session for the user.
-// This is sometimes necessary for mounting home directories and decrypting file
-// systems.
+// It is responsible for informing the system of a new login session for the
+// user. This is sometimes necessary for mounting home directories and
+// decrypting file systems.
 //
-// Tailscaled launches the incubator as the same user as it was
-// launched as.  The incubator then registers a new session with the
-// OS, sets its UID and groups to the specified `--uid`, `--gid` and
-// `--groups` and then launches the requested `--cmd`.
+// Tailscaled launches the incubator as the same user as it was launched as.
 func beIncubator(args []string) error {
 	// To defend against issues like https://golang.org/issue/1435,
 	// defensively lock our current goroutine's thread to the current
@@ -226,22 +232,25 @@ func beIncubator(args []string) error {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
-	ia := parseIncubatorArgs(args)
+	ia, err := parseIncubatorArgs(args)
+	if err != nil {
+		return err
+	}
 	if ia.isSFTP && ia.isShell {
 		return fmt.Errorf("--sftp and --shell are mutually exclusive")
 	}
 
-	logf := logger.Discard
+	dlogf := logger.Discard
 	if debugIncubator {
 		// We don't own stdout or stderr, so the only place we can log is syslog.
 		if sl, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "tailscaled-ssh"); err == nil {
-			logf = log.New(sl, "", 0).Printf
+			dlogf = log.New(sl, "", 0).Printf
 		}
 	} else if ia.debugTest {
-		// In testing, we don't always have syslog, log to a temp file
+		// In testing, we don't always have syslog, so log to a temp file.
 		if logFile, err := os.OpenFile("/tmp/tailscalessh.log", os.O_APPEND|os.O_WRONLY, 0666); err == nil {
 			lf := log.New(logFile, "", 0)
-			logf = func(msg string, args ...any) {
+			dlogf = func(msg string, args ...any) {
 				lf.Printf(msg, args...)
 				logFile.Sync()
 			}
@@ -249,72 +258,233 @@ func beIncubator(args []string) error {
 		}
 	}
 
-	euid := os.Geteuid()
-	runningAsRoot := euid == 0
-	if runningAsRoot && ia.loginCmdPath != "" {
-		// Check if we can exec into the login command instead of trying to
-		// incubate ourselves.
-		if la := ia.loginArgs(); la != nil {
-			return unix.Exec(ia.loginCmdPath, la, os.Environ())
-		}
+	if !shouldAttemptLoginShell(dlogf, ia) {
+		dlogf("not attempting login shell")
+		return handleInProcess(dlogf, ia)
 	}
 
-	// Inform the system that we are about to log someone in.
-	// We can only do this if we are running as root.
-	// This is best effort to still allow running on machines where
-	// we don't support starting sessions, e.g. darwin.
-	sessionCloser, err := maybeStartLoginSession(logf, ia)
-	if err == nil && sessionCloser != nil {
-		defer sessionCloser()
-	}
-
-	var groupIDs []int
-	for _, g := range strings.Split(ia.groups, ",") {
-		gid, err := strconv.ParseInt(g, 10, 32)
-		if err != nil {
-			return err
-		}
-		groupIDs = append(groupIDs, int(gid))
-	}
-
-	if err := dropPrivileges(logf, ia.uid, ia.gid, groupIDs); err != nil {
+	// First try the login command
+	if err := tryExecLogin(dlogf, ia); err != nil {
 		return err
 	}
 
-	if ia.isSFTP {
-		logf("handling sftp")
+	// If we got here, we weren't able to use login (because tryExecLogin
+	// returned without replacing the running process), maybe we can use
+	// su.
+	if handled, err := trySU(dlogf, ia); handled {
+		return err
+	} else {
+		dlogf("not attempting su")
+		return handleInProcess(dlogf, ia)
+	}
+}
 
-		server, err := sftp.NewServer(stdRWC{})
-		if err != nil {
-			return err
-		}
-		// TODO(https://github.com/pkg/sftp/pull/554): Revert the check for io.EOF,
-		// when sftp is patched to report clean termination.
-		if err := server.Serve(); err != nil && err != io.EOF {
-			return err
-		}
+func handleInProcess(dlogf logger.Logf, ia incubatorArgs) error {
+	if ia.isSFTP {
+		return handleSFTPInProcess(dlogf, ia)
+	}
+	return handleSSHInProcess(dlogf, ia)
+}
+
+func handleSFTPInProcess(dlogf logger.Logf, ia incubatorArgs) error {
+	dlogf("handling sftp")
+
+	sessionCloser := maybeStartLoginSession(dlogf, ia)
+	if sessionCloser != nil {
+		defer sessionCloser()
+	}
+
+	if err := dropPrivileges(dlogf, ia); err != nil {
+		return err
+	}
+
+	return serveSFTP()
+}
+
+// beSFTP serves SFTP in-process.
+func beSFTP(args []string) error {
+	return serveSFTP()
+}
+
+func serveSFTP() error {
+	server, err := sftp.NewServer(stdRWC{})
+	if err != nil {
+		return err
+	}
+	// TODO(https://github.com/pkg/sftp/pull/554): Revert the check for io.EOF,
+	// when sftp is patched to report clean termination.
+	if err := server.Serve(); err != nil && err != io.EOF {
+		return err
+	}
+	return nil
+}
+
+// shouldAttemptLoginShell decides whether we should attempt to get a full
+// login shell with the login or su commands. We will attempt a login shell
+// if all of the following conditions are met.
+//
+// - We are running as root
+// - This is not an SELinuxEnforcing host
+//
+// The last condition exists because if we're running on a SELinux-enabled
+// system, neiher login nor su will be able to set the correct context for the
+// shell. So, we don't bother trying to run them and instead fall back to using
+// the incubator to launch the shell.
+// See http://github.com/tailscale/tailscale/issues/4908.
+func shouldAttemptLoginShell(dlogf logger.Logf, ia incubatorArgs) bool {
+	if ia.forceV1Behavior && ia.isSFTP {
+		// v1 behavior did not run SFTP within a login shell.
+		dlogf("Forcing v1 behavior, won't use login shell for SFTP")
+		return false
+	}
+
+	return runningAsRoot() && !hostinfo.IsSELinuxEnforcing()
+}
+
+func runningAsRoot() bool {
+	euid := os.Geteuid()
+	return euid == 0
+}
+
+// tryExecLogin attempts to handle the ssh session by creating a full login
+// shell using the login command. If it never tried, it returns nil. If it
+// failed to do so, it returns an error.
+//
+// Creating a login shell in this way allows us to register the remote IP of
+// the login session, trigger PAM authentication, and get the "remote" PAM
+// profile.
+//
+// However, login is subject to some limitations.
+//
+// 1. login cannot be used to execute commands except on macOS.
+// 2. On Linux and BSD, login requires a TTY to keep running.
+//
+// In these cases, tryExecLogin returns (false, nil) to indicate that processing
+// should fall through to other methods, such as using the su command.
+//
+// Note that this uses unix.Exec to replace the current process, so in cases
+// where we actually do run login, no subsequent Go code will execute.
+func tryExecLogin(dlogf logger.Logf, ia incubatorArgs) error {
+	// Only the macOS version of the login command supports executing a
+	// command, all other versions only support launching a shell without
+	// taking any arguments.
+	if !ia.isShell && runtime.GOOS != "darwin" {
+		dlogf("won't use login because we're not in a shell or on macOS")
 		return nil
 	}
 
-	cmd := exec.Command(ia.cmdName, ia.cmdArgs...)
-	cmd.Stdin = os.Stdin
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Env = os.Environ()
-
-	if ia.hasTTY {
-		// If we were launched with a tty then we should
-		// mark that as the ctty of the child. However,
-		// as the ctty is being passed from the parent
-		// we set the child to foreground instead which
-		// also passes the ctty.
-		// However, we can not do this if never had a tty to
-		// begin with.
-		cmd.SysProcAttr = &syscall.SysProcAttr{
-			Foreground: true,
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd":
+		if !ia.hasTTY {
+			dlogf("can't use login because of missing TTY")
+			// We can only use the login command if a shell was requested with
+			// a TTY. If there is no TTY, login exits immediately, which
+			// breaks things like mosh and VSCode.
+			return nil
 		}
 	}
-	err = cmd.Run()
+
+	loginCmdPath, err := exec.LookPath("login")
+	if err != nil {
+		dlogf("failed to get login args: %s", err)
+		return nil
+	}
+	loginArgs := ia.loginArgs(loginCmdPath)
+	dlogf("logging in with %s %+v", loginCmdPath, loginArgs)
+	// replace the running process
+	return unix.Exec(loginCmdPath, loginArgs, os.Environ())
+}
+
+// trySU attempts to start a login shell using su. If su is available and
+// supports the necessary arguments, this returns true, plus the result of
+// executing su. Otherwise, it returns (false, nil).
+//
+// Creating a login shell in this way allows us to trigger PAM authentication
+// and get the "login" PAM profile.
+//
+// Unlike login, su often does not require a TTY, so on Linux hosts that have
+// an su command which accepts the right flags, we'll use su instead of login
+// when no TTY is available.
+func trySU(dlogf logger.Logf, ia incubatorArgs) (handled bool, err error) {
+	if ia.forceV1Behavior {
+		// v1 behavior did not use su.
+		dlogf("Forcing v1 behavior, won't use su")
+		return false, nil
+	}
+
+	su := findSU(dlogf, ia)
+	if su == "" {
+		return false, nil
+	}
+
+	sessionCloser := maybeStartLoginSession(dlogf, ia)
+	if sessionCloser != nil {
+		defer sessionCloser()
+	}
+
+	loginArgs := []string{"-l", ia.localUser}
+	if ia.cmd != "" {
+		// Note - unlike the login command, su allows using both -l and -c.
+		loginArgs = append(loginArgs, "-c", ia.cmd)
+	}
+
+	dlogf("logging in with %s %q", su, loginArgs)
+	cmd := newCommand(ia.hasTTY, su, loginArgs)
+	return true, cmd.Run()
+}
+
+// findSU attempts to find an su command which supports the -l and -c flags.
+// This actually calls the su command, which can cause side effects like
+// triggering pam_mkhomedir. If a suitable su is not available, this returns
+// "".
+func findSU(dlogf logger.Logf, ia incubatorArgs) string {
+	// Currently, we only support falling back to su on Linux. This
+	// potentially could work on BSDs as well, but requires testing.
+	if runtime.GOOS != "linux" {
+		return ""
+	}
+
+	// gokrazy doesn't include su. And, if someone installs a breakglass/
+	// debugging package on gokrazy, we don't want to use its su.
+	if distro.Get() == distro.Gokrazy {
+		return ""
+	}
+
+	su, err := exec.LookPath("su")
+	if err != nil {
+		dlogf("can't find su command: %v", err)
+		return ""
+	}
+
+	// First try to execute su -l <user> -c true to make sure su supports the
+	// necessary arguments.
+	err = exec.Command(su, "-l", ia.localUser, "-c", "true").Run()
+	if err != nil {
+		dlogf("su check failed: %s", err)
+		return ""
+	}
+
+	return su
+}
+
+// handleSSHInProcess is a last resort if we couldn't use login or su. It
+// registers a new session with the OS, sets its UID, GID and groups to the
+// specified values, and then launches the requested `--cmd` in the user's
+// login shell.
+func handleSSHInProcess(dlogf logger.Logf, ia incubatorArgs) error {
+	sessionCloser := maybeStartLoginSession(dlogf, ia)
+	if sessionCloser != nil {
+		defer sessionCloser()
+	}
+
+	if err := dropPrivileges(dlogf, ia); err != nil {
+		return err
+	}
+
+	args := shellArgs(ia.isShell, ia.cmd)
+	dlogf("running %s %q", ia.loginShell, args)
+	cmd := newCommand(ia.hasTTY, ia.loginShell, args)
+	err := cmd.Run()
 	if ee, ok := err.(*exec.ExitError); ok {
 		ps := ee.ProcessState
 		code := ps.ExitCode()
@@ -330,6 +500,26 @@ func beIncubator(args []string) error {
 	return err
 }
 
+func newCommand(hasTTY bool, cmdPath string, cmdArgs []string) *exec.Cmd {
+	cmd := exec.Command(cmdPath, cmdArgs...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+
+	if hasTTY {
+		// If we were launched with a tty then we should mark that as the ctty
+		// of the child. However, as the ctty is being passed from the parent
+		// we set the child to foreground instead which also passes the ctty.
+		// However, we can not do this if never had a tty to begin with.
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Foreground: true,
+		}
+	}
+
+	return cmd
+}
+
 const (
 	// This controls whether we assert that our privileges were dropped
 	// using geteuid/getegid; it's a const and not an envknob because the
@@ -344,19 +534,26 @@ const (
 	assertPrivilegesWereDroppedByAttemptingToUnDrop = false
 )
 
-// dropPrivileges contains all the logic for dropping privileges to a different
+// dropPrivileges calls doDropPrivileges with uid, gid, and gids from the given
+// incubatorArgs.
+func dropPrivileges(dlogf logger.Logf, ia incubatorArgs) error {
+	return doDropPrivileges(dlogf, ia.uid, ia.gid, ia.gids)
+}
+
+// doDropPrivileges contains all the logic for dropping privileges to a different
 // UID, GID, and set of supplementary groups. This function is
 // security-sensitive and ordering-dependent; please be very cautious if/when
 // refactoring.
 //
-// WARNING: if you change this function, you *MUST* run the TestDropPrivileges
+// WARNING: if you change this function, you *MUST* run the TestDoDropPrivileges
 // test in this package as root on at least Linux, FreeBSD and Darwin. This can
 // be done by running:
 //
-//	go test -c ./ssh/tailssh/ && sudo ./tailssh.test -test.v -test.run TestDropPrivileges
-func dropPrivileges(logf logger.Logf, wantUid, wantGid int, supplementaryGroups []int) error {
+//	go test -c ./ssh/tailssh/ && sudo ./tailssh.test -test.v -test.run TestDoDropPrivileges
+func doDropPrivileges(dlogf logger.Logf, wantUid, wantGid int, supplementaryGroups []int) error {
+	dlogf("dropping privileges")
 	fatalf := func(format string, args ...any) {
-		logf("[unexpected] error dropping privileges: "+format, args...)
+		dlogf("[unexpected] error dropping privileges: "+format, args...)
 		os.Exit(1)
 	}
 
@@ -448,7 +645,11 @@ func dropPrivileges(logf logger.Logf, wantUid, wantGid int, supplementaryGroups
 //
 // It sets ss.cmd, stdin, stdout, and stderr.
 func (ss *sshSession) launchProcess() error {
-	ss.cmd = ss.newIncubatorCommand()
+	var err error
+	ss.cmd, err = ss.newIncubatorCommand(ss.logf)
+	if err != nil {
+		return err
+	}
 
 	cmd := ss.cmd
 	homeDir := ss.conn.localUser.HomeDir
@@ -749,18 +950,11 @@ func fileExists(path string) bool {
 }
 
 // loginArgs returns the arguments to use to exec the login binary.
-// It returns nil if the login binary should not be used.
-// The login binary is only used:
-//   - on darwin, if the client is requesting a shell or a command.
-//   - on linux and BSD, if the client is requesting a shell with a TTY.
-func (ia *incubatorArgs) loginArgs() []string {
-	if ia.isSFTP {
-		return nil
-	}
+func (ia *incubatorArgs) loginArgs(loginCmdPath string) []string {
 	switch runtime.GOOS {
 	case "darwin":
 		args := []string{
-			ia.loginCmdPath,
+			loginCmdPath,
 			"-f", // already authenticated
 
 			// login typically discards the previous environment, but we want to
@@ -773,39 +967,35 @@ func (ia *incubatorArgs) loginArgs() []string {
 		if !ia.hasTTY {
 			args[2] = "-pq" // -q is "quiet" which suppresses the login banner
 		}
-		if ia.cmdName != "" {
-			args = append(args, ia.cmdName)
-			args = append(args, ia.cmdArgs...)
+		if ia.cmd != "" {
+			args = append(args, ia.loginShell, "-c", ia.cmd)
 		}
+
 		return args
 	case "linux":
-		if !ia.isShell || !ia.hasTTY {
-			// We can only use login command if a shell was requested with a TTY. If
-			// there is no TTY, login exits immediately, which breaks things likes
-			// mosh and VSCode.
-			return nil
-		}
 		if distro.Get() == distro.Arch && !fileExists("/etc/pam.d/remote") {
 			// See https://github.com/tailscale/tailscale/issues/4924
 			//
 			// Arch uses a different login binary that makes the -h flag set the PAM
 			// service to "remote". So if they don't have that configured, don't
 			// pass -h.
-			return []string{ia.loginCmdPath, "-f", ia.localUser, "-p"}
+			return []string{loginCmdPath, "-f", ia.localUser, "-p"}
 		}
-		return []string{ia.loginCmdPath, "-f", ia.localUser, "-h", ia.remoteIP, "-p"}
+		return []string{loginCmdPath, "-f", ia.localUser, "-h", ia.remoteIP, "-p"}
 	case "freebsd", "openbsd":
-		if !ia.isShell || !ia.hasTTY {
-			// We can only use login command if a shell was requested with a TTY. If
-			// there is no TTY, login exits immediately, which breaks things likes
-			// mosh and VSCode.
-			return nil
-		}
-		return []string{ia.loginCmdPath, "-fp", "-h", ia.remoteIP, ia.localUser}
+		return []string{loginCmdPath, "-fp", "-h", ia.remoteIP, ia.localUser}
 	}
 	panic("unimplemented")
 }
 
+func shellArgs(isShell bool, cmd string) []string {
+	if isShell {
+		return []string{"-l"}
+	} else {
+		return []string{"-c", cmd}
+	}
+}
+
 func setGroups(groupIDs []int) error {
 	if runtime.GOOS == "darwin" && len(groupIDs) > 16 {
 		// darwin returns "invalid argument" if more than 16 groups are passed to syscall.Setgroups

ssh/tailssh/incubator_linux.go

@@ -146,11 +146,11 @@ func releaseSession(sessionID string) error {
 }
 
 // maybeStartLoginSessionLinux is the linux implementation of maybeStartLoginSession.
-func maybeStartLoginSessionLinux(logf logger.Logf, ia incubatorArgs) (func() error, error) {
+func maybeStartLoginSessionLinux(dlogf logger.Logf, ia incubatorArgs) func() error {
 	if os.Geteuid() != 0 {
-		return nil, nil
+		return nil
 	}
-	logf("starting session for user %d", ia.uid)
+	dlogf("starting session for user %d", ia.uid)
 	// The only way we can actually start a new session is if we are
 	// running outside one and are root, which is typically the case
 	// for systemd managed tailscaled.
@@ -160,14 +160,14 @@ func maybeStartLoginSessionLinux(logf logger.Logf, ia incubatorArgs) (func() err
 		// We can look at the DBus GetSessionByPID API.
 		// https://www.freedesktop.org/software/systemd/man/org.freedesktop.login1.html
 		// For now best effort is fine.
-		logf("ssh: failed to CreateSession for user %q (%d) %v", ia.localUser, ia.uid, err)
-		return nil, nil
+		dlogf("ssh: failed to CreateSession for user %q (%d) %v", ia.localUser, ia.uid, err)
+		return nil
 	}
 	os.Setenv("DBUS_SESSION_BUS_ADDRESS", fmt.Sprintf("unix:path=%v/bus", resp.runtimePath))
 	if !resp.existing {
 		return func() error {
 			return releaseSession(resp.sessionID)
-		}, nil
+		}
 	}
-	return nil, nil
+	return nil
 }

ssh/tailssh/privs_test.go

@@ -23,7 +23,7 @@ import (
 	"tailscale.com/types/logger"
 )
 
-func TestDropPrivileges(t *testing.T) {
+func TestDoDropPrivileges(t *testing.T) {
 	type SubprocInput struct {
 		UID              int
 		GID              int
@@ -49,7 +49,7 @@ func TestDropPrivileges(t *testing.T) {
 		f := os.NewFile(3, "out.json")
 
 		// We're in our subprocess; actually drop privileges now.
-		dropPrivileges(t.Logf, input.UID, input.GID, input.AdditionalGroups)
+		doDropPrivileges(t.Logf, input.UID, input.GID, input.AdditionalGroups)
 
 		additional, _ := syscall.Getgroups()
 

ssh/tailssh/tailssh_integration_test.go

@@ -8,6 +8,7 @@ package tailssh
 
 import (
 	"bufio"
+	"context"
 	"crypto/ecdsa"
 	"crypto/ed25519"
 	"crypto/elliptic"
@@ -28,6 +29,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/bramvdbogaerde/go-scp"
 	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/sftp"
 	gossh "github.com/tailscale/golang-x-crypto/ssh"
@@ -36,6 +38,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/set"
 )
 
 // This file contains integration tests of the SSH functionality. These tests
@@ -58,7 +61,7 @@ func TestMain(m *testing.M) {
 	file.Close()
 
 	// Tail our log file.
-	cmd := exec.Command("tail", "-f", "/tmp/tailscalessh.log")
+	cmd := exec.Command("tail", "-F", "/tmp/tailscalessh.log")
 
 	r, err := cmd.StdoutPipe()
 	if err != nil {
@@ -77,6 +80,12 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		return
 	}
+	defer func() {
+		// tail -f has a default sleep interval of 1 second, so it takes a
+		// moment for it to finish reading our log file after we've terminated.
+		// So, wait a bit to let it catch up.
+		time.Sleep(2 * time.Second)
+	}()
 
 	m.Run()
 }
@@ -93,20 +102,40 @@ func TestIntegrationSSH(t *testing.T) {
 	}
 
 	tests := []struct {
-		cmd  string
-		want []string
+		cmd             string
+		want            []string
+		forceV1Behavior bool
+		skip            bool
 	}{
 		{
-			cmd:  "id",
-			want: []string{"testuser", "groupone", "grouptwo"},
+			cmd:             "id",
+			want:            []string{"testuser", "groupone", "grouptwo"},
+			forceV1Behavior: false,
 		},
 		{
-			cmd:  "pwd",
-			want: []string{homeDir},
+			cmd:             "id",
+			want:            []string{"testuser", "groupone", "grouptwo"},
+			forceV1Behavior: true,
+		},
+		{
+			cmd:             "pwd",
+			want:            []string{homeDir},
+			skip:            !fallbackToSUAvailable(),
+			forceV1Behavior: false,
+		},
+		{
+			cmd:             "echo 'hello'",
+			want:            []string{"hello"},
+			skip:            !fallbackToSUAvailable(),
+			forceV1Behavior: false,
 		},
 	}
 
 	for _, test := range tests {
+		if test.skip {
+			continue
+		}
+
 		// run every test both without and with a shell
 		for _, shell := range []bool{false, true} {
 			shellQualifier := "no_shell"
@@ -114,8 +143,13 @@ func TestIntegrationSSH(t *testing.T) {
 				shellQualifier = "shell"
 			}
 
-			t.Run(fmt.Sprintf("%s_%s", test.cmd, shellQualifier), func(t *testing.T) {
-				s := testSession(t)
+			versionQualifier := "v2"
+			if test.forceV1Behavior {
+				versionQualifier = "v1"
+			}
+
+			t.Run(fmt.Sprintf("%s_%s_%s", test.cmd, shellQualifier, versionQualifier), func(t *testing.T) {
+				s := testSession(t, test.forceV1Behavior)
 
 				if shell {
 					err := s.RequestPty("xterm", 40, 80, ssh.TerminalModes{
@@ -123,12 +157,20 @@ func TestIntegrationSSH(t *testing.T) {
 						ssh.TTY_OP_ISPEED: 14400,
 						ssh.TTY_OP_OSPEED: 14400,
 					})
+					if err != nil {
+						t.Fatalf("unable to request PTY: %s", err)
+					}
+
+					err = s.Shell()
 					if err != nil {
 						t.Fatalf("unable to request shell: %s", err)
 					}
+
+					// Read the shell prompt
+					s.read()
 				}
 
-				got := s.run(t, test.cmd)
+				got := s.run(t, test.cmd, shell)
 				for _, want := range test.want {
 					if !strings.Contains(got, want) {
 						t.Errorf("%q does not contain %q", got, want)
@@ -145,48 +187,133 @@ func TestIntegrationSFTP(t *testing.T) {
 		debugTest.Store(false)
 	})
 
-	filePath := "/tmp/sftptest.dat"
-	wantText := "hello world"
+	for _, forceV1Behavior := range []bool{false, true} {
+		name := "v2"
+		if forceV1Behavior {
+			name = "v1"
+		}
+		t.Run(name, func(t *testing.T) {
+			filePath := "/home/testuser/sftptest.dat"
+			if forceV1Behavior || !fallbackToSUAvailable() {
+				filePath = "/tmp/sftptest.dat"
+			}
+			wantText := "hello world"
 
-	cl := testClient(t)
-	scl, err := sftp.NewClient(cl)
-	if err != nil {
-		t.Fatalf("can't get sftp client: %s", err)
+			cl := testClient(t, forceV1Behavior)
+			scl, err := sftp.NewClient(cl)
+			if err != nil {
+				t.Fatalf("can't get sftp client: %s", err)
+			}
+
+			file, err := scl.Create(filePath)
+			if err != nil {
+				t.Fatalf("can't create file: %s", err)
+			}
+			_, err = file.Write([]byte(wantText))
+			if err != nil {
+				t.Fatalf("can't write to file: %s", err)
+			}
+			err = file.Close()
+			if err != nil {
+				t.Fatalf("can't close file: %s", err)
+			}
+
+			file, err = scl.OpenFile(filePath, os.O_RDONLY)
+			if err != nil {
+				t.Fatalf("can't open file: %s", err)
+			}
+			defer file.Close()
+			gotText, err := io.ReadAll(file)
+			if err != nil {
+				t.Fatalf("can't read file: %s", err)
+			}
+			if diff := cmp.Diff(string(gotText), wantText); diff != "" {
+				t.Fatalf("unexpected file contents (-got +want):\n%s", diff)
+			}
+
+			s := testSessionFor(t, cl)
+			got := s.run(t, "ls -l "+filePath, false)
+			if !strings.Contains(got, "testuser") {
+				t.Fatalf("unexpected file owner user: %s", got)
+			} else if !strings.Contains(got, "testuser") {
+				t.Fatalf("unexpected file owner group: %s", got)
+			}
+		})
+	}
+}
+
+func TestIntegrationSCP(t *testing.T) {
+	debugTest.Store(true)
+	t.Cleanup(func() {
+		debugTest.Store(false)
+	})
+
+	for _, forceV1Behavior := range []bool{false, true} {
+		name := "v2"
+		if forceV1Behavior {
+			name = "v1"
+		}
+		t.Run(name, func(t *testing.T) {
+			filePath := "/home/testuser/scptest.dat"
+			if !fallbackToSUAvailable() {
+				filePath = "/tmp/scptest.dat"
+			}
+			wantText := "hello world"
+
+			cl := testClient(t, forceV1Behavior)
+			scl, err := scp.NewClientBySSH(cl)
+			if err != nil {
+				t.Fatalf("can't get sftp client: %s", err)
+			}
+
+			err = scl.Copy(context.Background(), strings.NewReader(wantText), filePath, "0644", int64(len(wantText)))
+			if err != nil {
+				t.Fatalf("can't create file: %s", err)
+			}
+
+			outfile, err := os.CreateTemp("", "")
+			if err != nil {
+				t.Fatalf("can't create temp file: %s", err)
+			}
+			err = scl.CopyFromRemote(context.Background(), outfile, filePath)
+			if err != nil {
+				t.Fatalf("can't copy file from remote: %s", err)
+			}
+			outfile.Close()
+
+			gotText, err := os.ReadFile(outfile.Name())
+			if err != nil {
+				t.Fatalf("can't read file: %s", err)
+			}
+			if diff := cmp.Diff(string(gotText), wantText); diff != "" {
+				t.Fatalf("unexpected file contents (-got +want):\n%s", diff)
+			}
+
+			s := testSessionFor(t, cl)
+			got := s.run(t, "ls -l "+filePath, false)
+			if !strings.Contains(got, "testuser") {
+				t.Fatalf("unexpected file owner user: %s", got)
+			} else if !strings.Contains(got, "testuser") {
+				t.Fatalf("unexpected file owner group: %s", got)
+			}
+		})
+	}
+}
+
+func fallbackToSUAvailable() bool {
+	if runtime.GOOS != "linux" {
+		return false
 	}
 
-	file, err := scl.Create(filePath)
+	_, err := exec.LookPath("su")
 	if err != nil {
-		t.Fatalf("can't create file: %s", err)
-	}
-	_, err = file.Write([]byte(wantText))
-	if err != nil {
-		t.Fatalf("can't write to file: %s", err)
-	}
-	err = file.Close()
-	if err != nil {
-		t.Fatalf("can't close file: %s", err)
+		return false
 	}
 
-	file, err = scl.OpenFile(filePath, os.O_RDONLY)
-	if err != nil {
-		t.Fatalf("can't open file: %s", err)
-	}
-	defer file.Close()
-	gotText, err := io.ReadAll(file)
-	if err != nil {
-		t.Fatalf("can't read file: %s", err)
-	}
-	if diff := cmp.Diff(string(gotText), wantText); diff != "" {
-		t.Fatalf("unexpected file contents (-got +want):\n%s", diff)
-	}
-
-	s := testSessionFor(t, cl)
-	got := s.run(t, "ls -l "+filePath)
-	if !strings.Contains(got, "testuser") {
-		t.Fatalf("unexpected file owner user: %s", got)
-	} else if !strings.Contains(got, "testuser") {
-		t.Fatalf("unexpected file owner group: %s", got)
-	}
+	// Some operating systems like Fedora seem to require login to be present
+	// in order for su to work.
+	_, err = exec.LookPath("login")
+	return err == nil
 }
 
 type session struct {
@@ -197,14 +324,25 @@ type session struct {
 	stderr io.ReadCloser
 }
 
-func (s *session) run(t *testing.T, cmdString string) string {
+func (s *session) run(t *testing.T, cmdString string, shell bool) string {
 	t.Helper()
 
-	err := s.Start(cmdString)
-	if err != nil {
-		t.Fatalf("unable to start command: %s", err)
+	if shell {
+		_, err := s.stdin.Write([]byte(fmt.Sprintf("%s\n", cmdString)))
+		if err != nil {
+			t.Fatalf("unable to send command to shell: %s", err)
+		}
+	} else {
+		err := s.Start(cmdString)
+		if err != nil {
+			t.Fatalf("unable to start command: %s", err)
+		}
 	}
 
+	return s.read()
+}
+
+func (s *session) read() string {
 	ch := make(chan []byte)
 	go func() {
 		for {
@@ -228,7 +366,7 @@ readLoop:
 		select {
 		case b := <-ch:
 			_got = append(_got, b...)
-		case <-time.After(25 * time.Millisecond):
+		case <-time.After(1 * time.Second):
 			break readLoop
 		}
 	}
@@ -236,12 +374,12 @@ readLoop:
 	return string(_got)
 }
 
-func testClient(t *testing.T) *ssh.Client {
+func testClient(t *testing.T, forceV1Behavior bool) *ssh.Client {
 	t.Helper()
 
 	username := "testuser"
 	srv := &server{
-		lb:             &testBackend{localUser: username},
+		lb:             &testBackend{localUser: username, forceV1Behavior: forceV1Behavior},
 		logf:           log.Printf,
 		tailscaledPath: os.Getenv("TAILSCALED_PATH"),
 		timeNow:        time.Now,
@@ -271,8 +409,8 @@ func testClient(t *testing.T) *ssh.Client {
 	return cl
 }
 
-func testSession(t *testing.T) *session {
-	cl := testClient(t)
+func testSession(t *testing.T, forceV1Behavior bool) *session {
+	cl := testClient(t, forceV1Behavior)
 	return testSessionFor(t, cl)
 }
 
@@ -299,7 +437,8 @@ func testSessionFor(t *testing.T, cl *ssh.Client) *session {
 
 // testBackend implements ipnLocalBackend
 type testBackend struct {
-	localUser string
+	localUser       string
+	forceV1Behavior bool
 }
 
 func (tb *testBackend) GetSSH_HostKeys() ([]gossh.Signer, error) {
@@ -339,16 +478,21 @@ func (tb *testBackend) ShouldRunSSH() bool {
 }
 
 func (tb *testBackend) NetMap() *netmap.NetworkMap {
+	capMap := make(set.Set[tailcfg.NodeCapability])
+	if tb.forceV1Behavior {
+		capMap[tailcfg.NodeAttrSSHBehaviorV1] = struct{}{}
+	}
 	return &netmap.NetworkMap{
 		SSHPolicy: &tailcfg.SSHPolicy{
 			Rules: []*tailcfg.SSHRule{
-				&tailcfg.SSHRule{
+				{
 					Principals: []*tailcfg.SSHPrincipal{{Any: true}},
 					Action:     &tailcfg.SSHAction{Accept: true},
 					SSHUsers:   map[string]string{"*": tb.localUser},
 				},
 			},
 		},
+		AllCaps: capMap,
 	}
 }
 

ssh/tailssh/testcontainers/Dockerfile

@@ -1,18 +1,51 @@
 ARG BASE
 FROM ${BASE}
 
+RUN echo "Install openssh, needed for scp."
+RUN apt-get update -y && apt-get install -y openssh-client
+
 RUN groupadd -g 10000 groupone
 RUN groupadd -g 10001 grouptwo
-RUN useradd -g 10000 -G 10001 -u 10002 -m testuser
-COPY . .
+# Note - we do not create the user's home directory, pam_mkhomedir will do that
+# for us, and we want to test that PAM gets triggered by Tailscale SSH.
+RUN useradd -g 10000 -G 10001 -u 10002 testuser
 
-# First run tests normally.
-RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.run TestIntegration
+RUN echo "Set up pam_mkhomedir."
+RUN sed -i -e 's/Default: no/Default: yes/g' /usr/share/pam-configs/mkhomedir || echo "might not be ubuntu"
+RUN cat /usr/share/pam-configs/mkhomedir
+RUN pam-auth-update --enable mkhomedir
 
-# Then remove the login command and make sure tests still pass.
-RUN rm `which login`
-RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.run TestIntegration
+COPY tailscaled .
+COPY tailssh.test .
 
-# Then run tests as non-root user testuser.
+RUN chmod 755 tailscaled
+
+RUN echo "First run tests normally."
+RUN rm -Rf /home/testuser
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegrationSFTP
+RUN rm -Rf /home/testuser
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegrationSCP
+RUN rm -Rf /home/testuser
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegrationSSH
+
+RUN echo "Then run tests as non-root user testuser and make sure tests still pass."
 RUN chown testuser:groupone /tmp/tailscalessh.log
-RUN TAILSCALED_PATH=`pwd`tailscaled su -m testuser -c "./tailssh.test -test.run TestIntegration"
+RUN TAILSCALED_PATH=`pwd`tailscaled su -m testuser -c "./tailssh.test -test.v -test.run TestIntegration TestDoDropPrivileges"
+
+RUN echo "Then remove the login command and make sure tests still pass."
+RUN chown root:root /tmp/tailscalessh.log
+RUN rm `which login`
+RUN rm -Rf /home/testuser
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegrationSFTP
+RUN rm -Rf /home/testuser
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegrationSCP
+RUN rm -Rf /home/testuser
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegrationSSH
+
+RUN echo "Then remove the su command and make sure tests still pass."
+RUN chown root:root /tmp/tailscalessh.log
+RUN rm `which su`
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestIntegration
+
+RUN echo "Test doDropPrivileges"
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.v -test.run TestDoDropPrivileges

tailcfg/tailcfg.go

@@ -136,7 +136,8 @@ type CapabilityVersion int
 //   - 93: 2024-05-06: added support for stateful firewalling.
 //   - 94: 2024-05-06: Client understands Node.IsJailed.
 //   - 95: 2024-05-06: Client uses NodeAttrUserDialUseRoutes to change DNS dialing behavior.
-const CurrentCapabilityVersion CapabilityVersion = 95
+//   - 96: 2024-05-29: Client understands NodeAttrSSHBehaviorV1
+const CurrentCapabilityVersion CapabilityVersion = 96
 
 type StableID string
 
@@ -608,10 +609,11 @@ func isAlpha(b byte) bool {
 //
 // We might relax these rules later.
 func CheckTag(tag string) error {
-	if !strings.HasPrefix(tag, "tag:") {
+	var ok bool
+	tag, ok = strings.CutPrefix(tag, "tag:")
+	if !ok {
 		return errors.New("tags must start with 'tag:'")
 	}
-	tag = tag[4:]
 	if tag == "" {
 		return errors.New("tag names must not be empty")
 	}
@@ -2274,6 +2276,10 @@ const (
 	// depending on the destination address and the configured routes. When present, it also makes
 	// the DNS forwarder use UserDial instead of SystemDial when dialing resolvers.
 	NodeAttrUserDialUseRoutes NodeCapability = "user-dial-routes"
+
+	// NodeAttrSSHBehaviorV1 forces SSH to use the V1 behavior (no su, run SFTP in-process)
+	// Added 2024-05-29 in Tailscale version 1.68.
+	NodeAttrSSHBehaviorV1 NodeCapability = "ssh-behavior-v1"
 )
 
 // SetDNSRequest is a request to add a DNS record.

tailcfg/tailcfg_test.go

@@ -859,8 +859,33 @@ func TestDeps(t *testing.T) {
 		BadDeps: map[string]string{
 			// Make sure we don't again accidentally bring in a dependency on
 			// drive or its transitive dependencies
+			"testing":                        "do not use testing package in production code",
 			"tailscale.com/drive/driveimpl":  "https://github.com/tailscale/tailscale/pull/10631",
 			"github.com/studio-b12/gowebdav": "https://github.com/tailscale/tailscale/pull/10631",
 		},
 	}.Check(t)
 }
+
+func TestCheckTag(t *testing.T) {
+	tests := []struct {
+		name string
+		tag  string
+		want bool
+	}{
+		{"empty", "", false},
+		{"good", "tag:foo", true},
+		{"bad", "tag:", false},
+		{"no_leading_num", "tag:1foo", false},
+		{"no_punctuation", "tag:foa@bar", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := CheckTag(tt.tag)
+			if err == nil && !tt.want {
+				t.Errorf("got nil; want error")
+			} else if err != nil && tt.want {
+				t.Errorf("got %v; want nil", err)
+			}
+		})
+	}
+}

tka/sig.go

@@ -8,6 +8,7 @@ import (
 	"crypto/ed25519"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/fxamacker/cbor/v2"
 	"github.com/hdevalence/ed25519consensus"
@@ -96,6 +97,41 @@ type NodeKeySignature struct {
 	WrappingPubkey []byte `cbor:"6,keyasint,omitempty"`
 }
 
+// String returns a human-readable representation of the NodeKeySignature,
+// making it easy to see nested signatures.
+func (s NodeKeySignature) String() string {
+	var b strings.Builder
+	var addToBuf func(NodeKeySignature, int)
+	addToBuf = func(sig NodeKeySignature, depth int) {
+		indent := strings.Repeat("  ", depth)
+		b.WriteString(indent + "SigKind: " + sig.SigKind.String() + "\n")
+		if len(sig.Pubkey) > 0 {
+			var pubKey string
+			var np key.NodePublic
+			if err := np.UnmarshalBinary(sig.Pubkey); err != nil {
+				pubKey = fmt.Sprintf("<error: %s>", err)
+			} else {
+				pubKey = np.ShortString()
+			}
+			b.WriteString(indent + "Pubkey: " + pubKey + "\n")
+		}
+		if len(sig.KeyID) > 0 {
+			keyID := key.NLPublicFromEd25519Unsafe(sig.KeyID).CLIString()
+			b.WriteString(indent + "KeyID: " + keyID + "\n")
+		}
+		if len(sig.WrappingPubkey) > 0 {
+			pubKey := key.NLPublicFromEd25519Unsafe(sig.WrappingPubkey).CLIString()
+			b.WriteString(indent + "WrappingPubkey: " + pubKey + "\n")
+		}
+		if sig.Nested != nil {
+			b.WriteString(indent + "Nested:\n")
+			addToBuf(*sig.Nested, depth+1)
+		}
+	}
+	addToBuf(s, 0)
+	return strings.TrimSpace(b.String())
+}
+
 // UnverifiedWrappingPublic returns the public key which must sign a
 // signature which embeds this one, if any.
 //
@@ -268,3 +304,78 @@ func (s *NodeKeySignature) verifySignature(nodeKey key.NodePublic, verificationK
 		return fmt.Errorf("unhandled signature type: %v", s.SigKind)
 	}
 }
+
+// RotationDetails holds additional information about a nodeKeySignature
+// of kind SigRotation.
+type RotationDetails struct {
+	// PrevNodeKeys is a list of node keys which have been rotated out.
+	PrevNodeKeys []key.NodePublic
+
+	// WrappingPubkey is the public key which has been authorized to sign
+	// this rotating signature.
+	WrappingPubkey []byte
+}
+
+// rotationDetails returns the RotationDetails for a SigRotation signature.
+func (s *NodeKeySignature) rotationDetails() (*RotationDetails, error) {
+	if s.SigKind != SigRotation {
+		return nil, nil
+	}
+
+	sri := &RotationDetails{}
+	nested := s.Nested
+	for nested != nil {
+		if len(nested.Pubkey) > 0 {
+			var nestedPub key.NodePublic
+			if err := nestedPub.UnmarshalBinary(nested.Pubkey); err != nil {
+				return nil, fmt.Errorf("nested pubkey: %v", err)
+			}
+			sri.PrevNodeKeys = append(sri.PrevNodeKeys, nestedPub)
+		}
+		if nested.SigKind != SigRotation {
+			break
+		}
+		nested = nested.Nested
+	}
+	sri.WrappingPubkey = nested.WrappingPubkey
+	return sri, nil
+}
+
+// ResignNKS re-signs a node-key signature for a new node-key.
+//
+// This only matters on network-locked tailnets, because node-key signatures are
+// how other nodes know that a node-key is authentic. When the node-key is
+// rotated then the existing signature becomes invalid, so this function is
+// responsible for generating a new wrapping signature to certify the new node-key.
+//
+// The signature itself is a SigRotation signature, which embeds the old signature
+// and certifies the new node-key as a replacement for the old by signing the new
+// signature with RotationPubkey (which is the node's own network-lock key).
+func ResignNKS(priv key.NLPrivate, nodeKey key.NodePublic, oldNKS tkatype.MarshaledSignature) (tkatype.MarshaledSignature, error) {
+	var oldSig NodeKeySignature
+	if err := oldSig.Unserialize(oldNKS); err != nil {
+		return nil, fmt.Errorf("decoding NKS: %w", err)
+	}
+
+	nk, err := nodeKey.MarshalBinary()
+	if err != nil {
+		return nil, fmt.Errorf("marshalling node-key: %w", err)
+	}
+
+	if bytes.Equal(nk, oldSig.Pubkey) {
+		// The old signature is valid for the node-key we are using, so just
+		// use it verbatim.
+		return oldNKS, nil
+	}
+
+	newSig := NodeKeySignature{
+		SigKind: SigRotation,
+		Pubkey:  nk,
+		Nested:  &oldSig,
+	}
+	if newSig.Signature, err = priv.SignNKS(newSig.SigHash()); err != nil {
+		return nil, fmt.Errorf("signing NKS: %w", err)
+	}
+
+	return newSig.Serialize(), nil
+}

tka/sig_test.go

@@ -5,6 +5,7 @@ package tka
 
 import (
 	"crypto/ed25519"
+	"reflect"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -298,3 +299,143 @@ func TestSigSerializeUnserialize(t *testing.T) {
 		t.Errorf("unmarshalled version differs (-want, +got):\n%s", diff)
 	}
 }
+
+func TestNodeKeySignatureRotationDetails(t *testing.T) {
+	// Trusted network lock key
+	pub, priv := testingKey25519(t, 1)
+	k := Key{Kind: Key25519, Public: pub, Votes: 2}
+
+	// 'credential' key (the one being delegated to)
+	cPub, cPriv := testingKey25519(t, 2)
+
+	n1, n2, n3 := key.NewNode(), key.NewNode(), key.NewNode()
+	n1pub, _ := n1.Public().MarshalBinary()
+	n2pub, _ := n2.Public().MarshalBinary()
+	n3pub, _ := n3.Public().MarshalBinary()
+
+	tests := []struct {
+		name    string
+		nodeKey key.NodePublic
+		sigFn   func() NodeKeySignature
+		want    *RotationDetails
+	}{
+		{
+			name:    "SigDirect",
+			nodeKey: n1.Public(),
+			sigFn: func() NodeKeySignature {
+				s := NodeKeySignature{
+					SigKind: SigDirect,
+					KeyID:   pub,
+					Pubkey:  n1pub,
+				}
+				sigHash := s.SigHash()
+				s.Signature = ed25519.Sign(priv, sigHash[:])
+				return s
+			},
+			want: nil,
+		},
+		{
+			name:    "SigWrappedCredential",
+			nodeKey: n1.Public(),
+			sigFn: func() NodeKeySignature {
+				nestedSig := NodeKeySignature{
+					SigKind:        SigCredential,
+					KeyID:          pub,
+					WrappingPubkey: cPub,
+				}
+				sigHash := nestedSig.SigHash()
+				nestedSig.Signature = ed25519.Sign(priv, sigHash[:])
+
+				sig := NodeKeySignature{
+					SigKind: SigRotation,
+					Pubkey:  n1pub,
+					Nested:  &nestedSig,
+				}
+				sigHash = sig.SigHash()
+				sig.Signature = ed25519.Sign(cPriv, sigHash[:])
+				return sig
+			},
+			want: &RotationDetails{
+				WrappingPubkey: cPub,
+			},
+		},
+		{
+			name:    "SigRotation",
+			nodeKey: n2.Public(),
+			sigFn: func() NodeKeySignature {
+				nestedSig := NodeKeySignature{
+					SigKind:        SigDirect,
+					Pubkey:         n1pub,
+					KeyID:          pub,
+					WrappingPubkey: cPub,
+				}
+				sigHash := nestedSig.SigHash()
+				nestedSig.Signature = ed25519.Sign(priv, sigHash[:])
+
+				sig := NodeKeySignature{
+					SigKind: SigRotation,
+					Pubkey:  n2pub,
+					Nested:  &nestedSig,
+				}
+				sigHash = sig.SigHash()
+				sig.Signature = ed25519.Sign(cPriv, sigHash[:])
+				return sig
+			},
+			want: &RotationDetails{
+				WrappingPubkey: cPub,
+				PrevNodeKeys:   []key.NodePublic{n1.Public()},
+			},
+		},
+		{
+			name:    "SigRotationNestedTwice",
+			nodeKey: n3.Public(),
+			sigFn: func() NodeKeySignature {
+				initialSig := NodeKeySignature{
+					SigKind:        SigDirect,
+					Pubkey:         n1pub,
+					KeyID:          pub,
+					WrappingPubkey: cPub,
+				}
+				sigHash := initialSig.SigHash()
+				initialSig.Signature = ed25519.Sign(priv, sigHash[:])
+
+				prevRotation := NodeKeySignature{
+					SigKind: SigRotation,
+					Pubkey:  n2pub,
+					Nested:  &initialSig,
+				}
+				sigHash = prevRotation.SigHash()
+				prevRotation.Signature = ed25519.Sign(cPriv, sigHash[:])
+
+				sig := NodeKeySignature{
+					SigKind: SigRotation,
+					Pubkey:  n3pub,
+					Nested:  &prevRotation,
+				}
+				sigHash = sig.SigHash()
+				sig.Signature = ed25519.Sign(cPriv, sigHash[:])
+
+				return sig
+			},
+			want: &RotationDetails{
+				WrappingPubkey: cPub,
+				PrevNodeKeys:   []key.NodePublic{n2.Public(), n1.Public()},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sig := tt.sigFn()
+			if err := sig.verifySignature(tt.nodeKey, k); err != nil {
+				t.Fatalf("verifySignature(node) failed: %v", err)
+			}
+			got, err := sig.rotationDetails()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("rotationDetails() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

tka/tka.go

@@ -668,25 +668,36 @@ func (a *Authority) Inform(storage Chonk, updates []AUM) error {
 // NodeKeyAuthorized checks if the provided nodeKeySignature authorizes
 // the given node key.
 func (a *Authority) NodeKeyAuthorized(nodeKey key.NodePublic, nodeKeySignature tkatype.MarshaledSignature) error {
+	_, err := a.NodeKeyAuthorizedWithDetails(nodeKey, nodeKeySignature)
+	return err
+}
+
+// NodeKeyAuthorized checks if the provided nodeKeySignature authorizes
+// the given node key, and returns RotationDetails if the signature is
+// a valid rotation signature.
+func (a *Authority) NodeKeyAuthorizedWithDetails(nodeKey key.NodePublic, nodeKeySignature tkatype.MarshaledSignature) (*RotationDetails, error) {
 	var decoded NodeKeySignature
 	if err := decoded.Unserialize(nodeKeySignature); err != nil {
-		return fmt.Errorf("unserialize: %v", err)
+		return nil, fmt.Errorf("unserialize: %v", err)
 	}
 	if decoded.SigKind == SigCredential {
-		return errors.New("credential signatures cannot authorize nodes on their own")
+		return nil, errors.New("credential signatures cannot authorize nodes on their own")
 	}
 
 	kID, err := decoded.authorizingKeyID()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	key, err := a.state.GetKey(kID)
 	if err != nil {
-		return fmt.Errorf("key: %v", err)
+		return nil, fmt.Errorf("key: %v", err)
 	}
 
-	return decoded.verifySignature(nodeKey, key)
+	if err := decoded.verifySignature(nodeKey, key); err != nil {
+		return nil, err
+	}
+	return decoded.rotationDetails()
 }
 
 // KeyTrusted returns true if the given keyID is trusted by the tailnet

tool/gocross/autoflags.go

@@ -36,7 +36,7 @@ func autoflagsForTest(argv []string, env *Environment, goroot, nativeGOOS, nativ
 		targetOS    = cmp.Or(env.Get("GOOS", ""), nativeGOOS)
 		targetArch  = cmp.Or(env.Get("GOARCH", ""), nativeGOARCH)
 		buildFlags  = []string{"-trimpath"}
-		cgoCflags   = []string{"-O3", "-std=gnu11"}
+		cgoCflags   = []string{"-O3", "-std=gnu11", "-g"}
 		cgoLdflags  []string
 		ldflags     []string
 		tags        = []string{"tailscale_go"}

tool/gocross/autoflags_test.go

@@ -41,7 +41,7 @@ func TestAutoflags(t *testing.T) {
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -67,7 +67,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -96,7 +96,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=0 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=riscv64 (was riscv64)
@@ -125,7 +125,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=0 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -151,7 +151,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -181,7 +181,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=0 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -210,7 +210,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=0 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -240,7 +240,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was 1)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -266,7 +266,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "arm64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=arm64 (was <nil>)
@@ -295,7 +295,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "arm64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=arm64 (was <nil>)
@@ -324,7 +324,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "arm64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=arm64 (was )
@@ -353,7 +353,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "arm64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was amd64)
@@ -382,7 +382,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "arm64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=arm64 (was <nil>)
@@ -415,7 +415,7 @@ TS_LINK_FAIL_REFLECT=1 (was <nil>)`,
 			nativeGOARCH: "arm64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 -mmacosx-version-min=11.3 -isysroot /my/sdk/root -arch x86_64 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g -mmacosx-version-min=11.3 -isysroot /my/sdk/root -arch x86_64 (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS=-mmacosx-version-min=11.3 -isysroot /my/sdk/root -arch x86_64 (was <nil>)
 GOARCH=amd64 (was amd64)
@@ -448,7 +448,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 -miphoneos-version-min=15.0 -isysroot /my/sdk/root -arch arm64 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g -miphoneos-version-min=15.0 -isysroot /my/sdk/root -arch arm64 (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS=-miphoneos-version-min=15.0 -isysroot /my/sdk/root -arch arm64 (was <nil>)
 GOARCH=arm64 (was arm64)
@@ -474,7 +474,7 @@ TS_LINK_FAIL_REFLECT=1 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -500,7 +500,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -529,7 +529,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -556,7 +556,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)
@@ -586,7 +586,7 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 			nativeGOARCH: "amd64",
 
 			envDiff: `CC=cc (was <nil>)
-CGO_CFLAGS=-O3 -std=gnu11 (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -g (was <nil>)
 CGO_ENABLED=1 (was <nil>)
 CGO_LDFLAGS= (was <nil>)
 GOARCH=amd64 (was <nil>)

tsnet/tsnet.go

@@ -106,7 +106,7 @@ type Server struct {
 	// AuthKey, if non-empty, is the auth key to create the node
 	// and will be preferred over the TS_AUTHKEY environment
 	// variable. If the node is already created (from state
-	// previously stored in in Store), then this field is not
+	// previously stored in Store), then this field is not
 	// used.
 	AuthKey string
 
@@ -562,14 +562,25 @@ func (s *Server) start() (reterr error) {
 		return ok
 	}
 	s.dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
-		// Note: don't just return ns.DialContextTCP or we'll
-		// return an interface containing a nil pointer.
+		// Note: don't just return ns.DialContextTCP or we'll return
+		// *gonet.TCPConn(nil) instead of a nil interface which trips up
+		// callers.
 		tcpConn, err := ns.DialContextTCP(ctx, dst)
 		if err != nil {
 			return nil, err
 		}
 		return tcpConn, nil
 	}
+	s.dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+		// Note: don't just return ns.DialContextUDP or we'll return
+		// *gonet.UDPConn(nil) instead of a nil interface which trips up
+		// callers.
+		udpConn, err := ns.DialContextUDP(ctx, dst)
+		if err != nil {
+			return nil, err
+		}
+		return udpConn, nil
+	}
 
 	if s.Store == nil {
 		stateFile := filepath.Join(s.rootPath, "tailscaled.state")
@@ -908,6 +919,34 @@ func (s *Server) Listen(network, addr string) (net.Listener, error) {
 	return s.listen(network, addr, listenOnTailnet)
 }
 
+// ListenPacket announces on the Tailscale network.
+//
+// The network must be "udp", "udp4" or "udp6". The addr must be of the form
+// "ip:port" (or "[ip]:port") where ip is a valid IPv4 or IPv6 address
+// corresponding to "udp4" or "udp6" respectively. IP must be specified.
+//
+// If s has not been started yet, it will be started.
+func (s *Server) ListenPacket(network, addr string) (net.PacketConn, error) {
+	ap, err := resolveListenAddr(network, addr)
+	if err != nil {
+		return nil, err
+	}
+	if !ap.Addr().IsValid() {
+		return nil, fmt.Errorf("tsnet.ListenPacket(%q, %q): address must be a valid IP", network, addr)
+	}
+	if network == "udp" {
+		if ap.Addr().Is4() {
+			network = "udp4"
+		} else {
+			network = "udp6"
+		}
+	}
+	if err := s.Start(); err != nil {
+		return nil, err
+	}
+	return s.netstack.ListenPacket(network, ap.String())
+}
+
 // ListenTLS announces only on the Tailscale network.
 // It returns a TLS listener wrapping the tsnet listener.
 // It will start the server if it has not been started yet.
@@ -1070,50 +1109,65 @@ const (
 	listenOnBoth    = listenOn("listen-on-both")
 )
 
-func (s *Server) listen(network, addr string, lnOn listenOn) (net.Listener, error) {
-	switch network {
-	case "", "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6":
-	default:
-		return nil, errors.New("unsupported network type")
-	}
+// resolveListenAddr resolves a network and address into a netip.AddrPort. The
+// returned netip.AddrPort.Addr will be the zero value if the address is empty.
+// The port must be a valid port number. The caller is responsible for checking
+// the network and address are valid.
+//
+// It resolves well-known port names and validates the address is a valid IP
+// literal for the network.
+func resolveListenAddr(network, addr string) (netip.AddrPort, error) {
+	var zero netip.AddrPort
 	host, portStr, err := net.SplitHostPort(addr)
 	if err != nil {
-		return nil, fmt.Errorf("tsnet: %w", err)
+		return zero, fmt.Errorf("tsnet: %w", err)
 	}
 	port, err := net.LookupPort(network, portStr)
 	if err != nil || port < 0 || port > math.MaxUint16 {
 		// LookupPort returns an error on out of range values so the bounds
 		// checks on port should be unnecessary, but harmless. If they do
 		// match, worst case this error message says "invalid port: <nil>".
-		return nil, fmt.Errorf("invalid port: %w", err)
+		return zero, fmt.Errorf("invalid port: %w", err)
 	}
-	var bindHostOrZero netip.Addr
-	if host != "" {
-		bindHostOrZero, err = netip.ParseAddr(host)
-		if err != nil {
-			return nil, fmt.Errorf("invalid Listen addr %q; host part must be empty or IP literal", host)
-		}
-		if strings.HasSuffix(network, "4") && !bindHostOrZero.Is4() {
-			return nil, fmt.Errorf("invalid non-IPv4 addr %v for network %q", host, network)
-		}
-		if strings.HasSuffix(network, "6") && !bindHostOrZero.Is6() {
-			return nil, fmt.Errorf("invalid non-IPv6 addr %v for network %q", host, network)
-		}
+	if host == "" {
+		return netip.AddrPortFrom(netip.Addr{}, uint16(port)), nil
 	}
 
+	bindHostOrZero, err := netip.ParseAddr(host)
+	if err != nil {
+		return zero, fmt.Errorf("invalid Listen addr %q; host part must be empty or IP literal", host)
+	}
+	if strings.HasSuffix(network, "4") && !bindHostOrZero.Is4() {
+		return zero, fmt.Errorf("invalid non-IPv4 addr %v for network %q", host, network)
+	}
+	if strings.HasSuffix(network, "6") && !bindHostOrZero.Is6() {
+		return zero, fmt.Errorf("invalid non-IPv6 addr %v for network %q", host, network)
+	}
+	return netip.AddrPortFrom(bindHostOrZero, uint16(port)), nil
+}
+
+func (s *Server) listen(network, addr string, lnOn listenOn) (*listener, error) {
+	switch network {
+	case "", "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6":
+	default:
+		return nil, errors.New("unsupported network type")
+	}
+	host, err := resolveListenAddr(network, addr)
+	if err != nil {
+		return nil, err
+	}
 	if err := s.Start(); err != nil {
 		return nil, err
 	}
-
 	var keys []listenKey
 	switch lnOn {
 	case listenOnTailnet:
-		keys = append(keys, listenKey{network, bindHostOrZero, uint16(port), false})
+		keys = append(keys, listenKey{network, host.Addr(), host.Port(), false})
 	case listenOnFunnel:
-		keys = append(keys, listenKey{network, bindHostOrZero, uint16(port), true})
+		keys = append(keys, listenKey{network, host.Addr(), host.Port(), true})
 	case listenOnBoth:
-		keys = append(keys, listenKey{network, bindHostOrZero, uint16(port), false})
-		keys = append(keys, listenKey{network, bindHostOrZero, uint16(port), true})
+		keys = append(keys, listenKey{network, host.Addr(), host.Port(), false})
+		keys = append(keys, listenKey{network, host.Addr(), host.Port(), true})
 	}
 
 	ln := &listener{

tsnet/tsnet_test.go

@@ -745,3 +745,73 @@ func TestCapturePcap(t *testing.T) {
 		t.Errorf("s2 pcap file size = %d, want > pcapHeaderSize(%d)", got, pcapHeaderSize)
 	}
 }
+
+func TestUDPConn(t *testing.T) {
+	tstest.ResourceCheck(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	controlURL, _ := startControl(t)
+	s1, s1ip, _ := startServer(t, ctx, controlURL, "s1")
+	s2, s2ip, _ := startServer(t, ctx, controlURL, "s2")
+
+	lc2, err := s2.LocalClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ping to make sure the connection is up.
+	res, err := lc2.Ping(ctx, s1ip, tailcfg.PingICMP)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("ping success: %#+v", res)
+
+	pc := must.Get(s1.ListenPacket("udp", fmt.Sprintf("%s:8081", s1ip)))
+	defer pc.Close()
+
+	// Dial to s1 from s2
+	w, err := s2.Dial(ctx, "udp", fmt.Sprintf("%s:8081", s1ip))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Close()
+
+	// Send a packet from s2 to s1
+	want := "hello"
+	if _, err := io.WriteString(w, want); err != nil {
+		t.Fatal(err)
+	}
+
+	// Receive the packet on s1
+	got := make([]byte, 1024)
+	n, from, err := pc.ReadFrom(got)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got = got[:n]
+	t.Logf("got: %q", got)
+	if string(got) != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+	if from.(*net.UDPAddr).AddrPort().Addr() != s2ip {
+		t.Errorf("got from %v, want %v", from, s2ip)
+	}
+
+	// Write a response back to s2
+	if _, err := pc.WriteTo([]byte("world"), from); err != nil {
+		t.Fatal(err)
+	}
+
+	// Receive the response on s2
+	got = make([]byte, 1024)
+	n, err = w.Read(got)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got = got[:n]
+	t.Logf("got: %q", got)
+	if string(got) != "world" {
+		t.Errorf("got %q, want world", got)
+	}
+}

tstest/iosdeps/iosdeps_test.go

@@ -14,6 +14,7 @@ func TestDeps(t *testing.T) {
 		GOOS:   "ios",
 		GOARCH: "arm64",
 		BadDeps: map[string]string{
+			"testing":       "do not use testing package in production code",
 			"text/template": "linker bloat (MethodByName)",
 			"html/template": "linker bloat (MethodByName)",
 		},

tstest/jsdeps/jsdeps_test.go

@@ -14,6 +14,7 @@ func TestDeps(t *testing.T) {
 		GOOS:   "js",
 		GOARCH: "wasm",
 		BadDeps: map[string]string{
+			"testing":                     "do not use testing package in production code",
 			"runtime/pprof":               "bloat",
 			"golang.org/x/net/http2/h2c":  "bloat",
 			"net/http/pprof":              "bloat",

tsweb/log.go

@@ -12,7 +12,7 @@ import (
 // AccessLogRecord is a record of one HTTP request served.
 type AccessLogRecord struct {
 	// Timestamp at which request processing started.
-	When time.Time `json:"when"`
+	Time time.Time `json:"time"`
 	// Time it took to finish processing the request. It does not
 	// include the entire lifetime of the underlying connection in
 	// cases like connection hijacking, only the lifetime of the HTTP
@@ -55,8 +55,8 @@ type AccessLogRecord struct {
 
 // String returns m as a JSON string.
 func (m AccessLogRecord) String() string {
-	if m.When.IsZero() {
-		m.When = time.Now()
+	if m.Time.IsZero() {
+		m.Time = time.Now()
 	}
 	var buf strings.Builder
 	json.NewEncoder(&buf).Encode(m)

tsweb/tsweb.go

@@ -299,7 +299,7 @@ type retHandler struct {
 // ServeHTTP implements the http.Handler interface.
 func (h retHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	msg := AccessLogRecord{
-		When:       h.opts.Now(),
+		Time:       h.opts.Now(),
 		RemoteAddr: r.RemoteAddr,
 		Proto:      r.Proto,
 		TLS:        r.TLS != nil,
@@ -371,7 +371,7 @@ func (h retHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		lw.code = 200
 	}
 
-	msg.Seconds = h.opts.Now().Sub(msg.When).Seconds()
+	msg.Seconds = h.opts.Now().Sub(msg.Time).Seconds()
 	msg.Code = lw.code
 	msg.Bytes = lw.bytes
 

tsweb/tsweb_test.go

@@ -87,7 +87,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				TLS:        false,
@@ -104,7 +104,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				TLS:        false,
@@ -121,7 +121,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 404,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -137,7 +137,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 404,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -153,7 +153,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 404,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -171,7 +171,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(RequestIDKey.WithValue(bgCtx, exampleRequestID), "http://example.com/foo"),
 			wantCode: 404,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -190,7 +190,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 404,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -208,7 +208,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(RequestIDKey.WithValue(bgCtx, exampleRequestID), "http://example.com/foo"),
 			wantCode: 404,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -227,7 +227,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 500,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -245,7 +245,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(RequestIDKey.WithValue(bgCtx, exampleRequestID), "http://example.com/foo"),
 			wantCode: 500,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -264,7 +264,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 500,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -282,7 +282,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(RequestIDKey.WithValue(bgCtx, exampleRequestID), "http://example.com/foo"),
 			wantCode: 500,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -301,7 +301,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 500,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -319,7 +319,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(RequestIDKey.WithValue(bgCtx, exampleRequestID), "http://example.com/foo"),
 			wantCode: 500,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -338,7 +338,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -355,7 +355,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(RequestIDKey.WithValue(bgCtx, exampleRequestID), "http://example.com/foo"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -373,7 +373,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -390,7 +390,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				Host:       "example.com",
@@ -412,7 +412,7 @@ func TestStdHandler(t *testing.T) {
 			r:        req(bgCtx, "http://example.com/foo"),
 			wantCode: 200,
 			wantLog: AccessLogRecord{
-				When:    startTime,
+				Time:    startTime,
 				Seconds: 1.0,
 
 				Proto:      "HTTP/1.1",
@@ -432,7 +432,7 @@ func TestStdHandler(t *testing.T) {
 				http.Error(w, e.Msg, 200)
 			},
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				TLS:        false,
@@ -455,7 +455,7 @@ func TestStdHandler(t *testing.T) {
 				http.Error(w, fmt.Sprintf("%s with request ID %s", e.Msg, requestID), 200)
 			},
 			wantLog: AccessLogRecord{
-				When:       startTime,
+				Time:       startTime,
 				Seconds:    1.0,
 				Proto:      "HTTP/1.1",
 				TLS:        false,

util/linuxfw/iptables_runner.go

@@ -20,6 +20,7 @@ import (
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/multierr"
+	"tailscale.com/version/distro"
 )
 
 // isNotExistError needs to be overridden in tests that rely on distinguishing
@@ -653,6 +654,11 @@ func (i *iptablesRunner) DelMagicsockPortRule(port uint16, network string) error
 // IPTablesCleanUp removes all Tailscale added iptables rules.
 // Any errors that occur are logged to the provided logf.
 func IPTablesCleanUp(logf logger.Logf) {
+	if distro.Get() == distro.Gokrazy {
+		// Gokrazy uses nftables and doesn't have the "iptables" command.
+		// Avoid log spam on cleanup. (#12277)
+		return
+	}
 	err := clearRules(iptables.ProtocolIPv4, logf)
 	if err != nil {
 		logf("linuxfw: clear iptables: %v", err)

util/pool/pool.go

@@ -0,0 +1,213 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package pool contains a generic type for managing a pool of resources; for
+// example, connections to a database, or to a remote service.
+//
+// Unlike sync.Pool from the Go standard library, this pool does not remove
+// items from the pool when garbage collection happens, nor is it safe for
+// concurrent use like sync.Pool.
+package pool
+
+import (
+	"fmt"
+	"math/rand/v2"
+
+	"tailscale.com/types/ptr"
+)
+
+// consistencyCheck enables additional runtime checks to ensure that the pool
+// is well-formed; it is disabled by default, and can be enabled during tests
+// to catch additional bugs.
+const consistencyCheck = false
+
+// Pool is a pool of resources. It is not safe for concurrent use.
+type Pool[V any] struct {
+	s []itemAndIndex[V]
+}
+
+type itemAndIndex[V any] struct {
+	// item is the element in the pool
+	item V
+
+	// index is the current location of this item in pool.s. It gets set to
+	// -1 when the item is removed from the pool.
+	index *int
+}
+
+// Handle is an opaque handle to a resource in a pool. It is used to delete an
+// item from the pool, without requiring the item to be comparable.
+type Handle[V any] struct {
+	idx *int // pointer to index; -1 if not in slice
+}
+
+// Len returns the current size of the pool.
+func (p *Pool[V]) Len() int {
+	return len(p.s)
+}
+
+// Clear removes all items from the pool.
+func (p *Pool[V]) Clear() {
+	p.s = nil
+}
+
+// AppendTakeAll removes all items from the pool, appending them to the
+// provided slice (which can be nil) and returning them. The returned slice can
+// be nil if the provided slice was nil and the pool was empty.
+//
+// This function does not free the backing storage for the pool; to do that,
+// use the Clear function.
+func (p *Pool[V]) AppendTakeAll(dst []V) []V {
+	ret := dst
+	for i := range p.s {
+		e := p.s[i]
+		if consistencyCheck && e.index == nil {
+			panic(fmt.Sprintf("pool: index is nil at %d", i))
+		}
+		if *e.index >= 0 {
+			ret = append(ret, p.s[i].item)
+		}
+	}
+	p.s = p.s[:0]
+	return ret
+}
+
+// Add adds an item to the pool and returns a handle to it. The handle can be
+// used to delete the item from the pool with the Delete method.
+func (p *Pool[V]) Add(item V) Handle[V] {
+	// Store the index in a pointer, so that we can pass it to both the
+	// handle and store it in the itemAndIndex.
+	idx := ptr.To(len(p.s))
+	p.s = append(p.s, itemAndIndex[V]{
+		item:  item,
+		index: idx,
+	})
+	return Handle[V]{idx}
+}
+
+// Peek will return the item with the given handle without removing it from the
+// pool.
+//
+// It will return ok=false if the item has been deleted or previously taken.
+func (p *Pool[V]) Peek(h Handle[V]) (v V, ok bool) {
+	p.checkHandle(h)
+	idx := *h.idx
+	if idx < 0 {
+		var zero V
+		return zero, false
+	}
+	p.checkIndex(idx)
+	return p.s[idx].item, true
+}
+
+// Delete removes the item from the pool.
+//
+// It reports whether the element was deleted; it will return false if the item
+// has been taken with the TakeRandom function, or if the item was already
+// deleted.
+func (p *Pool[V]) Delete(h Handle[V]) bool {
+	p.checkHandle(h)
+	idx := *h.idx
+	if idx < 0 {
+		return false
+	}
+	p.deleteIndex(idx)
+	return true
+}
+
+func (p *Pool[V]) deleteIndex(idx int) {
+	// Mark the item as deleted.
+	p.checkIndex(idx)
+	*(p.s[idx].index) = -1
+
+	// If this isn't the last element in the slice, overwrite the element
+	// at this item's index with the last element.
+	lastIdx := len(p.s) - 1
+
+	if idx < lastIdx {
+		last := p.s[lastIdx]
+		p.checkElem(lastIdx, last)
+		*last.index = idx
+		p.s[idx] = last
+	}
+
+	// Zero out last element (for GC) and truncate slice.
+	p.s[lastIdx] = itemAndIndex[V]{}
+	p.s = p.s[:lastIdx]
+}
+
+// Take will remove the item with the given handle from the pool and return it.
+//
+// It will return ok=false and the zero value if the item has been deleted or
+// previously taken.
+func (p *Pool[V]) Take(h Handle[V]) (v V, ok bool) {
+	p.checkHandle(h)
+	idx := *h.idx
+	if idx < 0 {
+		var zero V
+		return zero, false
+	}
+
+	e := p.s[idx]
+	p.deleteIndex(idx)
+	return e.item, true
+}
+
+// TakeRandom returns and removes a random element from p
+// and reports whether there was one to take.
+//
+// It will return ok=false and the zero value if the pool is empty.
+func (p *Pool[V]) TakeRandom() (v V, ok bool) {
+	if len(p.s) == 0 {
+		var zero V
+		return zero, false
+	}
+	pick := rand.IntN(len(p.s))
+	e := p.s[pick]
+	p.checkElem(pick, e)
+	p.deleteIndex(pick)
+	return e.item, true
+}
+
+// checkIndex verifies that the provided index is within the bounds of the
+// pool's slice, and that the corresponding element has a non-nil index
+// pointer, and panics if not.
+func (p *Pool[V]) checkIndex(idx int) {
+	if !consistencyCheck {
+		return
+	}
+
+	if idx >= len(p.s) {
+		panic(fmt.Sprintf("pool: index %d out of range (len %d)", idx, len(p.s)))
+	}
+	if p.s[idx].index == nil {
+		panic(fmt.Sprintf("pool: index is nil at %d", idx))
+	}
+}
+
+// checkHandle verifies that the provided handle is not nil, and panics if it
+// is.
+func (p *Pool[V]) checkHandle(h Handle[V]) {
+	if !consistencyCheck {
+		return
+	}
+
+	if h.idx == nil {
+		panic("pool: nil handle")
+	}
+}
+
+// checkElem verifies that the provided itemAndIndex has a non-nil index, and
+// that the stored index matches the expected position within the slice.
+func (p *Pool[V]) checkElem(idx int, e itemAndIndex[V]) {
+	if !consistencyCheck {
+		return
+	}
+
+	if e.index == nil {
+		panic("pool: index is nil")
+	}
+	if got := *e.index; got != idx {
+		panic(fmt.Sprintf("pool: index is incorrect: want %d, got %d", idx, got))
+	}
+}

util/pool/pool_test.go

@@ -0,0 +1,203 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package pool
+
+import (
+	"slices"
+	"testing"
+)
+
+func TestPool(t *testing.T) {
+	p := Pool[int]{}
+
+	if got, want := p.Len(), 0; got != want {
+		t.Errorf("got initial length %v; want %v", got, want)
+	}
+
+	h1 := p.Add(101)
+	h2 := p.Add(102)
+	h3 := p.Add(103)
+	h4 := p.Add(104)
+
+	if got, want := p.Len(), 4; got != want {
+		t.Errorf("got length %v; want %v", got, want)
+	}
+
+	tests := []struct {
+		h    Handle[int]
+		want int
+	}{
+		{h1, 101},
+		{h2, 102},
+		{h3, 103},
+		{h4, 104},
+	}
+	for i, test := range tests {
+		got, ok := p.Peek(test.h)
+		if !ok {
+			t.Errorf("test[%d]: did not find item", i)
+			continue
+		}
+		if got != test.want {
+			t.Errorf("test[%d]: got %v; want %v", i, got, test.want)
+		}
+	}
+
+	if deleted := p.Delete(h2); !deleted {
+		t.Errorf("h2 not deleted")
+	}
+	if deleted := p.Delete(h2); deleted {
+		t.Errorf("h2 should not be deleted twice")
+	}
+	if got, want := p.Len(), 3; got != want {
+		t.Errorf("got length %v; want %v", got, want)
+	}
+	if _, ok := p.Peek(h2); ok {
+		t.Errorf("h2 still in pool")
+	}
+
+	// Remove an item by handle
+	got, ok := p.Take(h4)
+	if !ok {
+		t.Errorf("h4 not found")
+	}
+	if got != 104 {
+		t.Errorf("got %v; want 104", got)
+	}
+
+	// Take doesn't work on previously-taken or deleted items.
+	if _, ok := p.Take(h4); ok {
+		t.Errorf("h4 should not be taken twice")
+	}
+	if _, ok := p.Take(h2); ok {
+		t.Errorf("h2 should not be taken after delete")
+	}
+
+	// Remove all items and return them
+	items := p.AppendTakeAll(nil)
+	want := []int{101, 103}
+	if !slices.Equal(items, want) {
+		t.Errorf("got items %v; want %v", items, want)
+	}
+	if got := p.Len(); got != 0 {
+		t.Errorf("got length %v; want 0", got)
+	}
+
+	// Insert and then clear should result in no items.
+	p.Add(105)
+	p.Clear()
+	if got := p.Len(); got != 0 {
+		t.Errorf("got length %v; want 0", got)
+	}
+}
+
+func TestTakeRandom(t *testing.T) {
+	p := Pool[int]{}
+	for i := 0; i < 10; i++ {
+		p.Add(i + 100)
+	}
+
+	seen := make(map[int]bool)
+	for i := 0; i < 10; i++ {
+		item, ok := p.TakeRandom()
+		if !ok {
+			t.Errorf("unexpected empty pool")
+			break
+		}
+		if seen[item] {
+			t.Errorf("got duplicate item %v", item)
+		}
+		seen[item] = true
+	}
+
+	// Verify that the pool is empty
+	if _, ok := p.TakeRandom(); ok {
+		t.Errorf("expected empty pool")
+	}
+
+	for i := 0; i < 10; i++ {
+		want := 100 + i
+		if !seen[want] {
+			t.Errorf("item %v not seen", want)
+		}
+	}
+
+	if t.Failed() {
+		t.Logf("seen: %+v", seen)
+	}
+}
+
+func BenchmarkPool_AddDelete(b *testing.B) {
+	b.Run("impl=Pool", func(b *testing.B) {
+		p := Pool[int]{}
+
+		// Warm up/force an initial allocation
+		h := p.Add(0)
+		p.Delete(h)
+
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			h := p.Add(i)
+			p.Delete(h)
+		}
+	})
+	b.Run("impl=map", func(b *testing.B) {
+		p := make(map[int]bool)
+
+		// Force initial allocation
+		p[0] = true
+		delete(p, 0)
+
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			p[i] = true
+			delete(p, i)
+		}
+	})
+}
+
+func BenchmarkPool_TakeRandom(b *testing.B) {
+	b.Run("impl=Pool", func(b *testing.B) {
+		p := Pool[int]{}
+
+		// Insert the number of items we'll be taking, then reset the timer.
+		for i := 0; i < b.N; i++ {
+			p.Add(i)
+		}
+		b.ResetTimer()
+
+		// Now benchmark taking all the items.
+		for i := 0; i < b.N; i++ {
+			p.TakeRandom()
+		}
+
+		if p.Len() != 0 {
+			b.Errorf("pool not empty")
+		}
+	})
+	b.Run("impl=map", func(b *testing.B) {
+		p := make(map[int]bool)
+
+		// Insert the number of items we'll be taking, then reset the timer.
+		for i := 0; i < b.N; i++ {
+			p[i] = true
+		}
+		b.ResetTimer()
+
+		// Now benchmark taking all the items.
+		for i := 0; i < b.N; i++ {
+			// Taking a random item is simulated by a single map iteration.
+			for k := range p {
+				delete(p, k) // "take" the item by removing it
+				break
+			}
+		}
+
+		if len(p) != 0 {
+			b.Errorf("map not empty")
+		}
+	})
+}

util/syspolicy/handler.go

@@ -6,7 +6,6 @@ package syspolicy
 import (
 	"errors"
 	"sync/atomic"
-	"testing"
 )
 
 var (
@@ -69,7 +68,14 @@ func RegisterHandler(h Handler) {
 	}
 }
 
-func SetHandlerForTest(tb testing.TB, h Handler) {
+// TB is a subset of testing.TB that we use to set up test helpers.
+// It's defined here to avoid pulling in the testing package.
+type TB interface {
+	Helper()
+	Cleanup(func())
+}
+
+func SetHandlerForTest(tb TB, h Handler) {
 	tb.Helper()
 	oldHandler := handler
 	handler = h

wgengine/magicsock/magicsock.go

@@ -687,7 +687,6 @@ func (c *Conn) updateNetInfo(ctx context.Context) (*netcheck.Report, error) {
 	ni := &tailcfg.NetInfo{
 		DERPLatency:           map[string]float64{},
 		MappingVariesByDestIP: report.MappingVariesByDestIP,
-		HairPinning:           report.HairPinning,
 		UPnP:                  report.UPnP,
 		PMP:                   report.PMP,
 		PCP:                   report.PCP,

wgengine/netstack/netstack.go

@@ -1326,6 +1326,50 @@ func (ns *Impl) forwardTCP(getClient func(...tcpip.SettableSocketOption) *gonet.
 	return
 }
 
+// ListenPacket listens for incoming packets for the given network and address.
+// Address must be of the form "ip:port" or "[ip]:port".
+//
+// As of 2024-05-18, only udp4 and udp6 are supported.
+func (ns *Impl) ListenPacket(network, address string) (net.PacketConn, error) {
+	ap, err := netip.ParseAddrPort(address)
+	if err != nil {
+		return nil, fmt.Errorf("netstack: ParseAddrPort(%q): %v", address, err)
+	}
+
+	var networkProto tcpip.NetworkProtocolNumber
+	switch network {
+	case "udp":
+		return nil, fmt.Errorf("netstack: udp not supported; use udp4 or udp6")
+	case "udp4":
+		networkProto = ipv4.ProtocolNumber
+		if !ap.Addr().Is4() {
+			return nil, fmt.Errorf("netstack: udp4 requires an IPv4 address")
+		}
+	case "udp6":
+		networkProto = ipv6.ProtocolNumber
+		if !ap.Addr().Is6() {
+			return nil, fmt.Errorf("netstack: udp6 requires an IPv6 address")
+		}
+	default:
+		return nil, fmt.Errorf("netstack: unsupported network %q", network)
+	}
+	var wq waiter.Queue
+	ep, nserr := ns.ipstack.NewEndpoint(udp.ProtocolNumber, networkProto, &wq)
+	if nserr != nil {
+		return nil, fmt.Errorf("netstack: NewEndpoint: %v", nserr)
+	}
+	localAddress := tcpip.FullAddress{
+		NIC:  nicID,
+		Addr: tcpip.AddrFromSlice(ap.Addr().AsSlice()),
+		Port: ap.Port(),
+	}
+	if err := ep.Bind(localAddress); err != nil {
+		ep.Close()
+		return nil, fmt.Errorf("netstack: Bind(%v): %v", localAddress, err)
+	}
+	return gonet.NewUDPConn(&wq, ep), nil
+}
+
 func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {
 	sess := r.ID()
 	if debugNetstack() {

wgengine/router/router_linux.go

@@ -25,6 +25,7 @@ import (
 	"tailscale.com/health"
 	"tailscale.com/net/netmon"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/opt"
 	"tailscale.com/types/preftype"
 	"tailscale.com/util/linuxfw"
 	"tailscale.com/util/multierr"
@@ -58,9 +59,9 @@ type linuxRouter struct {
 	ipRuleFixLimiter   *rate.Limiter
 
 	// Various feature checks for the network stack.
-	ipRuleAvailable bool // whether kernel was built with IP_MULTIPLE_TABLES
-	v6Available     bool // whether the kernel supports IPv6
-	fwmaskWorks     bool // whether we can use 'ip rule...fwmark <mark>/<mask>'
+	ipRuleAvailable bool     // whether kernel was built with IP_MULTIPLE_TABLES
+	v6Available     bool     // whether the kernel supports IPv6
+	fwmaskWorksLazy opt.Bool // whether we can use 'ip rule...fwmark <mark>/<mask>'; set lazily
 
 	// ipPolicyPrefBase is the base priority at which ip rules are installed.
 	ipPolicyPrefBase int
@@ -110,20 +111,6 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netMon *netmon
 		}
 	}
 
-	// To be a good denizen of the 4-byte 'fwmark' bitspace on every packet, we try to
-	// only use the third byte. However, support for masking to part of the fwmark bitspace
-	// was only added to busybox in 1.33.0. As such, we want to detect older versions and
-	// not issue such a stanza.
-	var err error
-	if r.fwmaskWorks, err = ipCmdSupportsFwmask(); err != nil {
-		r.logf("failed to determine ip command fwmask support: %v", err)
-	}
-	if r.fwmaskWorks {
-		r.logf("[v1] ip command supports fwmark masks")
-	} else {
-		r.logf("[v1] ip command does NOT support fwmark masks")
-	}
-
 	// A common installation of OpenWRT involves use of the 'mwan3' package.
 	// This package installs ip-tables rules like:
 	//  -A mwan3_fallback_policy -m mark --mark 0x0/0x3f00 -j MARK --set-xmark 0x100/0x3f00
@@ -260,6 +247,31 @@ func (r *linuxRouter) useIPCommand() bool {
 	return !ok
 }
 
+// fwmaskWorks reports whether we can use 'ip rule...fwmark <mark>/<mask>'.
+// This is computed lazily on first use. By default, we don't run the "ip"
+// command, so never actually runs this. But the "ip" command is used in tests
+// and can be forced. (see useIPCommand)
+func (r *linuxRouter) fwmaskWorks() bool {
+	if v, ok := r.fwmaskWorksLazy.Get(); ok {
+		return v
+	}
+	// To be a good denizen of the 4-byte 'fwmark' bitspace on every packet, we try to
+	// only use the third byte. However, support for masking to part of the fwmark bitspace
+	// was only added to busybox in 1.33.0. As such, we want to detect older versions and
+	// not issue such a stanza.
+	v, err := ipCmdSupportsFwmask()
+	if err != nil {
+		r.logf("failed to determine ip command fwmask support: %v", err)
+	}
+	r.fwmaskWorksLazy.Set(v)
+	if v {
+		r.logf("[v1] ip command supports fwmark masks")
+	} else {
+		r.logf("[v1] ip command does NOT support fwmark masks")
+	}
+	return v
+}
+
 // onIPRuleDeleted is the callback from the network monitor for when an IP
 // policy rule is deleted. See Issue 1591.
 //
@@ -469,7 +481,7 @@ func (r *linuxRouter) updateStatefulFilteringWithDockerWarning(cfg *Config) {
 		if _, found := ifstate.Interface["docker0"]; found {
 			r.health.SetWarnable(warnStatefulFilteringWithDocker, fmt.Errorf(""+
 				"Stateful filtering is enabled and Docker was detected; this may prevent Docker containers "+
-				"on this host from connecting to Tailscale nodes. "+
+				"on this host from resolving DNS and connecting to Tailscale nodes. "+
 				"See https://tailscale.com/s/stateful-docker",
 			))
 			return
@@ -1266,7 +1278,7 @@ func (r *linuxRouter) addIPRulesWithIPCommand() error {
 				"pref", strconv.Itoa(rule.Priority + r.ipPolicyPrefBase),
 			}
 			if rule.Mark != 0 {
-				if r.fwmaskWorks {
+				if r.fwmaskWorks() {
 					args = append(args, "fwmark", fmt.Sprintf("0x%x/%s", rule.Mark, linuxfw.TailscaleFwmarkMask))
 				} else {
 					args = append(args, "fwmark", fmt.Sprintf("0x%x", rule.Mark))