cmd/derper,derp: implement per-client rate limits

Signed-off-by: Tom DNetto <tom@tailscale.com>

cmd/derper/bootstrap_dns.go

@@ -17,16 +17,31 @@ import (
 	"tailscale.com/syncs"
 )
 
-var dnsCache syncs.AtomicValue[[]byte]
+const refreshTimeout = time.Minute
 
-var bootstrapDNSRequests = expvar.NewInt("counter_bootstrap_dns_requests")
+type dnsEntryMap map[string][]net.IP
+
+var (
+	dnsCache            syncs.AtomicValue[dnsEntryMap]
+	dnsCacheBytes       syncs.AtomicValue[[]byte] // of JSON
+	unpublishedDNSCache syncs.AtomicValue[dnsEntryMap]
+)
+
+var (
+	bootstrapDNSRequests = expvar.NewInt("counter_bootstrap_dns_requests")
+	publishedDNSHits     = expvar.NewInt("counter_bootstrap_dns_published_hits")
+	publishedDNSMisses   = expvar.NewInt("counter_bootstrap_dns_published_misses")
+	unpublishedDNSHits   = expvar.NewInt("counter_bootstrap_dns_unpublished_hits")
+	unpublishedDNSMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
+)
 
 func refreshBootstrapDNSLoop() {
-	if *bootstrapDNS == "" {
+	if *bootstrapDNS == "" && *unpublishedDNS == "" {
 		return
 	}
 	for {
 		refreshBootstrapDNS()
+		refreshUnpublishedDNS()
 		time.Sleep(10 * time.Minute)
 	}
 }
@@ -35,10 +50,34 @@ func refreshBootstrapDNS() {
 	if *bootstrapDNS == "" {
 		return
 	}
-	dnsEntries := make(map[string][]net.IP)
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
 	defer cancel()
-	names := strings.Split(*bootstrapDNS, ",")
+	dnsEntries := resolveList(ctx, strings.Split(*bootstrapDNS, ","))
+	j, err := json.MarshalIndent(dnsEntries, "", "\t")
+	if err != nil {
+		// leave the old values in place
+		return
+	}
+
+	dnsCache.Store(dnsEntries)
+	dnsCacheBytes.Store(j)
+}
+
+func refreshUnpublishedDNS() {
+	if *unpublishedDNS == "" {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout)
+	defer cancel()
+
+	dnsEntries := resolveList(ctx, strings.Split(*unpublishedDNS, ","))
+	unpublishedDNSCache.Store(dnsEntries)
+}
+
+func resolveList(ctx context.Context, names []string) dnsEntryMap {
+	dnsEntries := make(dnsEntryMap)
+
 	var r net.Resolver
 	for _, name := range names {
 		addrs, err := r.LookupIP(ctx, "ip", name)
@@ -48,21 +87,47 @@ func refreshBootstrapDNS() {
 		}
 		dnsEntries[name] = addrs
 	}
-	j, err := json.MarshalIndent(dnsEntries, "", "\t")
-	if err != nil {
-		// leave the old values in place
-		return
-	}
-	dnsCache.Store(j)
+	return dnsEntries
 }
 
 func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	bootstrapDNSRequests.Add(1)
+
 	w.Header().Set("Content-Type", "application/json")
-	j := dnsCache.Load()
-	// Bootstrap DNS requests occur cross-regions,
-	// and are randomized per request,
-	// so keeping a connection open is pointlessly expensive.
+	// Bootstrap DNS requests occur cross-regions, and are randomized per
+	// request, so keeping a connection open is pointlessly expensive.
 	w.Header().Set("Connection", "close")
+
+	// Try answering a query from our hidden map first
+	if q := r.URL.Query().Get("q"); q != "" {
+		if ips, ok := unpublishedDNSCache.Load()[q]; ok && len(ips) > 0 {
+			unpublishedDNSHits.Add(1)
+
+			// Only return the specific query, not everything.
+			m := dnsEntryMap{q: ips}
+			j, err := json.MarshalIndent(m, "", "\t")
+			if err == nil {
+				w.Write(j)
+				return
+			}
+		}
+
+		// If we have a "q" query for a name in the published cache
+		// list, then track whether that's a hit/miss.
+		if m, ok := dnsCache.Load()[q]; ok {
+			if len(m) > 0 {
+				publishedDNSHits.Add(1)
+			} else {
+				publishedDNSMisses.Add(1)
+			}
+		} else {
+			// If it wasn't in either cache, treat this as a query
+			// for the unpublished cache, and thus a cache miss.
+			unpublishedDNSMisses.Add(1)
+		}
+	}
+
+	// Fall back to returning the public set of cached DNS names
+	j := dnsCacheBytes.Load()
 	w.Write(j)
 }

cmd/derper/bootstrap_dns_test.go

@@ -5,7 +5,12 @@
 package main
 
 import (
+	"encoding/json"
+	"net"
 	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
 	"testing"
 )
 
@@ -17,11 +22,12 @@ func BenchmarkHandleBootstrapDNS(b *testing.B) {
 	}()
 	refreshBootstrapDNS()
 	w := new(bitbucketResponseWriter)
+	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.io"), nil)
 	b.ReportAllocs()
 	b.ResetTimer()
 	b.RunParallel(func(b *testing.PB) {
 		for b.Next() {
-			handleBootstrapDNS(w, nil)
+			handleBootstrapDNS(w, req)
 		}
 	})
 }
@@ -33,3 +39,116 @@ func (b *bitbucketResponseWriter) Header() http.Header { return make(http.Header
 func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p), nil }
 
 func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {}
+
+func getBootstrapDNS(t *testing.T, q string) dnsEntryMap {
+	t.Helper()
+	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape(q), nil)
+	w := httptest.NewRecorder()
+	handleBootstrapDNS(w, req)
+
+	res := w.Result()
+	if res.StatusCode != 200 {
+		t.Fatalf("got status=%d; want %d", res.StatusCode, 200)
+	}
+	var ips dnsEntryMap
+	if err := json.NewDecoder(res.Body).Decode(&ips); err != nil {
+		t.Fatalf("error decoding response body: %v", err)
+	}
+	return ips
+}
+
+func TestUnpublishedDNS(t *testing.T) {
+	const published = "login.tailscale.com"
+	const unpublished = "log.tailscale.io"
+
+	prev1, prev2 := *bootstrapDNS, *unpublishedDNS
+	*bootstrapDNS = published
+	*unpublishedDNS = unpublished
+	t.Cleanup(func() {
+		*bootstrapDNS = prev1
+		*unpublishedDNS = prev2
+	})
+
+	refreshBootstrapDNS()
+	refreshUnpublishedDNS()
+
+	hasResponse := func(q string) bool {
+		_, found := getBootstrapDNS(t, q)[q]
+		return found
+	}
+
+	if !hasResponse(published) {
+		t.Errorf("expected response for: %s", published)
+	}
+	if !hasResponse(unpublished) {
+		t.Errorf("expected response for: %s", unpublished)
+	}
+
+	// Verify that querying for a random query or a real query does not
+	// leak our unpublished domain
+	m1 := getBootstrapDNS(t, published)
+	if _, found := m1[unpublished]; found {
+		t.Errorf("found unpublished domain %s: %+v", unpublished, m1)
+	}
+	m2 := getBootstrapDNS(t, "random.example.com")
+	if _, found := m2[unpublished]; found {
+		t.Errorf("found unpublished domain %s: %+v", unpublished, m2)
+	}
+}
+
+func resetMetrics() {
+	publishedDNSHits.Set(0)
+	publishedDNSMisses.Set(0)
+	unpublishedDNSHits.Set(0)
+	unpublishedDNSMisses.Set(0)
+}
+
+// Verify that we don't count an empty list in the unpublishedDNSCache as a
+// cache hit in our metrics.
+func TestUnpublishedDNSEmptyList(t *testing.T) {
+	pub := dnsEntryMap{
+		"tailscale.com": {net.IPv4(10, 10, 10, 10)},
+	}
+	dnsCache.Store(pub)
+	dnsCacheBytes.Store([]byte(`{"tailscale.com":["10.10.10.10"]}`))
+
+	unpublishedDNSCache.Store(dnsEntryMap{
+		"log.tailscale.io":           {},
+		"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
+	})
+
+	t.Run("CacheMiss", func(t *testing.T) {
+		// One domain in map but empty, one not in map at all
+		for _, q := range []string{"log.tailscale.io", "login.tailscale.com"} {
+			resetMetrics()
+			ips := getBootstrapDNS(t, q)
+
+			// Expected our public map to be returned on a cache miss
+			if !reflect.DeepEqual(ips, pub) {
+				t.Errorf("got ips=%+v; want %+v", ips, pub)
+			}
+			if v := unpublishedDNSHits.Value(); v != 0 {
+				t.Errorf("got hits=%d; want 0", v)
+			}
+			if v := unpublishedDNSMisses.Value(); v != 1 {
+				t.Errorf("got misses=%d; want 1", v)
+			}
+		}
+	})
+
+	// Verify that we do get a valid response and metric.
+	t.Run("CacheHit", func(t *testing.T) {
+		resetMetrics()
+		ips := getBootstrapDNS(t, "controlplane.tailscale.com")
+		want := dnsEntryMap{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}}
+		if !reflect.DeepEqual(ips, want) {
+			t.Errorf("got ips=%+v; want %+v", ips, want)
+		}
+		if v := unpublishedDNSHits.Value(); v != 1 {
+			t.Errorf("got hits=%d; want 1", v)
+		}
+		if v := unpublishedDNSMisses.Value(); v != 0 {
+			t.Errorf("got misses=%d; want 0", v)
+		}
+	})
+}

cmd/derper/derper.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"go4.org/mem"
 	"golang.org/x/time/rate"
 	"tailscale.com/atomicfile"
 	"tailscale.com/derp"
@@ -46,14 +47,21 @@ var (
 	certDir    = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
 	hostname   = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
 	runSTUN    = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
+	runDERP    = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
 
-	meshPSKFile   = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
-	meshWith      = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
-	bootstrapDNS  = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
-	verifyClients = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
+	meshPSKFile    = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
+	meshWith       = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
+	bootstrapDNS   = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
+	unpublishedDNS = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list")
+	verifyClients  = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 
 	acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection")
 	acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection")
+
+	egressInterface = flag.String("egress-interface", "", "the interface to monitor for automatic ratelimit tuning")
+	egressDataLimit = flag.Int("egress-data-limit", 100*1024*1024/8, "the bandwidth in bytes/s the server will try to stay under, only applies if egress-interface is set")
+	clientDataMin   = flag.Int("client-data-min-limit", 1024*1024/8, "minimum bandwidth in bytes/s for a single client, only applies if egress-interface is set")
+	clientDataBurst = flag.Int("client-data-burst", 3*1024*1024, "burst limit in bytes for forwarded data from a single client, only applies if egress-interface is set")
 )
 
 var (
@@ -151,6 +159,12 @@ func main() {
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)
 
+	if *egressInterface != "" && *egressDataLimit > 0 {
+		if err := s.StartEgressRateLimiter(*egressInterface, *egressDataLimit, *clientDataMin, *clientDataBurst); err != nil {
+			log.Fatalf("failed to start egress rate limiter: %v", err)
+		}
+	}
+
 	if *meshPSKFile != "" {
 		b, err := ioutil.ReadFile(*meshPSKFile)
 		if err != nil {
@@ -169,9 +183,15 @@ func main() {
 	expvar.Publish("derp", s.ExpVar())
 
 	mux := http.NewServeMux()
-	derpHandler := derphttp.Handler(s)
-	derpHandler = addWebSocketSupport(s, derpHandler)
-	mux.Handle("/derp", derpHandler)
+	if *runDERP {
+		derpHandler := derphttp.Handler(s)
+		derpHandler = addWebSocketSupport(s, derpHandler)
+		mux.Handle("/derp", derpHandler)
+	} else {
+		mux.Handle("/derp", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "derp server disabled", http.StatusNotFound)
+		}))
+	}
 	mux.HandleFunc("/derp/probe", probeHandler)
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", handleBootstrapDNS)
@@ -187,10 +207,17 @@ func main() {
   server.
 </p>
 `)
+		if !*runDERP {
+			io.WriteString(w, `<p>Status: <b>disabled</b></p>`)
+		}
 		if tsweb.AllowDebugAccess(r) {
 			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
 		}
 	}))
+	mux.Handle("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "User-agent: *\nDisallow: /\n")
+	}))
+	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -208,9 +235,11 @@ func main() {
 		go serveSTUN(listenHost, *stunPort)
 	}
 
+	quietLogger := log.New(logFilter{}, "", 0)
 	httpsrv := &http.Server{
-		Addr:    *addr,
-		Handler: mux,
+		Addr:     *addr,
+		Handler:  mux,
+		ErrorLog: quietLogger,
 
 		// Set read/write timeout. For derper, this basically
 		// only affects TLS setup, as read/write deadlines are
@@ -276,9 +305,13 @@ func main() {
 		})
 		if *httpPort > -1 {
 			go func() {
+				port80mux := http.NewServeMux()
+				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
-					Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
+					Handler:     port80mux,
+					ErrorLog:    quietLogger,
 					ReadTimeout: 30 * time.Second,
 					// Crank up WriteTimeout a bit more than usually
 					// necessary just so we can do long CPU profiles
@@ -304,6 +337,11 @@ func main() {
 	}
 }
 
+// For captive portal detection
+func serveNoContent(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusNoContent)
+}
+
 // probeHandler is the endpoint that js/wasm clients hit to measure
 // DERP latency, since they can't do UDP STUN queries.
 func probeHandler(w http.ResponseWriter, r *http.Request) {
@@ -449,3 +487,22 @@ func (l *rateLimitedListener) Accept() (net.Conn, error) {
 	l.numAccepts.Add(1)
 	return cn, nil
 }
+
+// logFilter is used to filter out useless error logs that are logged to
+// the net/http.Server.ErrorLog logger.
+type logFilter struct{}
+
+func (logFilter) Write(p []byte) (int, error) {
+	b := mem.B(p)
+	if mem.HasSuffix(b, mem.S(": EOF\n")) ||
+		mem.HasSuffix(b, mem.S(": i/o timeout\n")) ||
+		mem.HasSuffix(b, mem.S(": read: connection reset by peer\n")) ||
+		mem.HasSuffix(b, mem.S(": remote error: tls: bad certificate\n")) ||
+		mem.HasSuffix(b, mem.S(": tls: first record does not look like a TLS handshake\n")) {
+		// Skip this log message, but say that we processed it
+		return len(p), nil
+	}
+
+	log.Printf("%s", p)
+	return len(p), nil
+}

cmd/tailscale/cli/cli_test.go

@@ -762,6 +762,9 @@ func TestPrefFlagMapping(t *testing.T) {
 		case "NotepadURLs":
 			// TODO(bradfitz): https://github.com/tailscale/tailscale/issues/1830
 			continue
+		case "Egg":
+			// Not applicable.
+			continue
 		}
 		t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName)
 	}

cmd/tailscale/cli/licenses.go

@@ -19,24 +19,27 @@ var licensesCmd = &ffcli.Command{
 	Exec:       runLicenses,
 }
 
-func runLicenses(ctx context.Context, args []string) error {
-	var licenseURL string
+// licensesURL returns the absolute URL containing open source license information for the current platform.
+func licensesURL() string {
 	switch runtime.GOOS {
 	case "android":
-		licenseURL = "https://tailscale.com/licenses/android"
+		return "https://tailscale.com/licenses/android"
 	case "darwin", "ios":
-		licenseURL = "https://tailscale.com/licenses/apple"
+		return "https://tailscale.com/licenses/apple"
 	case "windows":
-		licenseURL = "https://tailscale.com/licenses/windows"
+		return "https://tailscale.com/licenses/windows"
 	default:
-		licenseURL = "https://tailscale.com/licenses/tailscale"
+		return "https://tailscale.com/licenses/tailscale"
 	}
+}
 
+func runLicenses(ctx context.Context, args []string) error {
+	licenses := licensesURL()
 	outln(`
 Tailscale wouldn't be possible without the contributions of thousands of open
 source developers. To see the open source packages included in Tailscale and
 their respective license information, visit:
 
-    ` + licenseURL)
+    ` + licenses)
 	return nil
 }

cmd/tailscale/cli/up.go

@@ -406,8 +406,12 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 }
 
 func runUp(ctx context.Context, args []string) (retErr error) {
+	var egg bool
 	if len(args) > 0 {
-		fatalf("too many non-flag arguments: %q", args)
+		egg = fmt.Sprint(args) == "[up down down left right left right b a]"
+		if !egg {
+			fatalf("too many non-flag arguments: %q", args)
+		}
 	}
 
 	st, err := localClient.Status(ctx)
@@ -493,6 +497,7 @@ func runUp(ctx context.Context, args []string) (retErr error) {
 		fatalf("%s", err)
 	}
 	if justEditMP != nil {
+		justEditMP.EggSet = true
 		_, err := localClient.EditPrefs(ctx, justEditMP)
 		return err
 	}

cmd/tailscale/cli/web.go

@@ -59,6 +59,7 @@ type tmplData struct {
 	IP                string
 	AdvertiseExitNode bool
 	AdvertiseRoutes   string
+	LicensesURL       string
 }
 
 var webCmd = &ffcli.Command{
@@ -392,6 +393,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 		Profile:      profile,
 		Status:       st.BackendState,
 		DeviceName:   deviceName,
+		LicensesURL:  licensesURL(),
 	}
 	exitNodeRouteV4 := netip.MustParsePrefix("0.0.0.0/0")
 	exitNodeRouteV6 := netip.MustParsePrefix("::/0")

cmd/tailscale/cli/web.html

@@ -11,7 +11,7 @@
 </head>
 
 <body class="py-14">
-<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
+<main class="container max-w-lg mx-auto mb-8 py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
 	<header class="flex justify-between items-center min-width-0 py-2 mb-8">
 		<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
 			class="flex-shrink-0 mr-4">
@@ -100,6 +100,9 @@
 	</div>
 	{{ end }}
 </main>
+<footer class="container max-w-lg mx-auto text-center">
+	<a class="text-xs text-gray-500 hover:text-gray-600" href="{{ .LicensesURL }}">Open Source Licenses</a>
+</footer>
 <script>(function () {
 const advertiseExitNode = {{.AdvertiseExitNode}};
 let fetchingUrl = false;

cmd/tailscaled/depaware.txt

@@ -212,7 +212,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
      💣 tailscale.com/metrics                                        from tailscale.com/derp+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
-        tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns/resolver
+        tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns/resolver+
         tailscale.com/net/dns/resolvconffile                         from tailscale.com/net/dns+
         tailscale.com/net/dns/resolver                               from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
@@ -281,7 +281,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
         tailscale.com/util/singleflight                              from tailscale.com/control/controlclient+
-   L    tailscale.com/util/strs                                      from tailscale.com/hostinfo
+        tailscale.com/util/strs                                      from tailscale.com/hostinfo+
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock
      💣 tailscale.com/util/winutil                                   from tailscale.com/cmd/tailscaled+

cmd/tailscaled/tailscaled.go

@@ -113,6 +113,7 @@ var args struct {
 	verbose        int
 	socksAddr      string // listen address for SOCKS5 server
 	httpProxyAddr  string // listen address for HTTP proxy server
+	disableLogs    bool
 }
 
 var (
@@ -144,6 +145,7 @@ func main() {
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
+	flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support")
 
 	if len(os.Args) > 0 && filepath.Base(os.Args[0]) == "tailscale" && beCLI != nil {
 		beCLI()
@@ -199,6 +201,10 @@ func main() {
 		args.statepath = paths.DefaultTailscaledStateFile()
 	}
 
+	if args.disableLogs {
+		envknob.SetNoLogsNoSupport()
+	}
+
 	if beWindowsSubprocess() {
 		return
 	}

cmd/tsconnect/README.md

@@ -38,3 +38,12 @@ The client is also available as an NPM package. To build it, run:
 ```
 
 That places the output in the `pkg/` directory, which may then be uploaded to a package registry (or installed from the file path directly).
+
+To do two-sided development (on both the NPM package and code that uses it), run:
+
+```
+./tool/go run ./cmd/tsconnect dev-pkg
+
+```
+
+This serves the module at http://localhost:9090/pkg/pkg.js and the generated wasm file at http://localhost:9090/pkg/main.wasm. The two files can be used as drop-in replacements for normal imports of the NPM module.

cmd/tsconnect/build-pkg.go

@@ -11,13 +11,12 @@ import (
 	"os"
 	"path"
 
-	esbuild "github.com/evanw/esbuild/pkg/api"
 	"github.com/tailscale/hujson"
 	"tailscale.com/version"
 )
 
 func runBuildPkg() {
-	buildOptions, err := commonSetup(prodMode)
+	buildOptions, err := commonPkgSetup(prodMode)
 	if err != nil {
 		log.Fatalf("Cannot setup: %v", err)
 	}
@@ -31,10 +30,6 @@ func runBuildPkg() {
 		log.Fatalf("Cannot clean %s: %v", *pkgDir, err)
 	}
 
-	buildOptions.EntryPoints = []string{"src/pkg/pkg.ts", "src/pkg/pkg.css"}
-	buildOptions.Outdir = *pkgDir
-	buildOptions.Format = esbuild.FormatESModule
-	buildOptions.AssetNames = "[name]"
 	buildOptions.Write = true
 	buildOptions.MinifyWhitespace = true
 	buildOptions.MinifyIdentifiers = true

cmd/tsconnect/common.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
 	"os/exec"
 	"path"
@@ -68,6 +69,18 @@ func commonSetup(dev bool) (*esbuild.BuildOptions, error) {
 	}, nil
 }
 
+func commonPkgSetup(dev bool) (*esbuild.BuildOptions, error) {
+	buildOptions, err := commonSetup(dev)
+	if err != nil {
+		return nil, err
+	}
+	buildOptions.EntryPoints = []string{"src/pkg/pkg.ts", "src/pkg/pkg.css"}
+	buildOptions.Outdir = *pkgDir
+	buildOptions.Format = esbuild.FormatESModule
+	buildOptions.AssetNames = "[name]"
+	return buildOptions, nil
+}
+
 // cleanDir removes files from dirPath, except the ones specified by
 // preserveFiles.
 func cleanDir(dirPath string, preserveFiles ...string) error {
@@ -90,6 +103,27 @@ func cleanDir(dirPath string, preserveFiles ...string) error {
 	return nil
 }
 
+func runEsbuildServe(buildOptions esbuild.BuildOptions) {
+	host, portStr, err := net.SplitHostPort(*addr)
+	if err != nil {
+		log.Fatalf("Cannot parse addr: %v", err)
+	}
+	port, err := strconv.ParseUint(portStr, 10, 16)
+	if err != nil {
+		log.Fatalf("Cannot parse port: %v", err)
+	}
+	result, err := esbuild.Serve(esbuild.ServeOptions{
+		Port:     uint16(port),
+		Host:     host,
+		Servedir: "./",
+	}, buildOptions)
+	if err != nil {
+		log.Fatalf("Cannot start esbuild server: %v", err)
+	}
+	log.Printf("Listening on http://%s:%d\n", result.Host, result.Port)
+	result.Wait()
+}
+
 func runEsbuild(buildOptions esbuild.BuildOptions) esbuild.BuildResult {
 	log.Printf("Running esbuild...\n")
 	result := esbuild.Build(buildOptions)

cmd/tsconnect/dev-pkg.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"log"
+)
+
+func runDevPkg() {
+	buildOptions, err := commonPkgSetup(devMode)
+	if err != nil {
+		log.Fatalf("Cannot setup: %v", err)
+	}
+	runEsbuildServe(*buildOptions)
+}

cmd/tsconnect/dev.go

@@ -6,10 +6,6 @@ package main
 
 import (
 	"log"
-	"net"
-	"strconv"
-
-	esbuild "github.com/evanw/esbuild/pkg/api"
 )
 
 func runDev() {
@@ -17,22 +13,5 @@ func runDev() {
 	if err != nil {
 		log.Fatalf("Cannot setup: %v", err)
 	}
-	host, portStr, err := net.SplitHostPort(*addr)
-	if err != nil {
-		log.Fatalf("Cannot parse addr: %v", err)
-	}
-	port, err := strconv.ParseUint(portStr, 10, 16)
-	if err != nil {
-		log.Fatalf("Cannot parse port: %v", err)
-	}
-	result, err := esbuild.Serve(esbuild.ServeOptions{
-		Port:     uint16(port),
-		Host:     host,
-		Servedir: "./",
-	}, *buildOptions)
-	if err != nil {
-		log.Fatalf("Cannot start esbuild server: %v", err)
-	}
-	log.Printf("Listening on http://%s:%d\n", result.Host, result.Port)
-	result.Wait()
+	runEsbuildServe(*buildOptions)
 }

cmd/tsconnect/package.json

@@ -10,8 +10,9 @@
     "qrcode": "^1.5.0",
     "tailwindcss": "^3.1.6",
     "typescript": "^4.7.4",
-    "xterm": "^4.18.0",
-    "xterm-addon-fit": "^0.5.0"
+    "xterm": "5.0.0-beta.58",
+    "xterm-addon-fit": "^0.5.0",
+    "xterm-addon-web-links": "0.7.0-beta.6"
   },
   "scripts": {
     "lint": "tsc --noEmit",

cmd/tsconnect/src/app/app.tsx

@@ -92,6 +92,12 @@ class App extends Component<{}, AppState> {
   }
 
   handleBrowseToURL = (url: string) => {
+    if (this.state.ipnState === "Running") {
+      // Ignore URL requests if we're already running -- it's most likely an
+      // SSH check mode trigger and we already linkify the displayed URL
+      // in the terminal.
+      return
+    }
     this.setState({ browseToURL: url })
   }
 

cmd/tsconnect/src/app/ssh.tsx

@@ -2,16 +2,24 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-import { useState, useCallback } from "preact/hooks"
+import { useState, useCallback, useMemo, useEffect, useRef } from "preact/hooks"
+import { createPortal } from "preact/compat"
+import type { VNode } from "preact"
 import { runSSHSession, SSHSessionDef } from "../lib/ssh"
 
 export function SSH({ netMap, ipn }: { netMap: IPNNetMap; ipn: IPN }) {
-  const [sshSessionDef, setSSHSessionDef] = useState<SSHSessionDef | null>(null)
+  const [sshSessionDef, setSSHSessionDef] = useState<SSHFormSessionDef | null>(
+    null
+  )
   const clearSSHSessionDef = useCallback(() => setSSHSessionDef(null), [])
   if (sshSessionDef) {
-    return (
+    const sshSession = (
       <SSHSession def={sshSessionDef} ipn={ipn} onDone={clearSSHSessionDef} />
     )
+    if (sshSessionDef.newWindow) {
+      return <NewWindow close={clearSSHSessionDef}>{sshSession}</NewWindow>
+    }
+    return sshSession
   }
   const sshPeers = netMap.peers.filter(
     (p) => p.tailscaleSSHEnabled && p.online !== false
@@ -24,6 +32,8 @@ export function SSH({ netMap, ipn }: { netMap: IPNNetMap; ipn: IPN }) {
   return <SSHForm sshPeers={sshPeers} onSubmit={setSSHSessionDef} />
 }
 
+type SSHFormSessionDef = SSHSessionDef & { newWindow?: boolean }
+
 function SSHSession({
   def,
   ipn,
@@ -33,20 +43,14 @@ function SSHSession({
   ipn: IPN
   onDone: () => void
 }) {
-  return (
-    <div
-      class="flex-grow bg-black p-2 overflow-hidden"
-      ref={(node) => {
-        if (node) {
-          // Run the SSH session aysnchronously, so that the React render
-          // loop is complete (otherwise the SSH form may still be visible,
-          // which affects the size of the terminal, leading to a spurious
-          // initial resize).
-          setTimeout(() => runSSHSession(node, def, ipn, onDone), 0)
-        }
-      }}
-    />
-  )
+  const ref = useRef<HTMLDivElement>(null)
+  useEffect(() => {
+    if (ref.current) {
+      runSSHSession(ref.current, def, ipn, onDone)
+    }
+  }, [ref])
+
+  return <div class="flex-grow bg-black p-2 overflow-hidden" ref={ref} />
 }
 
 function NoSSHPeers() {
@@ -66,7 +70,7 @@ function SSHForm({
   onSubmit,
 }: {
   sshPeers: IPNNetMapPeerNode[]
-  onSubmit: (def: SSHSessionDef) => void
+  onSubmit: (def: SSHFormSessionDef) => void
 }) {
   sshPeers = sshPeers.slice().sort((a, b) => a.name.localeCompare(b.name))
   const [username, setUsername] = useState("")
@@ -99,7 +103,51 @@ function SSHForm({
         type="submit"
         class="button bg-green-500 border-green-500 text-white hover:bg-green-600 hover:border-green-600"
         value="SSH"
+        onClick={(e) => {
+          if (e.altKey) {
+            e.preventDefault()
+            e.stopPropagation()
+            onSubmit({ username, hostname, newWindow: true })
+          }
+        }}
       />
     </form>
   )
 }
+
+const NewWindow = ({
+  children,
+  close,
+}: {
+  children: VNode
+  close: () => void
+}) => {
+  const newWindow = useMemo(() => {
+    const newWindow = window.open(undefined, undefined, "width=600,height=400")
+    if (newWindow) {
+      const containerNode = newWindow.document.createElement("div")
+      containerNode.className = "h-screen flex flex-col overflow-hidden"
+      newWindow.document.body.appendChild(containerNode)
+
+      for (const linkNode of document.querySelectorAll(
+        "head link[rel=stylesheet]"
+      )) {
+        const newLink = document.createElement("link")
+        newLink.rel = "stylesheet"
+        newLink.href = (linkNode as HTMLLinkElement).href
+        newWindow.document.head.appendChild(newLink)
+      }
+    }
+    return newWindow
+  }, [])
+  if (!newWindow) {
+    console.error("Could not open window")
+    return null
+  }
+  newWindow.onbeforeunload = () => {
+    close()
+  }
+
+  useEffect(() => () => newWindow.close(), [])
+  return createPortal(children, newWindow.document.body.lastChild as Element)
+}

cmd/tsconnect/src/lib/ssh.ts

@@ -1,5 +1,6 @@
-import { Terminal } from "xterm"
+import { Terminal, ITerminalOptions } from "xterm"
 import { FitAddon } from "xterm-addon-fit"
+import { WebLinksAddon } from "xterm-addon-web-links"
 
 export type SSHSessionDef = {
   username: string
@@ -10,16 +11,26 @@ export function runSSHSession(
   termContainerNode: HTMLDivElement,
   def: SSHSessionDef,
   ipn: IPN,
-  onDone: () => void
+  onDone: () => void,
+  terminalOptions?: ITerminalOptions
 ) {
+  const parentWindow = termContainerNode.ownerDocument.defaultView ?? window
   const term = new Terminal({
     cursorBlink: true,
+    allowProposedApi: true,
+    ...terminalOptions,
   })
+
   const fitAddon = new FitAddon()
   term.loadAddon(fitAddon)
   term.open(termContainerNode)
   fitAddon.fit()
 
+  const webLinksAddon = new WebLinksAddon((event, uri) =>
+    event.view?.open(uri, "_blank", "noopener")
+  )
+  term.loadAddon(webLinksAddon)
+
   let onDataHook: ((data: string) => void) | undefined
   term.onData((e) => {
     onDataHook?.(e)
@@ -47,19 +58,19 @@ export function runSSHSession(
       resizeObserver?.disconnect()
       term.dispose()
       if (handleBeforeUnload) {
-        window.removeEventListener("beforeunload", handleBeforeUnload)
+        parentWindow.removeEventListener("beforeunload", handleBeforeUnload)
       }
       onDone()
     },
   })
 
   // Make terminal and SSH session track the size of the containing DOM node.
-  resizeObserver = new ResizeObserver(() => fitAddon.fit())
+  resizeObserver = new parentWindow.ResizeObserver(() => fitAddon.fit())
   resizeObserver.observe(termContainerNode)
   term.onResize(({ rows, cols }) => sshSession.resize(rows, cols))
 
   // Close the session if the user closes the window without an explicit
   // exit.
   handleBeforeUnload = () => sshSession.close()
-  window.addEventListener("beforeunload", handleBeforeUnload)
+  parentWindow.addEventListener("beforeunload", handleBeforeUnload)
 }

cmd/tsconnect/src/types/wasm_js.d.ts

@@ -47,6 +47,7 @@ declare global {
     stateStorage?: IPNStateStorage
     authKey?: string
     controlURL?: string
+    hostname?: string
   }
 
   type IPNCallbacks = {

cmd/tsconnect/tsconnect.go

@@ -36,6 +36,8 @@ func main() {
 	switch flag.Arg(0) {
 	case "dev":
 		runDev()
+	case "dev-pkg":
+		runDevPkg()
 	case "build":
 		runBuild()
 	case "build-pkg":

cmd/tsconnect/wasm/wasm_js.go

@@ -61,26 +61,30 @@ func main() {
 func newIPN(jsConfig js.Value) map[string]any {
 	netns.SetEnabled(false)
 
-	jsStateStorage := jsConfig.Get("stateStorage")
 	var store ipn.StateStore
-	if jsStateStorage.IsUndefined() {
-		store = new(mem.Store)
-	} else {
+	if jsStateStorage := jsConfig.Get("stateStorage"); !jsStateStorage.IsUndefined() {
 		store = &jsStateStore{jsStateStorage}
+	} else {
+		store = new(mem.Store)
 	}
 
-	jsControlURL := jsConfig.Get("controlURL")
 	controlURL := ControlURL
-	if jsControlURL.Type() == js.TypeString {
+	if jsControlURL := jsConfig.Get("controlURL"); jsControlURL.Type() == js.TypeString {
 		controlURL = jsControlURL.String()
 	}
 
-	jsAuthKey := jsConfig.Get("authKey")
 	var authKey string
-	if jsAuthKey.Type() == js.TypeString {
+	if jsAuthKey := jsConfig.Get("authKey"); jsAuthKey.Type() == js.TypeString {
 		authKey = jsAuthKey.String()
 	}
 
+	var hostname string
+	if jsHostname := jsConfig.Get("hostname"); jsHostname.Type() == js.TypeString {
+		hostname = jsHostname.String()
+	} else {
+		hostname = generateHostname()
+	}
+
 	lpc := getOrCreateLogPolicyConfig(store)
 	c := logtail.Config{
 		Collection: lpc.Collection,
@@ -136,6 +140,7 @@ func newIPN(jsConfig js.Value) map[string]any {
 		lb:         lb,
 		controlURL: controlURL,
 		authKey:    authKey,
+		hostname:   hostname,
 	}
 
 	return map[string]any{
@@ -196,6 +201,7 @@ type jsIPN struct {
 	lb         *ipnlocal.LocalBackend
 	controlURL string
 	authKey    string
+	hostname   string
 }
 
 var jsIPNState = map[ipn.State]string{
@@ -284,7 +290,7 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 				RouteAll:         false,
 				AllowSingleHosts: true,
 				WantRunning:      true,
-				Hostname:         generateHostname(),
+				Hostname:         i.hostname,
 			},
 			AuthKey: i.authKey,
 		})
@@ -343,6 +349,9 @@ type jsSSHSession struct {
 	username   string
 	termConfig js.Value
 	session    *ssh.Session
+
+	pendingResizeRows int
+	pendingResizeCols int
 }
 
 func (s *jsSSHSession) Run() {
@@ -354,9 +363,6 @@ func (s *jsSSHSession) Run() {
 	onDone := s.termConfig.Get("onDone")
 	defer onDone.Invoke()
 
-	write := func(s string) {
-		writeFn.Invoke(s)
-	}
 	writeError := func(label string, err error) {
 		writeErrorFn.Invoke(fmt.Sprintf("%s Error: %v\r\n", label, err))
 	}
@@ -381,7 +387,6 @@ func (s *jsSSHSession) Run() {
 		return
 	}
 	defer sshConn.Close()
-	write("SSH Connected\r\n")
 
 	sshClient := ssh.NewClient(sshConn, nil, nil)
 	defer sshClient.Close()
@@ -392,7 +397,6 @@ func (s *jsSSHSession) Run() {
 		return
 	}
 	s.session = session
-	write("Session Established\r\n")
 	defer session.Close()
 
 	stdin, err := session.StdinPipe()
@@ -413,6 +417,14 @@ func (s *jsSSHSession) Run() {
 		return nil
 	}))
 
+	// We might have gotten a resize notification since we started opening the
+	// session, pick up the latest size.
+	if s.pendingResizeRows != 0 {
+		rows = s.pendingResizeRows
+	}
+	if s.pendingResizeCols != 0 {
+		cols = s.pendingResizeCols
+	}
 	err = session.RequestPty("xterm", rows, cols, ssh.TerminalModes{})
 
 	if err != nil {
@@ -438,6 +450,11 @@ func (s *jsSSHSession) Close() error {
 }
 
 func (s *jsSSHSession) Resize(rows, cols int) error {
+	if s.session == nil {
+		s.pendingResizeRows = rows
+		s.pendingResizeCols = cols
+		return nil
+	}
 	return s.session.WindowChange(rows, cols)
 }
 

cmd/tsconnect/yarn.lock

@@ -644,10 +644,15 @@ xterm-addon-fit@^0.5.0:
   resolved "https://registry.yarnpkg.com/xterm-addon-fit/-/xterm-addon-fit-0.5.0.tgz#2d51b983b786a97dcd6cde805e700c7f913bc596"
   integrity sha512-DsS9fqhXHacEmsPxBJZvfj2la30Iz9xk+UKjhQgnYNkrUIN5CYLbw7WEfz117c7+S86S/tpHPfvNxJsF5/G8wQ==
 
-xterm@^4.18.0:
-  version "4.18.0"
-  resolved "https://registry.yarnpkg.com/xterm/-/xterm-4.18.0.tgz#a1f6ab2c330c3918fb094ae5f4c2562987398ea1"
-  integrity sha512-JQoc1S0dti6SQfI0bK1AZvGnAxH4MVw45ZPFSO6FHTInAiau3Ix77fSxNx3mX4eh9OL4AYa8+4C8f5UvnSfppQ==
+xterm@5.0.0-beta.58:
+  version "5.0.0-beta.58"
+  resolved "https://registry.yarnpkg.com/xterm/-/xterm-5.0.0-beta.58.tgz#e3e96ab9fd24d006ec16cc9351a060cc79e67e80"
+  integrity sha512-gjg39oKdgUKful27+7I1hvSK51lu/LRhdimFhfZyMvdk0iATH0FAfzv1eAvBKWY2UBgYUfxhicTkanYioANdMw==
+
+xterm-addon-web-links@0.7.0-beta.6:
+  version "0.7.0-beta.6"
+  resolved "https://registry.yarnpkg.com/xterm-addon-web-links/-/xterm-addon-web-links-0.7.0-beta.6.tgz#ec63b681b4f0f0135fa039f53664f65fe9d9f43a"
+  integrity sha512-nD/r/GchGTN4c9gAIVLWVoxExTzAUV7E9xZnwsvhuwI4CEE6yqO15ns8g2hdcUrsPyCbNEw05mIrkF6W5Yj8qA==
 
 y18n@^4.0.0:
   version "4.0.3"

control/controlclient/direct.go

@@ -937,6 +937,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, readOnly bool
 			}
 			if resp.Debug.DisableLogTail {
 				logtail.Disable()
+				envknob.SetNoLogsNoSupport()
 			}
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)

derp/derp_server.go

@@ -107,6 +107,9 @@ type Server struct {
 	metaCert    []byte // the encoded x509 cert to send after LetsEncrypt cert+intermediate
 	dupPolicy   dupPolicy
 
+	clientDataLimit *uint64 // limit for how many bytes/s of content a client can send; atomic
+	clientDataBurst int     // burst limit for how many bytes/s of content a client can send
+
 	// Counters:
 	packetsSent, bytesSent       expvar.Int
 	packetsRecv, bytesRecv       expvar.Int
@@ -314,7 +317,10 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 		sentTo:               map[key.NodePublic]map[key.NodePublic]int64{},
 		avgQueueDuration:     new(uint64),
 		keyOfAddr:            map[netip.AddrPort]key.NodePublic{},
+		clientDataLimit:      new(uint64),
+		clientDataBurst:      10 * 1024 * 1024, // 10Mb default burst
 	}
+	atomic.StoreUint64(s.clientDataLimit, 12*1024*1024) // 12Mb/s default ratelimit
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
 	s.packetsRecvOther = s.packetsRecvByKind.Get("other")
@@ -325,12 +331,48 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 		s.packetsDroppedReason.Get("queue_head"),
 		s.packetsDroppedReason.Get("queue_tail"),
 		s.packetsDroppedReason.Get("write_error"),
+		s.packetsDroppedReason.Get("rate_limited"),
 	}
 	s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
 	s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
 	return s
 }
 
+// StartEgressRateLimiter starts dynamically adjusting the rate limit
+// based on the desired limit and the utilization of the specified interface.
+//
+// It must be called before serving begins. All limits are in bytes/s.
+func (s *Server) StartEgressRateLimiter(interfaceName string, egressDataLimit, clientDataMin, clientDataBurst int) error {
+	limiter, err := newEgressLimiter(interfaceName, uint64(egressDataLimit), uint64(clientDataMin))
+	if err != nil {
+		return fmt.Errorf("starting limiter: %v", err)
+	}
+
+	atomic.StoreUint64(s.clientDataLimit, uint64(egressDataLimit))
+	s.clientDataBurst = clientDataBurst
+
+	go func() {
+		t := time.NewTicker(time.Second)
+		defer t.Stop()
+
+		for {
+			limit, err := limiter.Limit()
+			if err != nil {
+				s.logf("derp: failed to update egress limiter: %v", err)
+				return
+			}
+			atomic.StoreUint64(s.clientDataLimit, uint64(limit))
+
+			<-t.C
+			if s.closed {
+				return
+			}
+		}
+	}()
+
+	return nil
+}
+
 // SetMesh sets the pre-shared key that regional DERP servers used to mesh
 // amongst themselves.
 //
@@ -664,6 +706,7 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 
 	remoteIPPort, _ := netip.ParseAddrPort(remoteAddr)
 
+	rateLimiter := rate.NewLimiter(rate.Limit(atomic.LoadUint64(s.clientDataLimit)), s.clientDataBurst)
 	c := &sclient{
 		connNum:        connNum,
 		s:              s,
@@ -681,6 +724,7 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 		sendPongCh:     make(chan [8]byte, 1),
 		peerGone:       make(chan key.NodePublic),
 		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+		rateLimiter:    rateLimiter,
 	}
 
 	if c.canMesh {
@@ -757,6 +801,18 @@ func (c *sclient) run(ctx context.Context) error {
 	}
 }
 
+func (c *sclient) shouldRatelimitData(dataLen int) bool {
+	if c.canMesh {
+		return false // Mesh connections arent regular clients.
+	}
+
+	now := time.Now()
+	if rateLimit := atomic.LoadUint64(c.s.clientDataLimit); rateLimit != uint64(c.rateLimiter.Limit()) {
+		c.rateLimiter.SetLimitAt(now, rate.Limit(rateLimit))
+	}
+	return !c.rateLimiter.AllowN(now, dataLen)
+}
+
 func (c *sclient) handleUnknownFrame(ft frameType, fl uint32) error {
 	_, err := io.CopyN(ioutil.Discard, c.br, int64(fl))
 	return err
@@ -858,6 +914,11 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 	}
 	s.packetsForwardedIn.Add(1)
 
+	if c.shouldRatelimitData(len(contents)) {
+		s.recordDrop(contents, c.key, dstKey, dropReasonRateLimited)
+		return nil
+	}
+
 	var dstLen int
 	var dst *sclient
 
@@ -908,6 +969,11 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 		return fmt.Errorf("client %x: recvPacket: %v", c.key, err)
 	}
 
+	if c.shouldRatelimitData(len(contents)) {
+		s.recordDrop(contents, c.key, dstKey, dropReasonRateLimited)
+		return nil
+	}
+
 	var fwd PacketForwarder
 	var dstLen int
 	var dst *sclient
@@ -962,6 +1028,7 @@ const (
 	dropReasonQueueTail                          // destination queue is full, dropped packet at queue tail
 	dropReasonWriteError                         // OS write() failed
 	dropReasonDupClient                          // the public key is connected 2+ times (active/active, fighting)
+	dropReasonRateLimited                        // send/forward packet content exceeds rate limit
 )
 
 func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) {
@@ -1254,6 +1321,7 @@ type sclient struct {
 	canMesh        bool                // clientInfo had correct mesh token for inter-region routing
 	isDup          atomic.Bool         // whether more than 1 sclient for key is connected
 	isDisabled     atomic.Bool         // whether sends to this peer are disabled due to active/active dups
+	rateLimiter    *rate.Limiter
 
 	// replaceLimiter controls how quickly two connections with
 	// the same client key can kick each other off the server by
@@ -1700,6 +1768,7 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("average_queue_duration_ms", expvar.Func(func() any {
 		return math.Float64frombits(atomic.LoadUint64(s.avgQueueDuration))
 	}))
+	m.Set("client_ratelimit_bytes_per_second", expvar.Func(func() any { return atomic.LoadUint64(s.clientDataLimit) }))
 	var expvarVersion expvar.String
 	expvarVersion.Set(version.Long)
 	m.Set("version", &expvarVersion)

derp/dropreason_string.go

@@ -19,11 +19,12 @@ func _() {
 	_ = x[dropReasonQueueTail-4]
 	_ = x[dropReasonWriteError-5]
 	_ = x[dropReasonDupClient-6]
+	_ = x[dropReasonRateLimited-7]
 }
 
-const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneQueueHeadQueueTailWriteErrorDupClient"
+const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneQueueHeadQueueTailWriteErrorDupClientRateLimited"
 
-var _dropReason_index = [...]uint8{0, 11, 27, 31, 40, 49, 59, 68}
+var _dropReason_index = [...]uint8{0, 11, 27, 31, 40, 49, 59, 68, 79}
 
 func (i dropReason) String() string {
 	if i < 0 || i >= dropReason(len(_dropReason_index)-1) {

derp/limiter.go

@@ -0,0 +1,171 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package derp
+
+import (
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func readTxBytes(interfaceName string) (uint64, error) {
+	v, err := ioutil.ReadFile("/sys/class/net/" + interfaceName + "/statistics/tx_bytes")
+	if err != nil {
+		return 0, err
+	}
+	tx, err := strconv.Atoi(strings.TrimSpace(string(v)))
+	if err != nil {
+		return 0, err
+	}
+	return uint64(tx), nil
+}
+
+type egressLimiter struct {
+	interfaceName string
+	limitBytesSec uint64 // the egress bytes/s we want to stay under.
+	minBytesSec   uint64 // the minimum bytes/s rate limit.
+
+	lastTxBytes uint64
+	controlLoop limiterLoop
+}
+
+func newEgressLimiter(interfaceName string, limitBytesSec, minBytesSec uint64) (*egressLimiter, error) {
+	initial, err := readTxBytes(interfaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &egressLimiter{
+		interfaceName: interfaceName,
+		limitBytesSec: limitBytesSec,
+		minBytesSec:   minBytesSec,
+		lastTxBytes:   initial,
+		controlLoop:   newLimiterLoop(limitBytesSec, time.Now()),
+	}, err
+}
+
+// Limit returns the current rate limit value based on interface utilization.
+func (e *egressLimiter) Limit() (uint64, error) {
+	rx, err := readTxBytes(e.interfaceName)
+	if err != nil {
+		return 0, err
+	}
+
+	last := e.lastTxBytes
+	e.lastTxBytes = rx
+
+	limit := e.controlLoop.tick(uint64(rx)-last, time.Now())
+	if limit < 0 || uint64(limit) < e.minBytesSec {
+		limit = float64(e.minBytesSec)
+	}
+	if uint64(limit) > e.limitBytesSec {
+		limit = float64(e.limitBytesSec)
+	}
+	return uint64(limit), nil
+}
+
+// PID loop values for the dynamic ratelimit.
+// The wikipedia page on PID is recommended reading if you are not familiar
+// with PID loops or open-loop control theory.
+//
+// Gain values are unitless, but operate on a feedback value in bytes
+// and a setpoint value in bytes/s, and a time delta (dt) of seconds.
+//
+// These values are initial and should be tuned: These are just initial
+// values based on first principles and vibin with pretty graphs.
+const (
+	// Proportional gain.
+	// Given this represents a global ratelimit, the P term doesnt make a lot of
+	// sense, as each clients contribution to link utilization is entirely
+	// dependent on the client workload.
+	//
+	// For this reason, its set super low: Its expected the I term will do
+	// most of the heavy lifting.
+	limiterP float64 = 1.0 / 1024
+	// Derivative gain.
+	// This term reacts against 'trends', that is, the first derivative of
+	// the feedback value. Think of it like a rapid-change damper.
+	//
+	// This isnt super important, so again we've set it fairly low.
+	limiterD float64 = 0.003
+	// Integral gain.
+	//
+	// This is where all the heavy lifting happens. Basically, we increase
+	// the ratelimit (by limiterIP) when we have room to spare, and
+	// decrease it once we exceed 4/5ths of the limit (by limiterIN).
+	// The increase is linear to the error between feedback and the setpoint,
+	// but clamped proportionate to the limit.
+	//
+	// The decrease term is stronger than the increase term, so we 'backoff
+	// quickly' when we are approaching limits, but test the waters on
+	// the other end cautiously.
+	limiterIP float64 = 0.008
+	limiterIN float64 = 0.3
+)
+
+// limiterLoop exposes a dynamic ratelimit, based on the egress rate
+// of some interface. The PID loop tries to keep egress at 4/5 of the limit.
+type limiterLoop struct {
+	limitBytesSec uint64 // the egress bytes/s we want to stay under.
+
+	integral   float64   // the integral sum at lastUpdate instant
+	lastEgress uint64    // feedback value of previous iteration, bytes/s
+	lastUpdate time.Time // instant at which last iteration occurred.
+}
+
+func newLimiterLoop(limitBytesSec uint64, now time.Time) limiterLoop {
+	return limiterLoop{
+		limitBytesSec: limitBytesSec * 4 / 5,
+		lastUpdate:    now,
+		lastEgress:    0,
+		integral:      float64(limitBytesSec),
+	}
+}
+
+// tick computes & returns the ratelimit value in bytes/s, computing
+// the next iteration of the PID loop in the process.
+func (l *limiterLoop) tick(egressBytesPerSec uint64, now time.Time) float64 {
+	var (
+		dt  = now.Sub(l.lastUpdate).Seconds()
+		err = float64(l.limitBytesSec) - float64(egressBytesPerSec)
+	)
+
+	// Integral term.
+	var iDelta float64
+	if err > 0 {
+		iDelta = err * dt * limiterIP
+	} else {
+		iDelta = err * dt * limiterIN
+	}
+	// Constrain integral sum change to a 20th of the setpoint per second.
+	maxDelta := dt * float64(l.limitBytesSec) / 20
+	if iDelta > maxDelta {
+		iDelta = maxDelta
+	} else if iDelta < -maxDelta {
+		iDelta = -maxDelta
+	}
+	l.integral += iDelta
+	// Constrain integral sum to prevent windup.
+	if max := float64(l.limitBytesSec); l.integral > max {
+		l.integral = max
+	} else if l.integral < -max {
+		l.integral = -max
+	}
+
+	// Derivative term.
+	var d float64
+	if dt > 0 {
+		d = -(float64(egressBytesPerSec-l.lastEgress) / dt) * limiterD
+	}
+	// Proportional term.
+	p := limiterP * err
+
+	l.lastEgress = egressBytesPerSec
+	l.lastUpdate = now
+	output := p + l.integral + d
+	// fmt.Printf("in=%d, out=%0.3f:   p=%0.2f d=%0.2f i=%0.2f\n", egressBytesPerSec, output, p, d, l.integral)
+	return output
+}

derp/limiter_test.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package derp
+
+import (
+	"testing"
+	"time"
+)
+
+func mb(mb uint64) uint64 {
+	return mb * 1024 * 1024
+}
+
+func TestLimiterLoopGradual(t *testing.T) {
+	// Make a limiter that tries to keep under 200Mb/s.
+	limit := mb(200)
+	start := time.Now()
+	l := newLimiterLoop(limit, start)
+
+	// Make sure the initial value is sane.
+	// Lets imagine the egress is only like 1Mb/s.
+	now := start.Add(time.Second)
+	if v := uint64(l.tick(1024*1024, now)); v < mb(150) || v > limit {
+		t.Errorf("initial value = %dMb/s, want 150 < value < limit", v/1024/1024)
+	}
+
+	// Tick through 10 minutes of low usage. Lets make sure the limit stays high.
+	lowUsage := limit / 10
+	for i := 0; i < 600; i++ {
+		now = now.Add(time.Second)
+		v := uint64(l.tick(lowUsage, now))
+
+		if v < mb(150) {
+			t.Errorf("[t=%0.f] limit too low for low usage: %d (expected >150)", now.Sub(start).Seconds(), v/1024/1024)
+		}
+	}
+
+	// Lets tick through 60 seconds of steadily-increasing usage.
+	for i := 0; i < 60; i++ {
+		now = now.Add(time.Second)
+		l.tick(uint64(i)*limit/60, now)
+	}
+	if v := uint64(l.tick(limit, now)); v > mb(100) || v < mb(1) {
+		t.Errorf("[t=%0.f] limit = %dMb/s, want 1-100Mb/s", now.Sub(start).Seconds(), v/1024/1024)
+	}
+	// Lets imagine we are at limits for 10s. Does the limit drop pretty hard?
+	for i := 0; i < 10; i++ {
+		now = now.Add(time.Second)
+		l.tick(limit, now)
+	}
+	if v := uint64(l.tick(limit, now)); v > mb(20) || v < mb(1) {
+		t.Errorf("[t=%0.f] limit = %dMb/s, want 1-20Mb/s", now.Sub(start).Seconds(), v/1024/1024)
+	}
+}

docs/k8s/run.sh

@@ -4,8 +4,6 @@
 
 #! /bin/sh
 
-set -m # enable job control
-
 export PATH=$PATH:/tailscale/bin
 
 TS_AUTH_KEY="${TS_AUTH_KEY:-}"
@@ -60,8 +58,16 @@ if [[ ! -z "${TS_TAILSCALED_EXTRA_ARGS}" ]]; then
   TAILSCALED_ARGS="${TAILSCALED_ARGS} ${TS_TAILSCALED_EXTRA_ARGS}"
 fi
 
+handler() {
+  echo "Caught SIGINT/SIGTERM, shutting down tailscaled"
+  kill -s SIGINT $PID
+  wait ${PID}
+}
+
 echo "Starting tailscaled"
 tailscaled ${TAILSCALED_ARGS} &
+PID=$!
+trap handler SIGINT SIGTERM
 
 UP_ARGS="--accept-dns=${TS_ACCEPT_DNS}"
 if [[ ! -z "${TS_ROUTES}" ]]; then
@@ -82,4 +88,5 @@ if [[ ! -z "${TS_DEST_IP}" ]]; then
   iptables -t nat -I PREROUTING -d "$(tailscale --socket=/tmp/tailscaled.sock ip -4)" -j DNAT --to-destination "${TS_DEST_IP}"
 fi
 
-fg
+echo "Waiting for tailscaled to exit"
+wait ${PID}

envknob/envknob.go

@@ -155,3 +155,14 @@ func SSHPolicyFile() string { return String("TS_DEBUG_SSH_POLICY_FILE") }
 
 // SSHIgnoreTailnetPolicy is whether to ignore the Tailnet SSH policy for development.
 func SSHIgnoreTailnetPolicy() bool { return Bool("TS_DEBUG_SSH_IGNORE_TAILNET_POLICY") }
+
+// NoLogsNoSupport reports whether the client's opted out of log uploads and
+// technical support.
+func NoLogsNoSupport() bool {
+	return Bool("TS_NO_LOGS_NO_SUPPORT")
+}
+
+// SetNoLogsNoSupport enables no-logs-no-support mode.
+func SetNoLogsNoSupport() {
+	os.Setenv("TS_NO_LOGS_NO_SUPPORT", "true")
+}

go.mod

@@ -63,9 +63,9 @@ require (
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
 	golang.org/x/tools v0.1.11
-	golang.zx2c4.com/wireguard v0.0.0-20220703234212-c31a7b1ab478
+	golang.zx2c4.com/wireguard v0.0.0-20220904105730-b51010ba13f0
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444
+	gvisor.dev/gvisor v0.0.0-20220817001344-846276b3dbc5
 	honnef.co/go/tools v0.4.0-0.dev.0.20220404092545-59d7a2877f83
 	inet.af/peercred v0.0.0-20210906144145-0893ea02156a
 	inet.af/wf v0.0.0-20220728202103-50d96caab2f6

go.sum

@@ -1633,8 +1633,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 h1:Ug9qvr1myri/zFN6xL17LSCBGFDnphBBhzmILHsM5TY=
 golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
-golang.zx2c4.com/wireguard v0.0.0-20220703234212-c31a7b1ab478 h1:vDy//hdR+GnROE3OdYbQKt9rdtNdHkDtONvpRwmls/0=
-golang.zx2c4.com/wireguard v0.0.0-20220703234212-c31a7b1ab478/go.mod h1:bVQfyl2sCM/QIIGHpWbFGfHPuDvqnCNkT6MQLTCjO/U=
+golang.zx2c4.com/wireguard v0.0.0-20220904105730-b51010ba13f0 h1:5ZkdpbduT/g+9OtbSDvbF3KvfQG45CtH/ppO8FUmvCQ=
+golang.zx2c4.com/wireguard v0.0.0-20220904105730-b51010ba13f0/go.mod h1:enML0deDxY1ux+B6ANGiwtg0yAJi1rctkTpcHNAVPyg=
 golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
 golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1816,8 +1816,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
-gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444 h1:0d3ygmOM5RgQB8rmsZNeAY/7Q98fKt1HrGO2XIp4pDI=
-gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444/go.mod h1:TIvkJD0sxe8pIob3p6T8IzxXunlp6yfgktvTNp+DGNM=
+gvisor.dev/gvisor v0.0.0-20220817001344-846276b3dbc5 h1:cv/zaNV0nr1mJzaeo4S5mHIm5va1W0/9J3/5prlsuRM=
+gvisor.dev/gvisor v0.0.0-20220817001344-846276b3dbc5/go.mod h1:TIvkJD0sxe8pIob3p6T8IzxXunlp6yfgktvTNp+DGNM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

go.toolchain.rev

@@ -1 +1 @@
-6dca83b256c7decd3dd6706ee47e04f21a0b935c
+b13188dd36c1ad2509796ce10b6a1231b200c36a

hostinfo/hostinfo.go

@@ -12,10 +12,12 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"go4.org/mem"
+	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/cloudenv"
@@ -31,25 +33,69 @@ func New() *tailcfg.Hostinfo {
 	hostname, _ := os.Hostname()
 	hostname = dnsname.FirstLabel(hostname)
 	return &tailcfg.Hostinfo{
-		IPNVersion:  version.Long,
-		Hostname:    hostname,
-		OS:          version.OS(),
-		OSVersion:   GetOSVersion(),
-		Desktop:     desktop(),
-		Package:     packageTypeCached(),
-		GoArch:      runtime.GOARCH,
-		GoVersion:   runtime.Version(),
-		DeviceModel: deviceModel(),
-		Cloud:       string(cloudenv.Get()),
+		IPNVersion:      version.Long,
+		Hostname:        hostname,
+		OS:              version.OS(),
+		OSVersion:       GetOSVersion(),
+		Container:       lazyInContainer.Get(),
+		Distro:          condCall(distroName),
+		DistroVersion:   condCall(distroVersion),
+		DistroCodeName:  condCall(distroCodeName),
+		Env:             string(GetEnvType()),
+		Desktop:         desktop(),
+		Package:         packageTypeCached(),
+		GoArch:          runtime.GOARCH,
+		GoVersion:       runtime.Version(),
+		DeviceModel:     deviceModel(),
+		Cloud:           string(cloudenv.Get()),
+		NoLogsNoSupport: envknob.NoLogsNoSupport(),
 	}
 }
 
 // non-nil on some platforms
 var (
-	osVersion   func() string
-	packageType func() string
+	osVersion      func() string
+	packageType    func() string
+	distroName     func() string
+	distroVersion  func() string
+	distroCodeName func() string
 )
 
+func condCall[T any](fn func() T) T {
+	var zero T
+	if fn == nil {
+		return zero
+	}
+	return fn()
+}
+
+var (
+	lazyInContainer = &lazyAtomicValue[opt.Bool]{f: ptrTo(inContainer)}
+)
+
+func ptrTo[T any](v T) *T { return &v }
+
+type lazyAtomicValue[T any] struct {
+	// f is a pointer to a fill function. If it's nil or points
+	// to nil, then Get returns the zero value for T.
+	f *func() T
+
+	once sync.Once
+	v    T
+}
+
+func (v *lazyAtomicValue[T]) Get() T {
+	v.once.Do(v.fill)
+	return v.v
+}
+
+func (v *lazyAtomicValue[T]) fill() {
+	if v.f == nil || *v.f == nil {
+		return
+	}
+	v.v = (*v.f)()
+}
+
 // GetOSVersion returns the OSVersion of current host if available.
 func GetOSVersion() string {
 	if s, _ := osVersionAtomic.Load().(string); s != "" {
@@ -179,22 +225,32 @@ func getEnvType() EnvType {
 }
 
 // inContainer reports whether we're running in a container.
-func inContainer() bool {
+func inContainer() opt.Bool {
 	if runtime.GOOS != "linux" {
-		return false
+		return ""
+	}
+	var ret opt.Bool
+	ret.Set(false)
+	if _, err := os.Stat("/.dockerenv"); err == nil {
+		ret.Set(true)
+		return ret
+	}
+	if _, err := os.Stat("/run/.containerenv"); err == nil {
+		// See https://github.com/cri-o/cri-o/issues/5461
+		ret.Set(true)
+		return ret
 	}
-	var ret bool
 	lineread.File("/proc/1/cgroup", func(line []byte) error {
 		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
 			mem.Contains(mem.B(line), mem.S("/lxc/")) {
-			ret = true
+			ret.Set(true)
 			return io.EOF // arbitrary non-nil error to stop loop
 		}
 		return nil
 	})
 	lineread.File("/proc/mounts", func(line []byte) error {
 		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
-			ret = true
+			ret.Set(true)
 			return io.EOF
 		}
 		return nil

hostinfo/hostinfo_freebsd.go

@@ -8,48 +8,58 @@
 package hostinfo
 
 import (
-	"fmt"
+	"bytes"
 	"os"
 	"os/exec"
-	"strings"
 
 	"golang.org/x/sys/unix"
 	"tailscale.com/version/distro"
 )
 
 func init() {
-	osVersion = osVersionFreebsd
+	osVersion = lazyOSVersion.Get
+	distroName = distroNameFreeBSD
+	distroVersion = distroVersionFreeBSD
 }
 
-func osVersionFreebsd() string {
-	un := unix.Utsname{}
+var (
+	lazyVersionMeta = &lazyAtomicValue[versionMeta]{f: ptrTo(freebsdVersionMeta)}
+	lazyOSVersion   = &lazyAtomicValue[string]{f: ptrTo(osVersionFreeBSD)}
+)
+
+func distroNameFreeBSD() string {
+	return lazyVersionMeta.Get().DistroName
+}
+
+func distroVersionFreeBSD() string {
+	return lazyVersionMeta.Get().DistroVersion
+}
+
+type versionMeta struct {
+	DistroName     string
+	DistroVersion  string
+	DistroCodeName string
+}
+
+func osVersionFreeBSD() string {
+	var un unix.Utsname
 	unix.Uname(&un)
+	return unix.ByteSliceToString(un.Release[:])
+}
 
-	var attrBuf strings.Builder
-	attrBuf.WriteString("; version=")
-	attrBuf.WriteString(unix.ByteSliceToString(un.Release[:]))
-	attr := attrBuf.String()
-
-	version := "FreeBSD"
-	switch distro.Get() {
+func freebsdVersionMeta() (meta versionMeta) {
+	d := distro.Get()
+	meta.DistroName = string(d)
+	switch d {
 	case distro.Pfsense:
 		b, _ := os.ReadFile("/etc/version")
-		version = fmt.Sprintf("pfSense %s", b)
+		meta.DistroVersion = string(bytes.TrimSpace(b))
 	case distro.OPNsense:
-		b, err := exec.Command("opnsense-version").Output()
-		if err == nil {
-			version = string(b)
-		} else {
-			version = "OPNsense"
-		}
+		b, _ := exec.Command("opnsense-version").Output()
+		meta.DistroVersion = string(bytes.TrimSpace(b))
 	case distro.TrueNAS:
-		b, err := os.ReadFile("/etc/version")
-		if err == nil {
-			version = string(b)
-		} else {
-			version = "TrueNAS"
-		}
+		b, _ := os.ReadFile("/etc/version")
+		meta.DistroVersion = string(bytes.TrimSpace(b))
 	}
-	// the /etc/version files end in a newline
-	return fmt.Sprintf("%s%s", strings.TrimSuffix(version, "\n"), attr)
+	return
 }

hostinfo/hostinfo_linux.go

@@ -9,7 +9,6 @@ package hostinfo
 
 import (
 	"bytes"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -21,14 +20,39 @@ import (
 )
 
 func init() {
-	osVersion = osVersionLinux
+	osVersion = lazyOSVersion.Get
 	packageType = packageTypeLinux
-
+	distroName = distroNameLinux
+	distroVersion = distroVersionLinux
+	distroCodeName = distroCodeNameLinux
 	if v := linuxDeviceModel(); v != "" {
 		SetDeviceModel(v)
 	}
 }
 
+var (
+	lazyVersionMeta = &lazyAtomicValue[versionMeta]{f: ptrTo(linuxVersionMeta)}
+	lazyOSVersion   = &lazyAtomicValue[string]{f: ptrTo(osVersionLinux)}
+)
+
+type versionMeta struct {
+	DistroName     string
+	DistroVersion  string
+	DistroCodeName string // "jammy", etc (VERSION_CODENAME from /etc/os-release)
+}
+
+func distroNameLinux() string {
+	return lazyVersionMeta.Get().DistroName
+}
+
+func distroVersionLinux() string {
+	return lazyVersionMeta.Get().DistroVersion
+}
+
+func distroCodeNameLinux() string {
+	return lazyVersionMeta.Get().DistroCodeName
+}
+
 func linuxDeviceModel() string {
 	for _, path := range []string{
 		// First try the Synology-specific location.
@@ -52,15 +76,22 @@ func linuxDeviceModel() string {
 func getQnapQtsVersion(versionInfo string) string {
 	for _, field := range strings.Fields(versionInfo) {
 		if suffix, ok := strs.CutPrefix(field, "QTSFW_"); ok {
-			return "QTS " + suffix
+			return suffix
 		}
 	}
 	return ""
 }
 
 func osVersionLinux() string {
-	// TODO(bradfitz,dgentry): cache this, or make caller(s) cache it.
+	var un unix.Utsname
+	unix.Uname(&un)
+	return unix.ByteSliceToString(un.Release[:])
+}
+
+func linuxVersionMeta() (meta versionMeta) {
 	dist := distro.Get()
+	meta.DistroName = string(dist)
+
 	propFile := "/etc/os-release"
 	switch dist {
 	case distro.Synology:
@@ -69,10 +100,12 @@ func osVersionLinux() string {
 		propFile = "/etc/openwrt_release"
 	case distro.WDMyCloud:
 		slurp, _ := ioutil.ReadFile("/etc/version")
-		return fmt.Sprintf("%s", string(bytes.TrimSpace(slurp)))
+		meta.DistroVersion = string(bytes.TrimSpace(slurp))
+		return
 	case distro.QNAP:
 		slurp, _ := ioutil.ReadFile("/etc/version_info")
-		return getQnapQtsVersion(string(slurp))
+		meta.DistroVersion = getQnapQtsVersion(string(slurp))
+		return
 	}
 
 	m := map[string]string{}
@@ -86,50 +119,45 @@ func osVersionLinux() string {
 		return nil
 	})
 
-	var un unix.Utsname
-	unix.Uname(&un)
-
-	var attrBuf strings.Builder
-	attrBuf.WriteString("; kernel=")
-	attrBuf.WriteString(unix.ByteSliceToString(un.Release[:]))
-	if inContainer() {
-		attrBuf.WriteString("; container")
+	if v := m["VERSION_CODENAME"]; v != "" {
+		meta.DistroCodeName = v
 	}
-	if env := GetEnvType(); env != "" {
-		fmt.Fprintf(&attrBuf, "; env=%s", env)
+	if v := m["VERSION_ID"]; v != "" {
+		meta.DistroVersion = v
 	}
-	attr := attrBuf.String()
-
 	id := m["ID"]
-
+	if id != "" {
+		meta.DistroName = id
+	}
 	switch id {
 	case "debian":
+		// Debian's VERSION_ID is just like "11". But /etc/debian_version has "11.5" normally.
+		// Or "bookworm/sid" on sid/testing.
 		slurp, _ := ioutil.ReadFile("/etc/debian_version")
-		return fmt.Sprintf("Debian %s (%s)%s", bytes.TrimSpace(slurp), m["VERSION_CODENAME"], attr)
-	case "ubuntu":
-		return fmt.Sprintf("Ubuntu %s%s", m["VERSION"], attr)
+		if v := string(bytes.TrimSpace(slurp)); v != "" {
+			if '0' <= v[0] && v[0] <= '9' {
+				meta.DistroVersion = v
+			} else if meta.DistroCodeName == "" {
+				meta.DistroCodeName = v
+			}
+		}
 	case "", "centos": // CentOS 6 has no /etc/os-release, so its id is ""
-		if cr, _ := ioutil.ReadFile("/etc/centos-release"); len(cr) > 0 { // "CentOS release 6.10 (Final)
-			return fmt.Sprintf("%s%s", bytes.TrimSpace(cr), attr)
-		}
-		fallthrough
-	case "fedora", "rhel", "alpine", "nixos":
-		// Their PRETTY_NAME is fine as-is for all versions I tested.
-		fallthrough
-	default:
-		if v := m["PRETTY_NAME"]; v != "" {
-			return fmt.Sprintf("%s%s", v, attr)
+		if meta.DistroVersion == "" {
+			if cr, _ := ioutil.ReadFile("/etc/centos-release"); len(cr) > 0 { // "CentOS release 6.10 (Final)
+				meta.DistroVersion = string(bytes.TrimSpace(cr))
+			}
 		}
 	}
+	if v := m["PRETTY_NAME"]; v != "" && meta.DistroVersion == "" && !strings.HasSuffix(v, "/sid") {
+		meta.DistroVersion = v
+	}
 	switch dist {
 	case distro.Synology:
-		return fmt.Sprintf("Synology %s%s", m["productversion"], attr)
+		meta.DistroVersion = m["productversion"]
 	case distro.OpenWrt:
-		return fmt.Sprintf("OpenWrt %s%s", m["DISTRIB_RELEASE"], attr)
-	case distro.Gokrazy:
-		return fmt.Sprintf("Gokrazy%s", attr)
+		meta.DistroVersion = m["DISTRIB_RELEASE"]
 	}
-	return fmt.Sprintf("Other%s", attr)
+	return
 }
 
 func packageTypeLinux() string {

hostinfo/hostinfo_linux_test.go

@@ -19,7 +19,7 @@ Date:   2022-05-30 16:08:45 +0800
 remotes/origin/QTSFW_5.0.0`
 
 	got := getQnapQtsVersion(version_info)
-	want := "QTS 5.0.0"
+	want := "5.0.0"
 	if got != want {
 		t.Errorf("got %q; want %q", got, want)
 	}

hostinfo/hostinfo_windows.go

@@ -11,21 +11,20 @@ import (
 
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/registry"
-	"tailscale.com/syncs"
 	"tailscale.com/util/winutil"
 )
 
 func init() {
-	osVersion = osVersionWindows
-	packageType = packageTypeWindows
+	osVersion = lazyOSVersion.Get
+	packageType = lazyPackageType.Get
 }
 
-var winVerCache syncs.AtomicValue[string]
+var (
+	lazyOSVersion   = &lazyAtomicValue[string]{f: ptrTo(osVersionWindows)}
+	lazyPackageType = &lazyAtomicValue[string]{f: ptrTo(packageTypeWindows)}
+)
 
 func osVersionWindows() string {
-	if s, ok := winVerCache.LoadOk(); ok {
-		return s
-	}
 	major, minor, build := windows.RtlGetNtVersionNumbers()
 	s := fmt.Sprintf("%d.%d.%d", major, minor, build)
 	// Windows 11 still uses 10 as its major number internally
@@ -34,9 +33,6 @@ func osVersionWindows() string {
 			s += fmt.Sprintf(".%d", ubr)
 		}
 	}
-	if s != "" {
-		winVerCache.Store(s)
-	}
 	return s // "10.0.19041.388", ideally
 }
 

ipn/ipn_clone.go

@@ -48,6 +48,7 @@ var _PrefsCloneNeedsRegeneration = Prefs(struct {
 	Hostname               string
 	NotepadURLs            bool
 	ForceDaemon            bool
+	Egg                    bool
 	AdvertiseRoutes        []netip.Prefix
 	NoSNAT                 bool
 	NetfilterMode          preftype.NetfilterMode

ipn/ipnlocal/local.go

@@ -34,6 +34,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
 	"tailscale.com/net/dns"
+	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsaddr"
@@ -164,6 +165,7 @@ type LocalBackend struct {
 	authURL          string // cleared on Notify
 	authURLSticky    string // not cleared on Notify
 	interact         bool
+	egg              bool
 	prevIfState      *interfaces.State
 	peerAPIServer    *peerAPIServer // or nil
 	peerAPIListeners []*peerAPIListener
@@ -423,7 +425,6 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		s.Version = version.Long
 		s.BackendState = b.state.String()
 		s.AuthURL = b.authURLSticky
-
 		if err := health.OverallError(); err != nil {
 			switch e := err.(type) {
 			case multierr.Error:
@@ -731,6 +732,9 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		b.e.SetNetworkMap(st.NetMap)
 		b.e.SetDERPMap(st.NetMap.DERPMap)
 
+		// Update our cached DERP map
+		dnsfallback.UpdateCache(st.NetMap.DERPMap)
+
 		b.send(ipn.Notify{NetMap: st.NetMap})
 	}
 	if st.URL != "" {
@@ -2015,6 +2019,11 @@ func (b *LocalBackend) isDefaultServerLocked() bool {
 
 func (b *LocalBackend) EditPrefs(mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
 	b.mu.Lock()
+	if mp.EggSet {
+		mp.EggSet = false
+		b.egg = true
+		go b.doSetHostinfoFilterServices(b.hostinfo.Clone())
+	}
 	p0 := b.prefs.Clone()
 	p1 := b.prefs.Clone()
 	p1.ApplyEdits(mp)
@@ -2211,6 +2220,9 @@ func (b *LocalBackend) doSetHostinfoFilterServices(hi *tailcfg.Hostinfo) {
 		return
 	}
 	peerAPIServices := b.peerAPIServicesLocked()
+	if b.egg {
+		peerAPIServices = append(peerAPIServices, tailcfg.Service{Proto: "egg"})
+	}
 	b.mu.Unlock()
 
 	// Make a shallow copy of hostinfo so we can mutate

ipn/ipnserver/server.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
 	"tailscale.com/logtail/backoff"
+	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/netstat"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsdial"
@@ -786,6 +787,8 @@ func New(logf logger.Logf, logid string, store ipn.StateStore, eng wgengine.Engi
 			b.SetTailnetKeyAuthority(authority, storage)
 			logf("tka initialized at head %x", authority.Head())
 		}
+
+		dnsfallback.SetCachePath(filepath.Join(root, "derpmap.cached.json"))
 	} else {
 		logf("network-lock unavailable; no state directory")
 	}

ipn/localapi/localapi.go

@@ -18,7 +18,6 @@ import (
 	"net/http/httputil"
 	"net/netip"
 	"net/url"
-	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -26,6 +25,7 @@ import (
 	"time"
 
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/ipnstate"
@@ -34,6 +34,7 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
 	"tailscale.com/version"
 )
 
@@ -213,6 +214,9 @@ func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	logMarker := fmt.Sprintf("BUG-%v-%v-%v", h.backendLogID, time.Now().UTC().Format("20060102150405Z"), randHex(8))
+	if envknob.NoLogsNoSupport() {
+		logMarker = "BUG-NO-LOGS-NO-SUPPORT-this-node-has-had-its-logging-disabled"
+	}
 	h.logf("user bugreport: %s", logMarker)
 	if note := r.FormValue("note"); len(note) > 0 {
 		h.logf("user bugreport note: %s", note)
@@ -527,7 +531,7 @@ func (h *Handler) serveFileTargets(w http.ResponseWriter, r *http.Request) {
 		writeErrorJSON(w, err)
 		return
 	}
-	makeNonNil(&fts)
+	mak.NonNilSliceForJSON(&fts)
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(fts)
 }
@@ -858,30 +862,3 @@ func defBool(a string, def bool) bool {
 	}
 	return v
 }
-
-// makeNonNil takes a pointer to a Go data structure
-// (currently only a slice or a map) and makes sure it's non-nil for
-// JSON serialization. (In particular, JavaScript clients usually want
-// the field to be defined after they decode the JSON.)
-func makeNonNil(ptr any) {
-	if ptr == nil {
-		panic("nil interface")
-	}
-	rv := reflect.ValueOf(ptr)
-	if rv.Kind() != reflect.Ptr {
-		panic(fmt.Sprintf("kind %v, not Ptr", rv.Kind()))
-	}
-	if rv.Pointer() == 0 {
-		panic("nil pointer")
-	}
-	rv = rv.Elem()
-	if rv.Pointer() != 0 {
-		return
-	}
-	switch rv.Type().Kind() {
-	case reflect.Slice:
-		rv.Set(reflect.MakeSlice(rv.Type(), 0, 0))
-	case reflect.Map:
-		rv.Set(reflect.MakeMap(rv.Type()))
-	}
-}

ipn/prefs.go

@@ -163,6 +163,9 @@ type Prefs struct {
 	// for Linux/etc, which always operate in daemon mode.
 	ForceDaemon bool `json:"ForceDaemon,omitempty"`
 
+	// Egg is a optional debug flag.
+	Egg bool
+
 	// The following block of options only have an effect on Linux.
 
 	// AdvertiseRoutes specifies CIDR prefixes to advertise into the
@@ -217,6 +220,7 @@ type MaskedPrefs struct {
 	HostnameSet               bool `json:",omitempty"`
 	NotepadURLsSet            bool `json:",omitempty"`
 	ForceDaemonSet            bool `json:",omitempty"`
+	EggSet                    bool `json:",omitempty"`
 	AdvertiseRoutesSet        bool `json:",omitempty"`
 	NoSNATSet                 bool `json:",omitempty"`
 	NetfilterModeSet          bool `json:",omitempty"`

ipn/prefs_test.go

@@ -52,6 +52,7 @@ func TestPrefsEqual(t *testing.T) {
 		"Hostname",
 		"NotepadURLs",
 		"ForceDaemon",
+		"Egg",
 		"AdvertiseRoutes",
 		"NoSNAT",
 		"NetfilterMode",

licenses/android.md

@@ -61,8 +61,8 @@ Client][].  See also the dependencies in the [Tailscale CLI][].
  - [golang.org/x/term](https://pkg.go.dev/golang.org/x/term) ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/03fcf44c:LICENSE))
  - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/18b340fc:LICENSE))
  - [golang.org/x/time/rate](https://pkg.go.dev/golang.org/x/time/rate) ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/f0f3c7e8:LICENSE))
- - [golang.zx2c4.com/wireguard](https://pkg.go.dev/golang.zx2c4.com/wireguard) ([MIT](https://git.zx2c4.com/wireguard-go/tree/LICENSE?id=c31a7b1ab478))
- - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/850e42eb4444/LICENSE))
+ - [golang.zx2c4.com/wireguard](https://pkg.go.dev/golang.zx2c4.com/wireguard) ([MIT](https://git.zx2c4.com/wireguard-go/tree/LICENSE?id=b51010ba13f0))
+ - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/846276b3dbc5/LICENSE))
  - [inet.af/netaddr](https://pkg.go.dev/inet.af/netaddr) ([BSD-3-Clause](https://github.com/inetaf/netaddr/blob/097006376321/LICENSE))
  - [nhooyr.io/websocket](https://pkg.go.dev/nhooyr.io/websocket) ([MIT](https://github.com/nhooyr/websocket/blob/v1.8.7/LICENSE.txt))
  - [tailscale.com](https://pkg.go.dev/tailscale.com) ([BSD-3-Clause](https://github.com/tailscale/tailscale/blob/HEAD/LICENSE))

licenses/apple.md

@@ -17,7 +17,7 @@ and [iOS][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/golang/groupcache/lru](https://pkg.go.dev/github.com/golang/groupcache/lru) ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
  - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.0.1/LICENSE))
  - [github.com/hdevalence/ed25519consensus](https://pkg.go.dev/github.com/hdevalence/ed25519consensus) ([BSD-3-Clause](https://github.com/hdevalence/ed25519consensus/blob/c00d1f31bab3/LICENSE))
- - [github.com/insomniacslk/dhcp](https://pkg.go.dev/github.com/insomniacslk/dhcp) ([BSD-3-Clause](https://github.com/insomniacslk/dhcp/blob/7d93572ebe8e/LICENSE))
+ - [github.com/insomniacslk/dhcp](https://pkg.go.dev/github.com/insomniacslk/dhcp) ([BSD-3-Clause](https://github.com/insomniacslk/dhcp/blob/1ca156eafb9f/LICENSE))
  - [github.com/josharian/native](https://pkg.go.dev/github.com/josharian/native) ([MIT](https://github.com/josharian/native/blob/v1.0.0/license))
  - [github.com/jsimonetti/rtnetlink](https://pkg.go.dev/github.com/jsimonetti/rtnetlink) ([MIT](https://github.com/jsimonetti/rtnetlink/blob/d380b505068b/LICENSE.md))
  - [github.com/klauspost/compress/flate](https://pkg.go.dev/github.com/klauspost/compress/flate) ([Apache-2.0](https://github.com/klauspost/compress/blob/v1.15.5/LICENSE))
@@ -44,8 +44,8 @@ and [iOS][].  See also the dependencies in the [Tailscale CLI][].
  - [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/c0bba94a:LICENSE))
  - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/18b340fc:LICENSE))
  - [golang.org/x/time/rate](https://pkg.go.dev/golang.org/x/time/rate) ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/f0f3c7e8:LICENSE))
- - [golang.zx2c4.com/wireguard](https://pkg.go.dev/golang.zx2c4.com/wireguard) ([MIT](https://git.zx2c4.com/wireguard-go/tree/LICENSE?id=c31a7b1ab478))
- - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/850e42eb4444/LICENSE))
+ - [golang.zx2c4.com/wireguard](https://pkg.go.dev/golang.zx2c4.com/wireguard) ([MIT](https://git.zx2c4.com/wireguard-go/tree/LICENSE?id=b51010ba13f0))
+ - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/846276b3dbc5/LICENSE))
  - [nhooyr.io/websocket](https://pkg.go.dev/nhooyr.io/websocket) ([MIT](https://github.com/nhooyr/websocket/blob/v1.8.7/LICENSE.txt))
  - [tailscale.com](https://pkg.go.dev/tailscale.com) ([BSD-3-Clause](https://github.com/tailscale/tailscale/blob/HEAD/LICENSE))
 

licenses/tailscale.md

@@ -78,9 +78,9 @@ Some packages may only be included on certain architectures or operating systems
  - [golang.org/x/text](https://pkg.go.dev/golang.org/x/text) ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/18b340fc:LICENSE))
  - [golang.org/x/time/rate](https://pkg.go.dev/golang.org/x/time/rate) ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/f0f3c7e8:LICENSE))
  - [golang.zx2c4.com/wintun](https://pkg.go.dev/golang.zx2c4.com/wintun) ([MIT](https://git.zx2c4.com/wintun-go/tree/LICENSE?id=415007cec224))
- - [golang.zx2c4.com/wireguard](https://pkg.go.dev/golang.zx2c4.com/wireguard) ([MIT](https://git.zx2c4.com/wireguard-go/tree/LICENSE?id=c31a7b1ab478))
+ - [golang.zx2c4.com/wireguard](https://pkg.go.dev/golang.zx2c4.com/wireguard) ([MIT](https://git.zx2c4.com/wireguard-go/tree/LICENSE?id=b51010ba13f0))
  - [golang.zx2c4.com/wireguard/windows/tunnel/winipcfg](https://pkg.go.dev/golang.zx2c4.com/wireguard/windows/tunnel/winipcfg) ([MIT](https://git.zx2c4.com/wireguard-windows/tree/COPYING?h=v0.5.3))
- - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/850e42eb4444/LICENSE))
+ - [gvisor.dev/gvisor/pkg](https://pkg.go.dev/gvisor.dev/gvisor/pkg) ([Apache-2.0](https://github.com/google/gvisor/blob/846276b3dbc5/LICENSE))
  - [inet.af/peercred](https://pkg.go.dev/inet.af/peercred) ([BSD-3-Clause](https://github.com/inetaf/peercred/blob/0893ea02156a/LICENSE))
  - [inet.af/wf](https://pkg.go.dev/inet.af/wf) ([BSD-3-Clause](https://github.com/inetaf/wf/blob/50d96caab2f6/LICENSE))
  - [nhooyr.io/websocket](https://pkg.go.dev/nhooyr.io/websocket) ([MIT](https://github.com/nhooyr/websocket/blob/v1.8.7/LICENSE.txt))

logpolicy/logpolicy.go

@@ -438,6 +438,13 @@ func tryFixLogStateLocation(dir, cmdname string) {
 // New returns a new log policy (a logger and its instance ID) for a
 // given collection name.
 func New(collection string) *Policy {
+	return NewWithConfigPath(collection, "", "")
+}
+
+// NewWithConfigPath is identical to New,
+// but uses the specified directory and command name.
+// If either is empty, it derives them automatically.
+func NewWithConfigPath(collection, dir, cmdName string) *Policy {
 	var lflags int
 	if term.IsTerminal(2) || runtime.GOOS == "windows" {
 		lflags = 0
@@ -460,9 +467,12 @@ func New(collection string) *Policy {
 		earlyErrBuf.WriteByte('\n')
 	}
 
-	dir := logsDir(earlyLogf)
-
-	cmdName := version.CmdName()
+	if dir == "" {
+		dir = logsDir(earlyLogf)
+	}
+	if cmdName == "" {
+		cmdName = version.CmdName()
+	}
 	tryFixLogStateLocation(dir, cmdName)
 
 	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", cmdName))
@@ -539,7 +549,10 @@ func New(collection string) *Policy {
 		conf.IncludeProcSequence = true
 	}
 
-	if val := getLogTarget(); val != "" {
+	if envknob.NoLogsNoSupport() {
+		log.Println("You have disabled logging. Tailscale will not be able to provide support.")
+		conf.HTTPC = &http.Client{Transport: noopPretendSuccessTransport{}}
+	} else if val := getLogTarget(); val != "" {
 		log.Println("You have enabled a non-default log target. Doing without being told to by Tailscale staff or your network administrator will make getting support difficult.")
 		conf.BaseURL = val
 		u, _ := url.Parse(val)
@@ -735,3 +748,14 @@ func goVersion() string {
 	}
 	return v
 }
+
+type noopPretendSuccessTransport struct{}
+
+func (noopPretendSuccessTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	io.ReadAll(req.Body)
+	req.Body.Close()
+	return &http.Response{
+		StatusCode: 200,
+		Status:     "200 OK",
+	}, nil
+}

net/dns/config.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"sort"
 
+	"tailscale.com/net/dns/publicdns"
 	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/dnstype"
@@ -78,13 +79,14 @@ func (c Config) hasRoutes() bool {
 }
 
 // hasDefaultIPResolversOnly reports whether the only resolvers in c are
-// DefaultResolvers, and that those resolvers are simple IP addresses.
+// DefaultResolvers, and that those resolvers are simple IP addresses
+// that speak regular port 53 DNS.
 func (c Config) hasDefaultIPResolversOnly() bool {
 	if !c.hasDefaultResolvers() || c.hasRoutes() {
 		return false
 	}
 	for _, r := range c.DefaultResolvers {
-		if ipp, ok := r.IPPort(); !ok || ipp.Port() != 53 {
+		if ipp, ok := r.IPPort(); !ok || ipp.Port() != 53 || publicdns.IPIsDoHOnlyServer(ipp.Addr()) {
 			return false
 		}
 	}

net/dns/manager.go

@@ -194,6 +194,7 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 			routes[suffix] = resolvers
 		}
 	}
+
 	// Similarly, the OS always gets search paths.
 	ocfg.SearchDomains = cfg.SearchDomains
 	if runtime.GOOS == "windows" {

net/dns/manager_test.go

@@ -562,6 +562,42 @@ func TestManager(t *testing.T) {
 					"bradfitz.ts.com.", "2.3.4.5"),
 			},
 		},
+		{
+			name: "corp-v6",
+			in: Config{
+				DefaultResolvers: mustRes("1::1"),
+			},
+			os: OSConfig{
+				Nameservers: mustIPs("1::1"),
+			},
+		},
+		{
+			// This one's structurally the same as the previous one (corp-v6), but
+			// instead of 1::1 as the IPv6 address, it uses a NextDNS IPv6 address which
+			// is specially recognized.
+			name: "corp-v6-nextdns",
+			in: Config{
+				DefaultResolvers: mustRes("2a07:a8c0::c3:a884"),
+			},
+			os: OSConfig{
+				Nameservers: mustIPs("100.100.100.100"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(".", "2a07:a8c0::c3:a884"),
+			},
+		},
+		{
+			name: "nextdns-doh",
+			in: Config{
+				DefaultResolvers: mustRes("https://dns.nextdns.io/c3a884"),
+			},
+			os: OSConfig{
+				Nameservers: mustIPs("100.100.100.100"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(".", "https://dns.nextdns.io/c3a884"),
+			},
+		},
 	}
 
 	trIP := cmp.Transformer("ipStr", func(ip netip.Addr) string { return ip.String() })

net/dns/publicdns/publicdns.go

@@ -7,26 +7,109 @@
 package publicdns
 
 import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
 	"net/netip"
+	"sort"
+	"strings"
 	"sync"
+
+	"tailscale.com/util/strs"
 )
 
-var knownDoH = map[netip.Addr]string{} // 8.8.8.8 => "https://..."
+// dohOfIP maps from public DNS IPs to their DoH base URL.
+//
+// This does not include NextDNS which is handled specially.
+var dohOfIP = map[netip.Addr]string{} // 8.8.8.8 => "https://..."
+
 var dohIPsOfBase = map[string][]netip.Addr{}
 var populateOnce sync.Once
 
-// KnownDoH returns a map of well-known public DNS IPs to their DoH URL.
-// The returned map should not be modified.
-func KnownDoH() map[netip.Addr]string {
+// DoHEndpointFromIP returns the DNS-over-HTTPS base URL for a given IP
+// and whether it's DoH-only (not speaking DNS on port 53).
+//
+// The ok result is whether the IP is a known DNS server.
+func DoHEndpointFromIP(ip netip.Addr) (dohBase string, dohOnly bool, ok bool) {
 	populateOnce.Do(populate)
-	return knownDoH
+	if b, ok := dohOfIP[ip]; ok {
+		return b, false, true
+	}
+
+	// NextDNS DoH URLs are of the form "https://dns.nextdns.io/c3a884"
+	// where the path component is the lower 8 bytes of the IPv6 address
+	// in lowercase hex without any zero padding.
+	if nextDNSv6RangeA.Contains(ip) || nextDNSv6RangeB.Contains(ip) {
+		a := ip.As16()
+		var sb strings.Builder
+		const base = "https://dns.nextdns.io/"
+		sb.Grow(len(base) + 8)
+		sb.WriteString(base)
+		for _, b := range bytes.TrimLeft(a[8:], "\x00") {
+			fmt.Fprintf(&sb, "%02x", b)
+		}
+		return sb.String(), true, true
+	}
+	return "", false, false
 }
 
-// DoHIPsOfBase returns a map of DNS server IP addresses keyed
-// by their DoH URL. It is the inverse of KnownDoH.
-func DoHIPsOfBase() map[string][]netip.Addr {
+// KnownDoHPrefixes returns the list of DoH base URLs.
+//
+// It returns a new copy each time, sorted. It's meant for tests.
+//
+// It does not include providers that have customer-specific DoH URLs like
+// NextDNS.
+func KnownDoHPrefixes() []string {
 	populateOnce.Do(populate)
-	return dohIPsOfBase
+	ret := make([]string, 0, len(dohIPsOfBase))
+	for b := range dohIPsOfBase {
+		ret = append(ret, b)
+	}
+	sort.Strings(ret)
+	return ret
+}
+
+func isSlashOrQuestionMark(r rune) bool {
+	return r == '/' || r == '?'
+}
+
+// DoHIPsOfBase returns the IP addresses to use to dial the provided DoH base
+// URL.
+//
+// It is basically the inverse of DoHEndpointFromIP with the exception that for
+// NextDNS it returns IPv4 addresses that DoHEndpointFromIP doesn't map back.
+func DoHIPsOfBase(dohBase string) []netip.Addr {
+	populateOnce.Do(populate)
+	if s := dohIPsOfBase[dohBase]; len(s) > 0 {
+		return s
+	}
+	if hexStr, ok := strs.CutPrefix(dohBase, "https://dns.nextdns.io/"); ok {
+		// The path is of the form /<profile-hex>[/<hostname>/<model>/<device id>...]
+		// or /<profile-hex>?<query params>
+		// but only the <profile-hex> is required. Ignore the rest:
+		if i := strings.IndexFunc(hexStr, isSlashOrQuestionMark); i != -1 {
+			hexStr = hexStr[:i]
+		}
+
+		// TODO(bradfitz): using the NextDNS anycast addresses works but is not
+		// ideal. Some of their regions have better latency via a non-anycast IP
+		// which we could get by first resolving A/AAAA "dns.nextdns.io" over
+		// DoH using their anycast address. For now we only use the anycast
+		// addresses. The IPv4 IPs we use are just the first one in their ranges.
+		// For IPv6 we put the profile ID in the lower bytes, but that seems just
+		// conventional for them and not required (it'll already be in the DoH path).
+		// (Really we shouldn't use either IPv4 or IPv6 anycast for DoH once we
+		// resolve "dns.nextdns.io".)
+		if b, err := hex.DecodeString(hexStr); err == nil && len(b) <= 8 && len(b) > 0 {
+			return []netip.Addr{
+				nextDNSv4One,
+				nextDNSv4Two,
+				nextDNSv6Gen(nextDNSv6RangeA.Addr(), b),
+				nextDNSv6Gen(nextDNSv6RangeB.Addr(), b),
+			}
+		}
+	}
+	return nil
 }
 
 // DoHV6 returns the first IPv6 DNS address from a given public DNS provider
@@ -45,7 +128,7 @@ func DoHV6(base string) (ip netip.Addr, ok bool) {
 // adds it to both knownDoH and dohIPsOFBase maps.
 func addDoH(ipStr, base string) {
 	ip := netip.MustParseAddr(ipStr)
-	knownDoH[ip] = base
+	dohOfIP[ip] = base
 	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
 }
 
@@ -106,3 +189,43 @@ func populate() {
 	addDoH("193.19.108.3", "https://adblock.doh.mullvad.net/dns-query")
 	addDoH("2a07:e340::3", "https://adblock.doh.mullvad.net/dns-query")
 }
+
+var (
+	// The NextDNS IPv6 ranges (primary and secondary). The customer ID is
+	// encoded in the lower bytes and is used (in hex form) as the DoH query
+	// path.
+	nextDNSv6RangeA = netip.MustParsePrefix("2a07:a8c0::/33")
+	nextDNSv6RangeB = netip.MustParsePrefix("2a07:a8c1::/33")
+
+	// The first two IPs in the /24 v4 ranges can be used for DoH to NextDNS.
+	//
+	// They're Anycast and usually okay, but NextDNS has some locations that
+	// don't do BGP and can get results for querying them over DoH to find the
+	// IPv4 address of "dns.mynextdns.io" and find an even better result.
+	//
+	// Note that the Tailscale DNS client does not do any of the "IP address
+	// linking" that NextDNS can do with its IPv4 addresses. These addresses
+	// are only used for DoH.
+	nextDNSv4RangeA = netip.MustParsePrefix("45.90.28.0/24")
+	nextDNSv4RangeB = netip.MustParsePrefix("45.90.30.0/24")
+	nextDNSv4One    = nextDNSv4RangeA.Addr()
+	nextDNSv4Two    = nextDNSv4RangeB.Addr()
+)
+
+// nextDNSv6Gen generates a NextDNS IPv6 address from the upper 8 bytes in the
+// provided ip and using id as the lowest 0-8 bytes.
+func nextDNSv6Gen(ip netip.Addr, id []byte) netip.Addr {
+	if len(id) > 8 {
+		return netip.Addr{}
+	}
+	a := ip.As16()
+	copy(a[16-len(id):], id)
+	return netip.AddrFrom16(a)
+}
+
+// IPIsDoHOnlyServer reports whether ip is a DNS server that should only use
+// DNS-over-HTTPS (not regular port 53 DNS).
+func IPIsDoHOnlyServer(ip netip.Addr) bool {
+	return nextDNSv6RangeA.Contains(ip) || nextDNSv6RangeB.Contains(ip) ||
+		nextDNSv4RangeA.Contains(ip) || nextDNSv4RangeB.Contains(ip)
+}

net/dns/publicdns/publicdns_test.go

@@ -6,20 +6,30 @@ package publicdns
 
 import (
 	"net/netip"
+	"reflect"
 	"testing"
 )
 
 func TestInit(t *testing.T) {
-	for baseKey, baseSet := range DoHIPsOfBase() {
+	for _, baseKey := range KnownDoHPrefixes() {
+		baseSet := DoHIPsOfBase(baseKey)
 		for _, addr := range baseSet {
-			if KnownDoH()[addr] != baseKey {
-				t.Errorf("Expected %v to map to %s, got %s", addr, baseKey, KnownDoH()[addr])
+			back, only, ok := DoHEndpointFromIP(addr)
+			if !ok {
+				t.Errorf("DoHEndpointFromIP(%v) not mapped back to %v", addr, baseKey)
+				continue
+			}
+			if only {
+				t.Errorf("unexpected DoH only bit set for %v", addr)
+			}
+			if back != baseKey {
+				t.Errorf("Expected %v to map to %s, got %s", addr, baseKey, back)
 			}
 		}
 	}
 }
 
-func TestDohV6(t *testing.T) {
+func TestDoHV6(t *testing.T) {
 	tests := []struct {
 		in      string
 		firstIP netip.Addr
@@ -38,3 +48,67 @@ func TestDohV6(t *testing.T) {
 		})
 	}
 }
+
+func TestDoHIPsOfBase(t *testing.T) {
+	ips := func(s ...string) (ret []netip.Addr) {
+		for _, ip := range s {
+			ret = append(ret, netip.MustParseAddr(ip))
+		}
+		return
+	}
+	tests := []struct {
+		base string
+		want []netip.Addr
+	}{
+		{
+			base: "https://cloudflare-dns.com/dns-query",
+			want: ips("1.1.1.1", "1.0.0.1", "2606:4700:4700::1111", "2606:4700:4700::1001"),
+		},
+		{
+			base: "https://dns.nextdns.io/",
+			want: ips(),
+		},
+		{
+			base: "https://dns.nextdns.io/ff",
+			want: ips(
+				"45.90.28.0",
+				"45.90.30.0",
+				"2a07:a8c0::ff",
+				"2a07:a8c1::ff",
+			),
+		},
+		{
+			base: "https://dns.nextdns.io/c3a884",
+			want: ips(
+				"45.90.28.0",
+				"45.90.30.0",
+				"2a07:a8c0::c3:a884",
+				"2a07:a8c1::c3:a884",
+			),
+		},
+		{
+			base: "https://dns.nextdns.io/c3a884/with/more/stuff",
+			want: ips(
+				"45.90.28.0",
+				"45.90.30.0",
+				"2a07:a8c0::c3:a884",
+				"2a07:a8c1::c3:a884",
+			),
+		},
+		{
+			base: "https://dns.nextdns.io/c3a884?with=query&params",
+			want: ips(
+				"45.90.28.0",
+				"45.90.30.0",
+				"2a07:a8c0::c3:a884",
+				"2a07:a8c1::c3:a884",
+			),
+		},
+	}
+	for _, tt := range tests {
+		got := DoHIPsOfBase(tt.base)
+		if !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("DoHIPsOfBase(%q) = %v; want %v", tt.base, got, tt.want)
+		}
+	}
+}

net/dns/resolved.go

@@ -205,7 +205,7 @@ func (m *resolvedManager) run(ctx context.Context) {
 			// When ctx goes away systemd-resolved auto reverts.
 			// Keeping for potential use in future refactor.
 			if call := rManager.CallWithContext(ctx, dbusResolvedInterface+".RevertLink", 0, m.ifidx); call.Err != nil {
-				m.logf("[v1] RevertLink: %w", call.Err)
+				m.logf("[v1] RevertLink: %v", call.Err)
 				return
 			}
 			return

net/dns/resolver/doh_test.go

@@ -41,7 +41,8 @@ func TestDoH(t *testing.T) {
 	if !*testDoH {
 		t.Skip("skipping manual test without --test-doh flag")
 	}
-	if len(publicdns.KnownDoH()) == 0 {
+	prefixes := publicdns.KnownDoHPrefixes()
+	if len(prefixes) == 0 {
 		t.Fatal("no known DoH")
 	}
 
@@ -49,7 +50,7 @@ func TestDoH(t *testing.T) {
 		dohSem: make(chan struct{}, 10),
 	}
 
-	for urlBase := range publicdns.DoHIPsOfBase() {
+	for _, urlBase := range prefixes {
 		t.Run(urlBase, func(t *testing.T) {
 			c, ok := f.getKnownDoHClientForProvider(urlBase)
 			if !ok {
@@ -86,13 +87,15 @@ func TestDoH(t *testing.T) {
 }
 
 func TestDoHV6Fallback(t *testing.T) {
-	for ip, base := range publicdns.KnownDoH() {
-		if ip.Is4() {
-			ip6, ok := publicdns.DoHV6(base)
-			if !ok {
-				t.Errorf("no v6 DoH known for %v", ip)
-			} else if !ip6.Is6() {
-				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+	for _, base := range publicdns.KnownDoHPrefixes() {
+		for _, ip := range publicdns.DoHIPsOfBase(base) {
+			if ip.Is4() {
+				ip6, ok := publicdns.DoHV6(base)
+				if !ok {
+					t.Errorf("no v6 DoH known for %v", ip)
+				} else if !ip6.Is6() {
+					t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+				}
 			}
 		}
 	}

net/dns/resolver/forwarder.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/types/nettype"
 	"tailscale.com/util/cloudenv"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/version"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -57,9 +58,6 @@ func truncatedFlagSet(pkt []byte) bool {
 }
 
 const (
-	// responseTimeout is the maximal amount of time to wait for a DNS response.
-	responseTimeout = 5 * time.Second
-
 	// dohTransportTimeout is how long to keep idle HTTP
 	// connections open to DNS-over-HTTPs servers. This is pretty
 	// arbitrary.
@@ -259,18 +257,26 @@ func (f *forwarder) Close() error {
 func resolversWithDelays(resolvers []*dnstype.Resolver) []resolverAndDelay {
 	rr := make([]resolverAndDelay, 0, len(resolvers)+2)
 
+	type dohState uint8
+	const addedDoH = dohState(1)
+	const addedDoHAndDontAddUDP = dohState(2)
+
 	// Add the known DoH ones first, starting immediately.
-	didDoH := map[string]bool{}
+	didDoH := map[string]dohState{}
 	for _, r := range resolvers {
 		ipp, ok := r.IPPort()
 		if !ok {
 			continue
 		}
-		dohBase, ok := publicdns.KnownDoH()[ipp.Addr()]
-		if !ok || didDoH[dohBase] {
+		dohBase, dohOnly, ok := publicdns.DoHEndpointFromIP(ipp.Addr())
+		if !ok || didDoH[dohBase] != 0 {
 			continue
 		}
-		didDoH[dohBase] = true
+		if dohOnly {
+			didDoH[dohBase] = addedDoHAndDontAddUDP
+		} else {
+			didDoH[dohBase] = addedDoH
+		}
 		rr = append(rr, resolverAndDelay{name: &dnstype.Resolver{Addr: dohBase}})
 	}
 
@@ -289,8 +295,12 @@ func resolversWithDelays(resolvers []*dnstype.Resolver) []resolverAndDelay {
 		}
 		ip := ipp.Addr()
 		var startDelay time.Duration
-		if host, ok := publicdns.KnownDoH()[ip]; ok {
+		if host, _, ok := publicdns.DoHEndpointFromIP(ip); ok {
+			if didDoH[host] == addedDoHAndDontAddUDP {
+				continue
+			}
 			// We already did the DoH query early. These
+			// are for normal dns53 UDP queries.
 			startDelay = dohHeadStart
 			key := hostAndFam{host, uint8(ip.BitLen())}
 			if done[key] > 0 {
@@ -391,7 +401,7 @@ func (f *forwarder) getKnownDoHClientForProvider(urlBase string) (c *http.Client
 	if c, ok := f.dohClient[urlBase]; ok {
 		return c, true
 	}
-	allIPs := publicdns.DoHIPsOfBase()[urlBase]
+	allIPs := publicdns.DoHIPsOfBase(urlBase)
 	if len(allIPs) == 0 {
 		return nil, false
 	}
@@ -407,7 +417,7 @@ func (f *forwarder) getKnownDoHClientForProvider(urlBase string) (c *http.Client
 	c = &http.Client{
 		Transport: &http.Transport{
 			ForceAttemptHTTP2: true,
-			IdleConnTimeout: dohTransportTimeout,
+			IdleConnTimeout:   dohTransportTimeout,
 			DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) {
 				if !strings.HasPrefix(netw, "tcp") {
 					return nil, fmt.Errorf("unexpected network %q", netw)
@@ -447,11 +457,8 @@ func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client,
 		return nil, err
 	}
 	req.Header.Set("Content-Type", dohType)
-	// Note: we don't currently set the Accept header (which is
-	// only a SHOULD in the spec) as iOS doesn't use HTTP/2 and
-	// we'd rather save a few bytes on outgoing requests when
-	// empirically no provider cares about the Accept header's
-	// absence.
+	req.Header.Set("Accept", dohType)
+	req.Header.Set("User-Agent", "tailscaled/"+version.Long)
 
 	hres, err := c.Do(req)
 	if err != nil {

net/dns/resolver/forwarder_test.go

@@ -79,6 +79,16 @@ func TestResolversWithDelays(t *testing.T) {
 			in:   q("9.9.9.9", "2620:fe::fe"),
 			want: o("https://dns.quad9.net/dns-query", "9.9.9.9+0.5s", "2620:fe::fe+0.5s"),
 		},
+		{
+			name: "nextdns-ipv6-expand",
+			in:   q("2a07:a8c0::c3:a884"),
+			want: o("https://dns.nextdns.io/c3a884"),
+		},
+		{
+			name: "nextdns-doh-input",
+			in:   q("https://dns.nextdns.io/c3a884"),
+			want: o("https://dns.nextdns.io/c3a884"),
+		},
 	}
 
 	for _, tt := range tests {

net/dnscache/dnscache.go

@@ -25,6 +25,8 @@ import (
 	"tailscale.com/util/singleflight"
 )
 
+var zaddr netip.Addr
+
 var single = &Resolver{
 	Forward: &net.Resolver{PreferGo: preferGoResolver()},
 }
@@ -90,14 +92,14 @@ type Resolver struct {
 
 // ipRes is the type used by the Resolver.sf singleflight group.
 type ipRes struct {
-	ip, ip6 net.IP
-	allIPs  []net.IPAddr
+	ip, ip6 netip.Addr
+	allIPs  []netip.Addr
 }
 
 type ipCacheEntry struct {
-	ip      net.IP       // either v4 or v6
-	ip6     net.IP       // nil if no v4 or no v6
-	allIPs  []net.IPAddr // 1+ v4 and/or v6
+	ip      netip.Addr   // either v4 or v6
+	ip6     netip.Addr   // nil if no v4 or no v6
+	allIPs  []netip.Addr // 1+ v4 and/or v6
 	expires time.Time
 }
 
@@ -147,34 +149,28 @@ var debug = envknob.Bool("TS_DEBUG_DNS_CACHE")
 //
 // If err is nil, ip will be non-nil. The v6 address may be nil even
 // with a nil error.
-func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, allIPs []net.IPAddr, err error) {
+func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 netip.Addr, allIPs []netip.Addr, err error) {
 	if r.SingleHostStaticResult != nil {
 		if r.SingleHost != host {
-			return nil, nil, nil, fmt.Errorf("dnscache: unexpected hostname %q doesn't match expected %q", host, r.SingleHost)
+			return zaddr, zaddr, nil, fmt.Errorf("dnscache: unexpected hostname %q doesn't match expected %q", host, r.SingleHost)
 		}
 		for _, naIP := range r.SingleHostStaticResult {
-			ipa := &net.IPAddr{
-				IP:   naIP.AsSlice(),
-				Zone: naIP.Zone(),
+			if !ip.IsValid() && naIP.Is4() {
+				ip = naIP
 			}
-			if ip == nil && naIP.Is4() {
-				ip = ipa.IP
+			if !v6.IsValid() && naIP.Is6() {
+				v6 = naIP
 			}
-			if v6 == nil && naIP.Is6() {
-				v6 = ipa.IP
-			}
-			allIPs = append(allIPs, *ipa)
+			allIPs = append(allIPs, naIP)
 		}
 		return
 	}
-	if ip := net.ParseIP(host); ip != nil {
-		if ip4 := ip.To4(); ip4 != nil {
-			return ip4, nil, []net.IPAddr{{IP: ip4}}, nil
-		}
+	if ip, err := netip.ParseAddr(host); err == nil {
+		ip = ip.Unmap()
 		if debug {
 			log.Printf("dnscache: %q is an IP", host)
 		}
-		return ip, nil, []net.IPAddr{{IP: ip}}, nil
+		return ip, zaddr, []netip.Addr{ip}, nil
 	}
 
 	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
@@ -205,7 +201,7 @@ func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, al
 			if debug {
 				log.Printf("dnscache: error resolving %q: %v", host, res.Err)
 			}
-			return nil, nil, nil, res.Err
+			return zaddr, zaddr, nil, res.Err
 		}
 		r := res.Val
 		return r.ip, r.ip6, r.allIPs, nil
@@ -213,26 +209,26 @@ func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, al
 		if debug {
 			log.Printf("dnscache: context done while resolving %q: %v", host, ctx.Err())
 		}
-		return nil, nil, nil, ctx.Err()
+		return zaddr, zaddr, nil, ctx.Err()
 	}
 }
 
-func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
+func (r *Resolver) lookupIPCache(host string) (ip, ip6 netip.Addr, allIPs []netip.Addr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok && ent.expires.After(time.Now()) {
 		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, nil, false
+	return zaddr, zaddr, nil, false
 }
 
-func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
+func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 netip.Addr, allIPs []netip.Addr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok {
 		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, nil, false
+	return zaddr, zaddr, nil, false
 }
 
 func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
@@ -252,7 +248,7 @@ func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	return 10 * time.Second
 }
 
-func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, err error) {
+func (r *Resolver) lookupIP(host string) (ip, ip6 netip.Addr, allIPs []netip.Addr, err error) {
 	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q found in cache as %v", host, ip)
@@ -262,47 +258,37 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, e
 
 	ctx, cancel := context.WithTimeout(context.Background(), r.lookupTimeoutForHost(host))
 	defer cancel()
-	ips, err := r.fwd().LookupIPAddr(ctx, host)
+	ips, err := r.fwd().LookupNetIP(ctx, "ip", host)
 	if err != nil || len(ips) == 0 {
 		if resolver, ok := r.cloudHostResolver(); ok {
-			ips, err = resolver.LookupIPAddr(ctx, host)
+			ips, err = resolver.LookupNetIP(ctx, "ip", host)
 		}
 	}
 	if (err != nil || len(ips) == 0) && r.LookupIPFallback != nil {
 		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
-		var fips []netip.Addr
-		fips, err = r.LookupIPFallback(ctx, host)
-		if err == nil {
-			ips = nil
-			for _, fip := range fips {
-				ips = append(ips, net.IPAddr{
-					IP:   fip.AsSlice(),
-					Zone: fip.Zone(),
-				})
-			}
-		}
+		ips, err = r.LookupIPFallback(ctx, host)
 	}
 	if err != nil {
-		return nil, nil, nil, err
+		return netip.Addr{}, netip.Addr{}, nil, err
 	}
 	if len(ips) == 0 {
-		return nil, nil, nil, fmt.Errorf("no IPs for %q found", host)
+		return netip.Addr{}, netip.Addr{}, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
 	have4 := false
 	for _, ipa := range ips {
-		if ip4 := ipa.IP.To4(); ip4 != nil {
+		if ipa.Is4() {
 			if !have4 {
 				ip6 = ip
-				ip = ip4
+				ip = ipa
 				have4 = true
 			}
 		} else {
 			if have4 {
-				ip6 = ipa.IP
+				ip6 = ipa
 			} else {
-				ip = ipa.IP
+				ip = ipa
 			}
 		}
 	}
@@ -310,7 +296,7 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, e
 	return ip, ip6, ips, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr, d time.Duration) {
+func (r *Resolver) addIPCache(host string, ip, ip6 netip.Addr, allIPs []netip.Addr, d time.Duration) {
 	if ip.IsPrivate() {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
@@ -375,7 +361,7 @@ func (d *dialer) DialContext(ctx context.Context, network, address string) (retC
 	defer func() {
 		// On failure, consider that our DNS might be wrong and ask the DNS fallback mechanism for
 		// some other IPs to try.
-		if ret == nil || ctx.Err() != nil || d.dnsCache.LookupIPFallback == nil || dc.dnsWasTrustworthy() {
+		if !d.shouldTryBootstrap(ctx, ret, dc) {
 			return
 		}
 		ips, err := d.dnsCache.LookupIPFallback(ctx, host)
@@ -399,20 +385,12 @@ func (d *dialer) DialContext(ctx context.Context, network, address string) (retC
 		if debug {
 			log.Printf("dnscache: dialing %s, %s for %s", network, ip, address)
 		}
-		ipNA, ok := netip.AddrFromSlice(ip)
-		if !ok {
-			return nil, fmt.Errorf("invalid IP %q", ip)
-		}
-		c, err := dc.dialOne(ctx, ipNA.Unmap())
+		c, err := dc.dialOne(ctx, ip.Unmap())
 		if err == nil || ctx.Err() != nil {
 			return c, err
 		}
 		// Fall back to trying IPv6, if any.
-		ip6NA, ok := netip.AddrFromSlice(ip6)
-		if !ok {
-			return nil, err
-		}
-		return dc.dialOne(ctx, ip6NA)
+		return dc.dialOne(ctx, ip6)
 	}
 
 	// Multiple IPv4 candidates, and 0+ IPv6.
@@ -420,6 +398,40 @@ func (d *dialer) DialContext(ctx context.Context, network, address string) (retC
 	return dc.raceDial(ctx, ipsToTry)
 }
 
+func (d *dialer) shouldTryBootstrap(ctx context.Context, err error, dc *dialCall) bool {
+	// No need to do anything when we succeeded.
+	if err == nil {
+		return false
+	}
+
+	// Can't try bootstrap DNS if we don't have a fallback function
+	if d.dnsCache.LookupIPFallback == nil {
+		if debug {
+			log.Printf("dnscache: not using bootstrap DNS: no fallback")
+		}
+		return false
+	}
+
+	// We can't retry if the context is canceled, since any further
+	// operations with this context will fail.
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		if debug {
+			log.Printf("dnscache: not using bootstrap DNS: context error: %v", ctxErr)
+		}
+		return false
+	}
+
+	wasTrustworthy := dc.dnsWasTrustworthy()
+	if wasTrustworthy {
+		if debug {
+			log.Printf("dnscache: not using bootstrap DNS: DNS was trustworthy")
+		}
+		return false
+	}
+
+	return true
+}
+
 // dialCall is the state around a single call to dial.
 type dialCall struct {
 	d                            *dialer
@@ -610,21 +622,20 @@ func interleaveSlices[T any](a, b []T) []T {
 	return ret
 }
 
-func v4addrs(aa []net.IPAddr) (ret []netip.Addr) {
+func v4addrs(aa []netip.Addr) (ret []netip.Addr) {
 	for _, a := range aa {
-		ip, ok := netip.AddrFromSlice(a.IP)
-		ip = ip.Unmap()
-		if ok && ip.Is4() {
-			ret = append(ret, ip)
+		a = a.Unmap()
+		if a.Is4() {
+			ret = append(ret, a)
 		}
 	}
 	return ret
 }
 
-func v6addrs(aa []net.IPAddr) (ret []netip.Addr) {
+func v6addrs(aa []netip.Addr) (ret []netip.Addr) {
 	for _, a := range aa {
-		if ip, ok := netip.AddrFromSlice(a.IP); ok && ip.Is6() {
-			ret = append(ret, ip)
+		if a.Is6() && !a.Is4In6() {
+			ret = append(ret, a)
 		}
 	}
 	return ret

net/dnscache/dnscache_test.go

@@ -131,7 +131,7 @@ func TestResolverAllHostStaticResult(t *testing.T) {
 	if got, want := ip6.String(), "2001:4860:4860::8888"; got != want {
 		t.Errorf("ip4 got %q; want %q", got, want)
 	}
-	if got, want := fmt.Sprintf("%q", allIPs), `[{"2001:4860:4860::8888" ""} {"2001:4860:4860::8844" ""} {"8.8.8.8" ""} {"8.8.4.4" ""}]`; got != want {
+	if got, want := fmt.Sprintf("%q", allIPs), `["2001:4860:4860::8888" "2001:4860:4860::8844" "8.8.8.8" "8.8.4.4"]`; got != want {
 		t.Errorf("allIPs got %q; want %q", got, want)
 	}
 
@@ -164,3 +164,104 @@ func TestInterleaveSlices(t *testing.T) {
 		})
 	}
 }
+
+func TestShouldTryBootstrap(t *testing.T) {
+	oldDebug := debug
+	t.Cleanup(func() {
+		debug = oldDebug
+	})
+	debug = true
+
+	type step struct {
+		ip  netip.Addr // IP we pretended to dial
+		err error      // the dial error or nil for success
+	}
+
+	canceled, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	deadlineExceeded, cancel := context.WithTimeout(context.Background(), 0)
+	defer cancel()
+
+	ctx := context.Background()
+	errFailed := errors.New("some failure")
+
+	cacheWithFallback := &Resolver{
+		LookupIPFallback: func(_ context.Context, _ string) ([]netip.Addr, error) {
+			panic("unimplemented")
+		},
+	}
+	cacheNoFallback := &Resolver{}
+
+	testCases := []struct {
+		name       string
+		steps      []step
+		ctx        context.Context
+		err        error
+		noFallback bool
+		want       bool
+	}{
+		{
+			name: "no-error",
+			ctx:  ctx,
+			err:  nil,
+			want: false,
+		},
+		{
+			name: "canceled",
+			ctx:  canceled,
+			err:  errFailed,
+			want: false,
+		},
+		{
+			name: "deadline-exceeded",
+			ctx:  deadlineExceeded,
+			err:  errFailed,
+			want: false,
+		},
+		{
+			name:       "no-fallback",
+			ctx:        ctx,
+			err:        errFailed,
+			noFallback: true,
+			want:       false,
+		},
+		{
+			name: "dns-was-trustworthy",
+			ctx:  ctx,
+			err:  errFailed,
+			steps: []step{
+				{netip.MustParseAddr("2003::1"), nil},
+				{netip.MustParseAddr("2003::1"), errFailed},
+			},
+			want: false,
+		},
+		{
+			name: "should-bootstrap",
+			ctx:  ctx,
+			err:  errFailed,
+			want: true,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			d := &dialer{
+				pastConnect: map[netip.Addr]time.Time{},
+			}
+			if tt.noFallback {
+				d.dnsCache = cacheNoFallback
+			} else {
+				d.dnsCache = cacheWithFallback
+			}
+			dc := &dialCall{d: d}
+			for _, st := range tt.steps {
+				dc.noteDialResult(st.ip, st.err)
+			}
+			got := d.shouldTryBootstrap(tt.ctx, tt.err, dc)
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

net/dnsfallback/dnsfallback.go

@@ -20,12 +20,18 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
+	"os"
+	"reflect"
+	"sync/atomic"
 	"time"
 
+	"tailscale.com/atomicfile"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
 	"tailscale.com/net/tshttpproxy"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 )
 
 func Lookup(ctx context.Context, host string) ([]netip.Addr, error) {
@@ -39,6 +45,7 @@ func Lookup(ctx context.Context, host string) ([]netip.Addr, error) {
 	}
 
 	dm := getDERPMap()
+
 	var cands4, cands6 []nameIP
 	for _, dr := range dm.Regions {
 		for _, n := range dr.Nodes {
@@ -72,16 +79,16 @@ func Lookup(ctx context.Context, host string) ([]netip.Addr, error) {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
-		log.Printf("trying bootstrapDNS(%q, %q) for %q ...", cand.dnsName, cand.ip, host)
+		logf("trying bootstrapDNS(%q, %q) for %q ...", cand.dnsName, cand.ip, host)
 		ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
 		defer cancel()
 		dm, err := bootstrapDNSMap(ctx, cand.dnsName, cand.ip, host)
 		if err != nil {
-			log.Printf("bootstrapDNS(%q, %q) for %q error: %v", cand.dnsName, cand.ip, host, err)
+			logf("bootstrapDNS(%q, %q) for %q error: %v", cand.dnsName, cand.ip, host, err)
 			continue
 		}
 		if ips := dm[host]; len(ips) > 0 {
-			log.Printf("bootstrapDNS(%q, %q) for %q = %v", cand.dnsName, cand.ip, host, ips)
+			logf("bootstrapDNS(%q, %q) for %q = %v", cand.dnsName, cand.ip, host, ips)
 			return ips, nil
 		}
 	}
@@ -94,7 +101,7 @@ func Lookup(ctx context.Context, host string) ([]netip.Addr, error) {
 // serverName and serverIP of are, say, "derpN.tailscale.com".
 // queryName is the name being sought (e.g. "controlplane.tailscale.com"), passed as hint.
 func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netip.Addr, queryName string) (dnsMap, error) {
-	dialer := netns.NewDialer(log.Printf)
+	dialer := netns.NewDialer(logf)
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.Proxy = tshttpproxy.ProxyFromEnvironment
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {
@@ -128,12 +135,45 @@ type dnsMap map[string][]netip.Addr
 // getDERPMap returns some DERP map. The DERP servers also run a fallback
 // DNS server.
 func getDERPMap() *tailcfg.DERPMap {
-	// TODO(bradfitz): try to read the last known DERP map from disk,
-	// at say /var/lib/tailscale/derpmap.txt and write it when it changes,
-	// and read it here.
-	// But ultimately the fallback will be to use a copy baked into the binary,
-	// which is this part:
+	dm := getStaticDERPMap()
 
+	// Merge in any DERP servers from the cached map that aren't in the
+	// static map; this ensures that we're getting new region(s) while not
+	// overriding the built-in fallbacks if things go horribly wrong and we
+	// get a bad DERP map.
+	//
+	// TODO(andrew): should we expect OmitDefaultRegions here? We're not
+	// forwarding traffic, just resolving DNS, so maybe we can ignore that
+	// value anyway?
+	cached := cachedDERPMap.Load()
+	if cached == nil {
+		return dm
+	}
+
+	for id, region := range cached.Regions {
+		dr, ok := dm.Regions[id]
+		if !ok {
+			dm.Regions[id] = region
+			continue
+		}
+
+		// Add any nodes that we don't already have.
+		seen := make(map[string]bool)
+		for _, n := range dr.Nodes {
+			seen[n.HostName] = true
+		}
+		for _, n := range region.Nodes {
+			if !seen[n.HostName] {
+				dr.Nodes = append(dr.Nodes, n)
+			}
+		}
+	}
+
+	return dm
+}
+
+// getStaticDERPMap returns the DERP map that was compiled into this binary.
+func getStaticDERPMap() *tailcfg.DERPMap {
 	dm := new(tailcfg.DERPMap)
 	if err := json.Unmarshal(staticDERPMapJSON, dm); err != nil {
 		panic(err)
@@ -143,3 +183,83 @@ func getDERPMap() *tailcfg.DERPMap {
 
 //go:embed dns-fallback-servers.json
 var staticDERPMapJSON []byte
+
+// cachedDERPMap is the path to a cached DERP map that we loaded from our on-disk cache.
+var cachedDERPMap atomic.Pointer[tailcfg.DERPMap]
+
+// cachePath is the path to the DERP map cache file, set by SetCachePath via
+// ipnserver.New() if we have a state directory.
+var cachePath string
+
+// UpdateCache stores the DERP map cache back to disk.
+//
+// The caller must not mutate 'c' after calling this function.
+func UpdateCache(c *tailcfg.DERPMap) {
+	// Don't do anything if nothing changed.
+	curr := cachedDERPMap.Load()
+	if reflect.DeepEqual(curr, c) {
+		return
+	}
+
+	d, err := json.Marshal(c)
+	if err != nil {
+		logf("[v1] dnsfallback: UpdateCache error marshaling: %v", err)
+		return
+	}
+
+	// Only store after we're confident this is at least valid JSON
+	cachedDERPMap.Store(c)
+
+	// Don't try writing if we don't have a cache path set; this can happen
+	// when we don't have a state path (e.g. /var/lib/tailscale) configured.
+	if cachePath != "" {
+		err = atomicfile.WriteFile(cachePath, d, 0600)
+		if err != nil {
+			logf("[v1] dnsfallback: UpdateCache error writing: %v", err)
+			return
+		}
+	}
+}
+
+// SetCachePath sets the path to the on-disk DERP map cache that we store and
+// update. Additionally, if a file at this path exists, we load it and merge it
+// with the DERP map baked into the binary.
+//
+// This function should be called before any calls to UpdateCache, as it is not
+// concurrency-safe.
+func SetCachePath(path string) {
+	cachePath = path
+
+	f, err := os.Open(path)
+	if err != nil {
+		logf("[v1] dnsfallback: SetCachePath error reading %q: %v", path, err)
+		return
+	}
+	defer f.Close()
+
+	dm := new(tailcfg.DERPMap)
+	if err := json.NewDecoder(f).Decode(dm); err != nil {
+		logf("[v1] dnsfallback: SetCachePath error decoding %q: %v", path, err)
+		return
+	}
+
+	cachedDERPMap.Store(dm)
+	logf("[v2] dnsfallback: SetCachePath loaded cached DERP map")
+}
+
+// logfunc stores the logging function to use for this package.
+var logfunc syncs.AtomicValue[logger.Logf]
+
+// SetLogger sets the logging function that this package will use. The default
+// logger if this function is not called is 'log.Printf'.
+func SetLogger(log logger.Logf) {
+	logfunc.Store(log)
+}
+
+func logf(format string, args ...any) {
+	if lf := logfunc.Load(); lf != nil {
+		lf(format, args...)
+	} else {
+		log.Printf(format, args...)
+	}
+}

net/dnsfallback/dnsfallback_test.go

@@ -4,7 +4,15 @@
 
 package dnsfallback
 
-import "testing"
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"tailscale.com/tailcfg"
+)
 
 func TestGetDERPMap(t *testing.T) {
 	dm := getDERPMap()
@@ -15,3 +23,161 @@ func TestGetDERPMap(t *testing.T) {
 		t.Fatal("no regions")
 	}
 }
+
+func TestCache(t *testing.T) {
+	oldlog := logfunc.Load()
+	SetLogger(t.Logf)
+	t.Cleanup(func() {
+		SetLogger(oldlog)
+	})
+	cacheFile := filepath.Join(t.TempDir(), "cache.json")
+
+	// Write initial cache value
+	initialCache := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			99: {
+				RegionID:   99,
+				RegionCode: "test",
+				RegionName: "Testville",
+				Nodes: []*tailcfg.DERPNode{{
+					Name:     "99a",
+					RegionID: 99,
+					HostName: "derp99a.tailscale.com",
+					IPv4:     "1.2.3.4",
+				}},
+			},
+
+			// Intentionally attempt to "overwrite" something
+			1: {
+				RegionID:   1,
+				RegionCode: "r1",
+				RegionName: "r1",
+				Nodes: []*tailcfg.DERPNode{{
+					Name:     "1c",
+					RegionID: 1,
+					HostName: "derp1c.tailscale.com",
+					IPv4:     "127.0.0.1",
+					IPv6:     "::1",
+				}},
+			},
+		},
+	}
+	d, err := json.Marshal(initialCache)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(cacheFile, d, 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Clear any existing cached DERP map(s)
+	cachedDERPMap.Store(nil)
+
+	// Load the cache
+	SetCachePath(cacheFile)
+	if cm := cachedDERPMap.Load(); !reflect.DeepEqual(initialCache, cm) {
+		t.Fatalf("cached map was %+v; want %+v", cm, initialCache)
+	}
+
+	// Verify that our DERP map is merged with the cache.
+	dm := getDERPMap()
+	region, ok := dm.Regions[99]
+	if !ok {
+		t.Fatal("expected region 99")
+	}
+	if !reflect.DeepEqual(region, initialCache.Regions[99]) {
+		t.Fatalf("region 99: got %+v; want %+v", region, initialCache.Regions[99])
+	}
+
+	// Verify that our cache can't override a statically-baked-in DERP server.
+	n0 := dm.Regions[1].Nodes[0]
+	if n0.IPv4 == "127.0.0.1" || n0.IPv6 == "::1" {
+		t.Errorf("got %+v; expected no overwrite for node", n0)
+	}
+
+	// Also, make sure that the static DERP map still has the same first
+	// node as when this test was last written/updated; this ensures that
+	// we don't accidentally start allowing overwrites due to some of the
+	// test's assumptions changing out from underneath us as we update the
+	// JSON file of fallback servers.
+	if getStaticDERPMap().Regions[1].Nodes[0].HostName != "derp1c.tailscale.com" {
+		t.Errorf("DERP server has a different name; please update this test")
+	}
+}
+
+func TestCacheUnchanged(t *testing.T) {
+	oldlog := logfunc.Load()
+	SetLogger(t.Logf)
+	t.Cleanup(func() {
+		SetLogger(oldlog)
+	})
+	cacheFile := filepath.Join(t.TempDir(), "cache.json")
+
+	// Write initial cache value
+	initialCache := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			99: {
+				RegionID:   99,
+				RegionCode: "test",
+				RegionName: "Testville",
+				Nodes: []*tailcfg.DERPNode{{
+					Name:     "99a",
+					RegionID: 99,
+					HostName: "derp99a.tailscale.com",
+					IPv4:     "1.2.3.4",
+				}},
+			},
+		},
+	}
+	d, err := json.Marshal(initialCache)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(cacheFile, d, 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Clear any existing cached DERP map(s)
+	cachedDERPMap.Store(nil)
+
+	// Load the cache
+	SetCachePath(cacheFile)
+	if cm := cachedDERPMap.Load(); !reflect.DeepEqual(initialCache, cm) {
+		t.Fatalf("cached map was %+v; want %+v", cm, initialCache)
+	}
+
+	// Remove the cache file on-disk, then re-set to the current value. If
+	// our equality comparison is working, we won't rewrite the file
+	// on-disk since the cached value won't have changed.
+	if err := os.Remove(cacheFile); err != nil {
+		t.Fatal(err)
+	}
+
+	UpdateCache(initialCache)
+	if _, err := os.Stat(cacheFile); !os.IsNotExist(err) {
+		t.Fatalf("got err=%v; expected to not find cache file", err)
+	}
+
+	// Now, update the cache with something slightly different and verify
+	// that we did re-write the file on-disk.
+	updatedCache := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			99: {
+				RegionID:   99,
+				RegionCode: "test",
+				RegionName: "Testville",
+				Nodes:      []*tailcfg.DERPNode{ /* set below */ },
+			},
+		},
+	}
+	clonedNode := *initialCache.Regions[99].Nodes[0]
+	clonedNode.IPv4 = "1.2.3.5"
+	updatedCache.Regions[99].Nodes = append(updatedCache.Regions[99].Nodes, &clonedNode)
+
+	UpdateCache(updatedCache)
+	if st, err := os.Stat(cacheFile); err != nil {
+		t.Fatalf("could not stat cache file; err=%v", err)
+	} else if !st.Mode().IsRegular() || st.Size() == 0 {
+		t.Fatalf("didn't find non-empty regular file; mode=%v size=%d", st.Mode(), st.Size())
+	}
+}

net/netns/netns_linux.go

@@ -15,6 +15,7 @@ import (
 	"syscall"
 
 	"golang.org/x/sys/unix"
+	"tailscale.com/envknob"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 )
@@ -62,9 +63,14 @@ func socketMarkWorks() bool {
 	return true
 }
 
-// useSocketMark reports whether SO_MARK works.
+var forceBindToDevice = envknob.Bool("TS_FORCE_LINUX_BIND_TO_DEVICE")
+
+// UseSocketMark reports whether SO_MARK is in use.
 // If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
-func useSocketMark() bool {
+func UseSocketMark() bool {
+	if forceBindToDevice {
+		return false
+	}
 	socketMarkWorksOnce.Do(func() {
 		socketMarkWorksOnce.v = socketMarkWorks()
 	})
@@ -97,7 +103,7 @@ func controlC(network, address string, c syscall.RawConn) error {
 
 	var sockErr error
 	err := c.Control(func(fd uintptr) {
-		if useSocketMark() {
+		if UseSocketMark() {
 			sockErr = setBypassMark(fd)
 		} else {
 			sockErr = bindToDevice(fd)

net/stun/stun.go

@@ -208,7 +208,7 @@ func ParseResponse(b []byte) (tID TxID, addr netip.AddrPort, err error) {
 		b = b[:attrsLen] // trim trailing packet bytes
 	}
 
-	var addr6, fallbackAddr, fallbackAddr6 netip.AddrPort
+	var fallbackAddr netip.AddrPort
 
 	// Read through the attributes.
 	// The the addr+port reported by XOR-MAPPED-ADDRESS
@@ -218,24 +218,20 @@ func ParseResponse(b []byte) (tID TxID, addr netip.AddrPort, err error) {
 	if err := foreachAttr(b, func(attrType uint16, attr []byte) error {
 		switch attrType {
 		case attrXorMappedAddress, attrXorMappedAddressAlt:
-			a, p, err := xorMappedAddress(tID, attr)
+			ipSlice, port, err := xorMappedAddress(tID, attr)
 			if err != nil {
 				return err
 			}
-			if len(a) == 16 {
-				addr6 = netip.AddrPortFrom(netip.AddrFrom16(*(*[16]byte)([]byte(a))), p)
-			} else {
-				addr = netip.AddrPortFrom(netip.AddrFrom4(*(*[4]byte)([]byte(a))), p)
+			if ip, ok := netip.AddrFromSlice(ipSlice); ok {
+				addr = netip.AddrPortFrom(ip.Unmap(), port)
 			}
 		case attrMappedAddress:
-			a, p, err := mappedAddress(attr)
+			ipSlice, port, err := mappedAddress(attr)
 			if err != nil {
 				return ErrMalformedAttrs
 			}
-			if len(a) == 16 {
-				fallbackAddr6 = netip.AddrPortFrom(netip.AddrFrom16(*(*[16]byte)([]byte(a))), p)
-			} else {
-				fallbackAddr = netip.AddrPortFrom(netip.AddrFrom4(*(*[4]byte)([]byte(a))), p)
+			if ip, ok := netip.AddrFromSlice(ipSlice); ok {
+				fallbackAddr = netip.AddrPortFrom(ip.Unmap(), port)
 			}
 		}
 		return nil
@@ -250,12 +246,6 @@ func ParseResponse(b []byte) (tID TxID, addr netip.AddrPort, err error) {
 	if fallbackAddr.IsValid() {
 		return tID, fallbackAddr, nil
 	}
-	if addr6.IsValid() {
-		return tID, addr6, nil
-	}
-	if fallbackAddr6.IsValid() {
-		return tID, fallbackAddr6, nil
-	}
 	return tID, netip.AddrPort{}, ErrMalformedAttrs
 }
 

net/stun/stun_test.go

@@ -6,11 +6,13 @@ package stun_test
 
 import (
 	"bytes"
+	"encoding/hex"
 	"fmt"
 	"net/netip"
 	"testing"
 
 	"tailscale.com/net/stun"
+	"tailscale.com/util/must"
 )
 
 // TODO(bradfitz): fuzz this.
@@ -175,6 +177,13 @@ var responseTests = []struct {
 		wantAddr: netip.AddrFrom4([4]byte{127, 0, 0, 1}),
 		wantPort: 61300,
 	},
+	{
+		name:     "no-4in6",
+		data:     must.Get(hex.DecodeString("010100182112a4424fd5d202dcb37d31fc773306002000140002cd3d2112a4424fd5d202dcb382ce2dc3fcc7")),
+		wantTID:  []byte{79, 213, 210, 2, 220, 179, 125, 49, 252, 119, 51, 6},
+		wantAddr: netip.AddrFrom4([4]byte{209, 180, 207, 193}),
+		wantPort: 60463,
+	},
 }
 
 func TestParseResponse(t *testing.T) {

portlist/clean.go

@@ -26,6 +26,9 @@ func argvSubject(argv ...string) string {
 		ret = filepath.Base(argv[1])
 	}
 
+	// Handle space separated argv
+	ret, _, _ = strings.Cut(ret, " ")
+
 	// Remove common noise.
 	ret = strings.TrimSpace(ret)
 	ret = strings.TrimSuffix(ret, ".exe")

portlist/clean_test.go

@@ -31,6 +31,22 @@ func TestArgvSubject(t *testing.T) {
 			in:   []string{"/bin/mono", "/sbin/exampleProgram.bin"},
 			want: "exampleProgram.bin",
 		},
+		{
+			in:   []string{"/usr/bin/sshd_config [listener] 1 of 10-100 startups"},
+			want: "sshd_config",
+		},
+		{
+			in:   []string{"/usr/bin/sshd [listener] 0 of 10-100 startups"},
+			want: "sshd",
+		},
+		{
+			in:   []string{"/opt/aws/bin/eic_run_authorized_keys %u %f -o AuthorizedKeysCommandUser ec2-instance-connect [listener] 0 of 10-100 startups"},
+			want: "eic_run_authorized_keys",
+		},
+		{
+			in:   []string{"/usr/bin/nginx worker"},
+			want: "nginx",
+		},
 	}
 
 	for _, test := range tests {

tailcfg/tailcfg.go

@@ -40,45 +40,46 @@ type CapabilityVersion int
 //
 // History of versions:
 //
-//	 3: implicit compression, keep-alives
-//	 4: opt-in keep-alives via KeepAlive field, opt-in compression via Compress
-//	 5: 2020-10-19, implies IncludeIPv6, delta Peers/UserProfiles, supports MagicDNS
-//	 6: 2020-12-07: means MapResponse.PacketFilter nil means unchanged
-//	 7: 2020-12-15: FilterRule.SrcIPs accepts CIDRs+ranges, doesn't warn about 0.0.0.0/::
-//	 8: 2020-12-19: client can buggily receive IPv6 addresses and routes if beta enabled server-side
-//	 9: 2020-12-30: client doesn't auto-add implicit search domains from peers; only DNSConfig.Domains
-//	10: 2021-01-17: client understands MapResponse.PeerSeenChange
-//	11: 2021-03-03: client understands IPv6, multiple default routes, and goroutine dumping
-//	12: 2021-03-04: client understands PingRequest
-//	13: 2021-03-19: client understands FilterRule.IPProto
-//	14: 2021-04-07: client understands DNSConfig.Routes and DNSConfig.Resolvers
-//	15: 2021-04-12: client treats nil MapResponse.DNSConfig as meaning unchanged
-//	16: 2021-04-15: client understands Node.Online, MapResponse.OnlineChange
-//	17: 2021-04-18: MapResponse.Domain empty means unchanged
-//	18: 2021-04-19: MapResponse.Node nil means unchanged (all fields now omitempty)
-//	19: 2021-04-21: MapResponse.Debug.SleepSeconds
-//	20: 2021-06-11: MapResponse.LastSeen used even less (https://github.com/tailscale/tailscale/issues/2107)
-//	21: 2021-06-15: added MapResponse.DNSConfig.CertDomains
-//	22: 2021-06-16: added MapResponse.DNSConfig.ExtraRecords
-//	23: 2021-08-25: DNSConfig.Routes values may be empty (for ExtraRecords support in 1.14.1+)
-//	24: 2021-09-18: MapResponse.Health from control to node; node shows in "tailscale status"
-//	25: 2021-11-01: MapResponse.Debug.Exit
-//	26: 2022-01-12: (nothing, just bumping for 1.20.0)
-//	27: 2022-02-18: start of SSHPolicy being respected
-//	28: 2022-03-09: client can communicate over Noise.
-//	29: 2022-03-21: MapResponse.PopBrowserURL
-//	30: 2022-03-22: client can request id tokens.
-//	31: 2022-04-15: PingRequest & PingResponse TSMP & disco support
-//	32: 2022-04-17: client knows FilterRule.CapMatch
-//	33: 2022-07-20: added MapResponse.PeersChangedPatch (DERPRegion + Endpoints)
-//	34: 2022-08-02: client understands CapabilityFileSharingTarget
-//	36: 2022-08-02: added PeersChangedPatch.{Key,DiscoKey,Online,LastSeen,KeyExpiry,Capabilities}
-//	37: 2022-08-09: added Debug.{SetForceBackgroundSTUN,SetRandomizeClientPort}; Debug are sticky
-//	38: 2022-08-11: added PingRequest.URLIsNoise
-//	39: 2022-08-15: clients can talk Noise over arbitrary HTTPS port
-//	40: 2022-08-22: added Node.KeySignature, PeersChangedPatch.KeySignature
-//	41: 2022-08-30: uses 100.100.100.100 for route-less ExtraRecords if global nameservers is set
-const CurrentCapabilityVersion CapabilityVersion = 41
+//   - 3: implicit compression, keep-alives
+//   - 4: opt-in keep-alives via KeepAlive field, opt-in compression via Compress
+//   - 5: 2020-10-19, implies IncludeIPv6, delta Peers/UserProfiles, supports MagicDNS
+//   - 6: 2020-12-07: means MapResponse.PacketFilter nil means unchanged
+//   - 7: 2020-12-15: FilterRule.SrcIPs accepts CIDRs+ranges, doesn't warn about 0.0.0.0/::
+//   - 8: 2020-12-19: client can buggily receive IPv6 addresses and routes if beta enabled server-side
+//   - 9: 2020-12-30: client doesn't auto-add implicit search domains from peers; only DNSConfig.Domains
+//   - 10: 2021-01-17: client understands MapResponse.PeerSeenChange
+//   - 11: 2021-03-03: client understands IPv6, multiple default routes, and goroutine dumping
+//   - 12: 2021-03-04: client understands PingRequest
+//   - 13: 2021-03-19: client understands FilterRule.IPProto
+//   - 14: 2021-04-07: client understands DNSConfig.Routes and DNSConfig.Resolvers
+//   - 15: 2021-04-12: client treats nil MapResponse.DNSConfig as meaning unchanged
+//   - 16: 2021-04-15: client understands Node.Online, MapResponse.OnlineChange
+//   - 17: 2021-04-18: MapResponse.Domain empty means unchanged
+//   - 18: 2021-04-19: MapResponse.Node nil means unchanged (all fields now omitempty)
+//   - 19: 2021-04-21: MapResponse.Debug.SleepSeconds
+//   - 20: 2021-06-11: MapResponse.LastSeen used even less (https://github.com/tailscale/tailscale/issues/2107)
+//   - 21: 2021-06-15: added MapResponse.DNSConfig.CertDomains
+//   - 22: 2021-06-16: added MapResponse.DNSConfig.ExtraRecords
+//   - 23: 2021-08-25: DNSConfig.Routes values may be empty (for ExtraRecords support in 1.14.1+)
+//   - 24: 2021-09-18: MapResponse.Health from control to node; node shows in "tailscale status"
+//   - 25: 2021-11-01: MapResponse.Debug.Exit
+//   - 26: 2022-01-12: (nothing, just bumping for 1.20.0)
+//   - 27: 2022-02-18: start of SSHPolicy being respected
+//   - 28: 2022-03-09: client can communicate over Noise.
+//   - 29: 2022-03-21: MapResponse.PopBrowserURL
+//   - 30: 2022-03-22: client can request id tokens.
+//   - 31: 2022-04-15: PingRequest & PingResponse TSMP & disco support
+//   - 32: 2022-04-17: client knows FilterRule.CapMatch
+//   - 33: 2022-07-20: added MapResponse.PeersChangedPatch (DERPRegion + Endpoints)
+//   - 34: 2022-08-02: client understands CapabilityFileSharingTarget
+//   - 36: 2022-08-02: added PeersChangedPatch.{Key,DiscoKey,Online,LastSeen,KeyExpiry,Capabilities}
+//   - 37: 2022-08-09: added Debug.{SetForceBackgroundSTUN,SetRandomizeClientPort}; Debug are sticky
+//   - 38: 2022-08-11: added PingRequest.URLIsNoise
+//   - 39: 2022-08-15: clients can talk Noise over arbitrary HTTPS port
+//   - 40: 2022-08-22: added Node.KeySignature, PeersChangedPatch.KeySignature
+//   - 41: 2022-08-30: uses 100.100.100.100 for route-less ExtraRecords if global nameservers is set
+//   - 42: 2022-09-06: NextDNS DoH support; see https://github.com/tailscale/tailscale/pull/5556
+const CurrentCapabilityVersion CapabilityVersion = 42
 
 type StableID string
 
@@ -465,17 +466,36 @@ type Service struct {
 // Because it contains pointers (slices), this type should not be used
 // as a value type.
 type Hostinfo struct {
-	IPNVersion      string         `json:",omitempty"` // version of this code
-	FrontendLogID   string         `json:",omitempty"` // logtail ID of frontend instance
-	BackendLogID    string         `json:",omitempty"` // logtail ID of backend instance
-	OS              string         `json:",omitempty"` // operating system the client runs on (a version.OS value)
-	OSVersion       string         `json:",omitempty"` // operating system version, with optional distro prefix ("Debian 10.4", "Windows 10 Pro 10.0.19041")
+	IPNVersion    string `json:",omitempty"` // version of this code (in version.Long format)
+	FrontendLogID string `json:",omitempty"` // logtail ID of frontend instance
+	BackendLogID  string `json:",omitempty"` // logtail ID of backend instance
+	OS            string `json:",omitempty"` // operating system the client runs on (a version.OS value)
+
+	// OSVersion is the version of the OS, if available.
+	//
+	// For Android, it's like "10", "11", "12", etc. For iOS and macOS it's like
+	// "15.6.1" or "12.4.0". For Windows it's like "10.0.19044.1889". For
+	// FreeBSD it's like "12.3-STABLE".
+	//
+	// For Linux, prior to Tailscale 1.32, we jammed a bunch of fields into this
+	// string on Linux, like "Debian 10.4; kernel=xxx; container; env=kn" and so
+	// on. As of Tailscale 1.32, this is simply the kernel version on Linux, like
+	// "5.10.0-17-amd64".
+	OSVersion string `json:",omitempty"`
+
+	Container      opt.Bool `json:",omitempty"` // whether the client is running in a container
+	Env            string   `json:",omitempty"` // a hostinfo.EnvType in string form
+	Distro         string   `json:",omitempty"` // "debian", "ubuntu", "nixos", ...
+	DistroVersion  string   `json:",omitempty"` // "20.04", ...
+	DistroCodeName string   `json:",omitempty"` // "jammy", "bullseye", ...
+
 	Desktop         opt.Bool       `json:",omitempty"` // if a desktop was detected on Linux
 	Package         string         `json:",omitempty"` // Tailscale package to disambiguate ("choco", "appstore", etc; "" for unknown)
 	DeviceModel     string         `json:",omitempty"` // mobile phone model ("Pixel 3a", "iPhone12,3")
 	Hostname        string         `json:",omitempty"` // name of the host the client runs on
 	ShieldsUp       bool           `json:",omitempty"` // indicates whether the host is blocking incoming connections
 	ShareeNode      bool           `json:",omitempty"` // indicates this node exists in netmap because it's owned by a shared-to user
+	NoLogsNoSupport bool           `json:",omitempty"` // indicates that the user has opted out of sending logs and support
 	GoArch          string         `json:",omitempty"` // the host's GOARCH value (of the running binary)
 	GoVersion       string         `json:",omitempty"` // Go version binary was built with
 	RoutableIPs     []netip.Prefix `json:",omitempty"` // set of IP ranges this client can route
@@ -903,11 +923,6 @@ type MapRequest struct {
 	Stream      bool // if true, multiple MapResponse objects are returned
 	Hostinfo    *Hostinfo
 
-	// TKA describes request parameters relating to a local instance of
-	// the tailnet key authority. This field is omitted if a local instance
-	// is not running.
-	TKA *TKAMapRequest `json:",omitempty"`
-
 	// Endpoints are the client's magicsock UDP ip:port endpoints (IPv4 or IPv6).
 	Endpoints []string
 	// EndpointTypes are the types of the corresponding endpoints in Endpoints.
@@ -1347,9 +1362,15 @@ type MapResponse struct {
 	// ControlTime, if non-zero, is the current timestamp according to the control server.
 	ControlTime *time.Time `json:",omitempty"`
 
-	// TKA, if non-nil, describes updates for the local instance of the
-	// tailnet key authority.
-	TKA *TKAMapResponse `json:",omitempty"`
+	// TKAInfo describes the control plane's view of tailnet
+	// key authority (TKA) state.
+	//
+	// An initial nil TKAInfo indicates that the control plane
+	// believes TKA should not be enabled. An initial non-nil TKAInfo
+	// indicates the control plane believes TKA should be enabled.
+	// A nil TKAInfo in a mapresponse stream (i.e. a 'delta' mapresponse)
+	// indicates no change from the value sent earlier.
+	TKAInfo *TKAInfo `json:",omitempty"`
 
 	// Debug is normally nil, except for when the control server
 	// is setting debug settings on a node.
@@ -1853,85 +1874,6 @@ type PeerChange struct {
 	Capabilities *[]string `json:",omitempty"`
 }
 
-// TKAInitBeginRequest submits a genesis AUM to seed the creation of the
-// tailnet's key authority.
-type TKAInitBeginRequest struct {
-	NodeID NodeID // NodeID of the initiating client
-
-	GenesisAUM tkatype.MarshaledAUM
-}
-
-// TKASignInfo describes information about an existing node that needs
-// to be signed into a node-key signature.
-type TKASignInfo struct {
-	NodeID     NodeID // NodeID of the node-key being signed
-	NodePublic key.NodePublic
-
-	// RotationPubkey specifies the public key which may sign
-	// a NodeKeySignature (NKS), which rotates the node key.
-	//
-	// This is necessary so the node can rotate its node-key without
-	// talking to a node which holds a trusted network-lock key.
-	// It does this by nesting the original NKS in a 'rotation' NKS,
-	// which it then signs with the key corresponding to RotationPubkey.
-	//
-	// This field expects a raw ed25519 public key.
-	RotationPubkey []byte
-}
-
-// TKAInitBeginResponse describes node information which must be signed to
-// complete initialization of the tailnets' key authority.
-type TKAInitBeginResponse struct {
-	NeedSignatures []TKASignInfo
-}
-
-// TKAInitFinishRequest finalizes initialization of the tailnet key authority
-// by submitting node-key signatures for all existing nodes.
-type TKAInitFinishRequest struct {
-	NodeID NodeID // NodeID of the initiating client
-
-	Signatures map[NodeID]tkatype.MarshaledSignature
-}
-
-// TKAInitFinishResponse describes the successful enablement of the tailnet's
-// key authority.
-type TKAInitFinishResponse struct{}
-
-// TKAMapRequest describes request parameters relating to the tailnet key
-// authority instance on this node. This information is transmitted as
-// part of the MapRequest.
-type TKAMapRequest struct {
-	// Head is the AUMHash of the latest authority update message committed
-	// by this node.
-	Head string // tka.AUMHash.String
-}
-
-// TKAMapResponse encodes information for the tailnet key authority
-// instance on the node. This information is transmitted as
-// part of the MapResponse.
-//
-// If there are no updates to be transmitted (in other words, if both
-// control and the node have the same head hash), len(Updates) == 0 and
-// WantSync is false.
-//
-// If control has updates that build off the head hash reported by the
-// node, they are simply transmitted in Updates (avoiding the more
-// expensive synchronization process).
-//
-// In all other cases, WantSync is set to true, and the node is expected
-// to reach out to control separately to synchronize.
-type TKAMapResponse struct {
-	// Updates is any AUMs that control believes the node should apply.
-	Updates []tkatype.MarshaledAUM `json:",omitempty"`
-
-	// WantSync is set by control to request the node complete AUM
-	// synchronization.
-	//
-	// TODO(tom): Implement AUM synchronization, probably as noise endpoints
-	// /machine/tka/sync/offer & /machine/tka/sync/send.
-	WantSync bool `json:",omitempty"`
-}
-
 // DerpMagicIP is a fake WireGuard endpoint IP address that means to
 // use DERP. When used (in the Node.DERP field), the port number of
 // the WireGuard endpoint is the DERP region ID number to use.

tailcfg/tailcfg_clone.go

@@ -120,12 +120,18 @@ var _HostinfoCloneNeedsRegeneration = Hostinfo(struct {
 	BackendLogID    string
 	OS              string
 	OSVersion       string
+	Container       opt.Bool
+	Env             string
+	Distro          string
+	DistroVersion   string
+	DistroCodeName  string
 	Desktop         opt.Bool
 	Package         string
 	DeviceModel     string
 	Hostname        string
 	ShieldsUp       bool
 	ShareeNode      bool
+	NoLogsNoSupport bool
 	GoArch          string
 	GoVersion       string
 	RoutableIPs     []netip.Prefix

tailcfg/tailcfg_test.go

@@ -31,13 +31,33 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestHostinfoEqual(t *testing.T) {
 	hiHandles := []string{
-		"IPNVersion", "FrontendLogID", "BackendLogID",
-		"OS", "OSVersion", "Desktop", "Package", "DeviceModel", "Hostname",
-		"ShieldsUp", "ShareeNode",
-		"GoArch", "GoVersion",
-		"RoutableIPs", "RequestTags",
-		"Services", "NetInfo", "SSH_HostKeys", "Cloud",
-		"Userspace", "UserspaceRouter",
+		"IPNVersion",
+		"FrontendLogID",
+		"BackendLogID",
+		"OS",
+		"OSVersion",
+		"Container",
+		"Env",
+		"Distro",
+		"DistroVersion",
+		"DistroCodeName",
+		"Desktop",
+		"Package",
+		"DeviceModel",
+		"Hostname",
+		"ShieldsUp",
+		"ShareeNode",
+		"NoLogsNoSupport",
+		"GoArch",
+		"GoVersion",
+		"RoutableIPs",
+		"RequestTags",
+		"Services",
+		"NetInfo",
+		"SSH_HostKeys",
+		"Cloud",
+		"Userspace",
+		"UserspaceRouter",
 	}
 	if have := fieldsOf(reflect.TypeOf(Hostinfo{})); !reflect.DeepEqual(have, hiHandles) {
 		t.Errorf("Hostinfo.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
@@ -661,7 +681,7 @@ func TestRegisterRequestNilClone(t *testing.T) {
 // We've screwed this up several times.
 func TestCurrentCapabilityVersion(t *testing.T) {
 	f := must.Get(os.ReadFile("tailcfg.go"))
-	matches := regexp.MustCompile(`(?m)^//\s+(\d+): \d\d\d\d-\d\d-\d\d: `).FindAllStringSubmatch(string(f), -1)
+	matches := regexp.MustCompile(`(?m)^//[\s-]+(\d+): \d\d\d\d-\d\d-\d\d: `).FindAllStringSubmatch(string(f), -1)
 	max := 0
 	for _, m := range matches {
 		n := must.Get(strconv.Atoi(m[1]))

tailcfg/tailcfg_view.go

@@ -250,19 +250,25 @@ func (v *HostinfoView) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
-func (v HostinfoView) IPNVersion() string    { return v.ж.IPNVersion }
-func (v HostinfoView) FrontendLogID() string { return v.ж.FrontendLogID }
-func (v HostinfoView) BackendLogID() string  { return v.ж.BackendLogID }
-func (v HostinfoView) OS() string            { return v.ж.OS }
-func (v HostinfoView) OSVersion() string     { return v.ж.OSVersion }
-func (v HostinfoView) Desktop() opt.Bool     { return v.ж.Desktop }
-func (v HostinfoView) Package() string       { return v.ж.Package }
-func (v HostinfoView) DeviceModel() string   { return v.ж.DeviceModel }
-func (v HostinfoView) Hostname() string      { return v.ж.Hostname }
-func (v HostinfoView) ShieldsUp() bool       { return v.ж.ShieldsUp }
-func (v HostinfoView) ShareeNode() bool      { return v.ж.ShareeNode }
-func (v HostinfoView) GoArch() string        { return v.ж.GoArch }
-func (v HostinfoView) GoVersion() string     { return v.ж.GoVersion }
+func (v HostinfoView) IPNVersion() string     { return v.ж.IPNVersion }
+func (v HostinfoView) FrontendLogID() string  { return v.ж.FrontendLogID }
+func (v HostinfoView) BackendLogID() string   { return v.ж.BackendLogID }
+func (v HostinfoView) OS() string             { return v.ж.OS }
+func (v HostinfoView) OSVersion() string      { return v.ж.OSVersion }
+func (v HostinfoView) Container() opt.Bool    { return v.ж.Container }
+func (v HostinfoView) Env() string            { return v.ж.Env }
+func (v HostinfoView) Distro() string         { return v.ж.Distro }
+func (v HostinfoView) DistroVersion() string  { return v.ж.DistroVersion }
+func (v HostinfoView) DistroCodeName() string { return v.ж.DistroCodeName }
+func (v HostinfoView) Desktop() opt.Bool      { return v.ж.Desktop }
+func (v HostinfoView) Package() string        { return v.ж.Package }
+func (v HostinfoView) DeviceModel() string    { return v.ж.DeviceModel }
+func (v HostinfoView) Hostname() string       { return v.ж.Hostname }
+func (v HostinfoView) ShieldsUp() bool        { return v.ж.ShieldsUp }
+func (v HostinfoView) ShareeNode() bool       { return v.ж.ShareeNode }
+func (v HostinfoView) NoLogsNoSupport() bool  { return v.ж.NoLogsNoSupport }
+func (v HostinfoView) GoArch() string         { return v.ж.GoArch }
+func (v HostinfoView) GoVersion() string      { return v.ж.GoVersion }
 func (v HostinfoView) RoutableIPs() views.IPPrefixSlice {
 	return views.IPPrefixSliceOf(v.ж.RoutableIPs)
 }
@@ -282,12 +288,18 @@ var _HostinfoViewNeedsRegeneration = Hostinfo(struct {
 	BackendLogID    string
 	OS              string
 	OSVersion       string
+	Container       opt.Bool
+	Env             string
+	Distro          string
+	DistroVersion   string
+	DistroCodeName  string
 	Desktop         opt.Bool
 	Package         string
 	DeviceModel     string
 	Hostname        string
 	ShieldsUp       bool
 	ShareeNode      bool
+	NoLogsNoSupport bool
 	GoArch          string
 	GoVersion       string
 	RoutableIPs     []netip.Prefix

tailcfg/tka.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tailcfg
+
+import (
+	"tailscale.com/types/key"
+	"tailscale.com/types/tkatype"
+)
+
+// TKAInitBeginRequest submits a genesis AUM to seed the creation of the
+// tailnet's key authority.
+type TKAInitBeginRequest struct {
+	// NodeID is the node of the initiating client.
+	// It must match the machine key being used to communicate over noise.
+	NodeID NodeID
+
+	// GenesisAUM is the initial (genesis) AUM that the node generated
+	// to bootstrap tailnet key authority state.
+	GenesisAUM tkatype.MarshaledAUM
+}
+
+// TKASignInfo describes information about an existing node that needs
+// to be signed into a node-key signature.
+type TKASignInfo struct {
+	// NodeID is the ID of the node which needs a signature. It must
+	// correspond to NodePublic.
+	NodeID NodeID
+	// NodePublic is the node (Wireguard) public key which is being
+	// signed.
+	NodePublic key.NodePublic
+
+	// RotationPubkey specifies the public key which may sign
+	// a NodeKeySignature (NKS), which rotates the node key.
+	//
+	// This is necessary so the node can rotate its node-key without
+	// talking to a node which holds a trusted network-lock key.
+	// It does this by nesting the original NKS in a 'rotation' NKS,
+	// which it then signs with the key corresponding to RotationPubkey.
+	//
+	// This field expects a raw ed25519 public key.
+	RotationPubkey []byte
+}
+
+// TKAInitBeginResponse is the JSON response from a /tka/init/begin RPC.
+// This structure describes node information which must be signed to
+// complete initialization of the tailnets' key authority.
+type TKAInitBeginResponse struct {
+	// NeedSignatures specify information about the nodes in your tailnet
+	// which need initial signatures to function once the tailnet key
+	// authority is in use. The generated signatures should then be
+	// submitted in a /tka/init/finish RPC.
+	NeedSignatures []TKASignInfo
+}
+
+// TKAInitFinishRequest is the JSON request of a /tka/init/finish RPC.
+// This RPC finalizes initialization of the tailnet key authority
+// by submitting node-key signatures for all existing nodes.
+type TKAInitFinishRequest struct {
+	// NodeID is the node ID of the initiating client.
+	NodeID NodeID
+
+	// Signatures are serialized tka.NodeKeySignatures for all nodes
+	// in the tailnet.
+	Signatures map[NodeID]tkatype.MarshaledSignature
+}
+
+// TKAInitFinishResponse is the JSON response from a /tka/init/finish RPC.
+// This schema describes the successful enablement of the tailnet's
+// key authority.
+type TKAInitFinishResponse struct {
+	// Nothing. (yet?)
+}
+
+// TKAInfo encodes the control plane's view of tailnet key authority (TKA)
+// state. This information is transmitted as part of the MapResponse.
+type TKAInfo struct {
+	// Head describes the hash of the latest AUM applied to the authority.
+	// Head is encoded as tka.AUMHash.MarshalText.
+	//
+	// If the Head state differs to that known locally, the node should perform
+	// synchronization via a separate RPC.
+	//
+	// TODO(tom): Implement AUM synchronization as noise endpoints
+	// /machine/tka/sync/offer & /machine/tka/sync/send.
+	Head string `json:",omitempty"`
+
+	// Disabled indicates the control plane believes TKA should be disabled,
+	// and the node should reach out to fetch a disablement
+	// secret. If the disablement secret verifies, then the node should then
+	// disable TKA locally.
+	// This field exists to disambiguate a nil TKAInfo in a delta mapresponse
+	// from a nil TKAInfo indicating TKA should be disabled.
+	//
+	// TODO(tom): Implement /machine/tka/boostrap as a noise endpoint, to
+	// communicate the genesis AUM & any disablement secrets.
+	Disabled bool `json:",omitempty"`
+}
+
+// TKABootstrapRequest is sent by a node to get information necessary for
+// enabling or disabling the tailnet key authority.
+type TKABootstrapRequest struct {
+	// Head represents the node's head AUMHash (tka.Authority.Head), if
+	// network lock is enabled.
+	Head string
+}
+
+// TKABootstrapResponse encodes values necessary to enable or disable
+// the tailnet key authority (TKA).
+type TKABootstrapResponse struct {
+	// GenesisAUM returns the initial AUM necessary to initialize TKA.
+	GenesisAUM tkatype.MarshaledAUM `json:",omitempty"`
+
+	// DisablementSecret encodes a secret necessary to disable TKA.
+	DisablementSecret []byte `json:",omitempty"`
+}
+
+// TKASyncOfferRequest encodes a request to synchronize tailnet key authority
+// state (TKA). Values of type tka.AUMHash are encoded as strings in their
+// MarshalText form.
+type TKASyncOfferRequest struct {
+	// Head represents the node's head AUMHash (tka.Authority.Head). This
+	// corresponds to tka.SyncOffer.Head.
+	Head string
+	// Ancestors represents a selection of ancestor AUMHash values ascending
+	// from the current head. This corresponds to tka.SyncOffer.Ancestors.
+	Ancestors []string
+}
+
+// TKASyncOfferResponse encodes a response in synchronizing a node's
+// tailnet key authority state. Values of type tka.AUMHash are encoded as
+// strings in their MarshalText form.
+type TKASyncOfferResponse struct {
+	// Head represents the control plane's head AUMHash (tka.Authority.Head).
+	// This corresponds to tka.SyncOffer.Head.
+	Head string
+	// Ancestors represents a selection of ancestor AUMHash values ascending
+	// from the control plane's head. This corresponds to
+	// tka.SyncOffer.Ancestors.
+	Ancestors []string
+	// MissingAUMs encodes AUMs that the control plane believes the node
+	// is missing.
+	MissingAUMs []tkatype.MarshaledAUM
+}
+
+// TKASyncSendRequest encodes AUMs that a node believes the control plane
+// is missing.
+type TKASyncSendRequest struct {
+	// MissingAUMs encodes AUMs that the node believes the control plane
+	// is missing.
+	MissingAUMs []tkatype.MarshaledAUM
+}
+
+// TKASyncSendResponse encodes the control plane's response to a node
+// submitting AUMs during AUM synchronization.
+type TKASyncSendResponse struct {
+	// Head represents the control plane's head AUMHash (tka.Authority.Head),
+	// after applying the missing AUMs.
+	Head string
+}

tsnet/tsnet.go

@@ -541,6 +541,9 @@ func (ln *listener) Close() error {
 	return nil
 }
 
+// Server returns the tsnet Server associated with the listener.
+func (ln *listener) Server() *Server { return ln.s }
+
 type addr struct{ ln *listener }
 
 func (a addr) Network() string { return a.ln.key.network }

tsnet/tsnet_test.go

@@ -0,0 +1,18 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsnet
+
+import "testing"
+
+// TestListener_Server ensures that the listener type always keeps the Server
+// method, which is used by some external applications to identify a tsnet.Listener
+// from other net.Listeners, as well as access the underlying Server.
+func TestListener_Server(t *testing.T) {
+	s := &Server{}
+	ln := listener{s: s}
+	if ln.Server() != s {
+		t.Errorf("listener.Server() returned %v, want %v", ln.Server(), s)
+	}
+}

types/dnstype/dnstype.go

@@ -17,8 +17,11 @@ type Resolver struct {
 	//  - A plain IP address for a "classic" UDP+TCP DNS resolver.
 	//    This is the common format as sent by the control plane.
 	//  - An IP:port, for tests.
+	//  - "https://resolver.com/path" for DNS over HTTPS; currently
+	//    as of 2022-09-08 only used for certain well-known resolvers
+	//    (see the publicdns package) for which the IP addresses to dial DoH are
+	//    known ahead of time, so bootstrap DNS resolution is not required.
 	//  - [TODO] "tls://resolver.com" for DNS over TCP+TLS
-	//  - [TODO] "https://resolver.com/query-tmpl" for DNS over HTTPS
 	Addr string `json:",omitempty"`
 
 	// BootstrapResolution is an optional suggested resolution for the
@@ -27,6 +30,8 @@ type Resolver struct {
 	// BootstrapResolution may be empty, in which case clients should
 	// look up the DoT/DoH server using their local "classic" DNS
 	// resolver.
+	//
+	// As of 2022-09-08, BootstrapResolution is not yet used.
 	BootstrapResolution []netip.Addr `json:",omitempty"`
 }
 

util/mak/mak.go

@@ -25,10 +25,8 @@ func Set[K comparable, V any, T ~map[K]V](m *T, k K, v V) {
 // (currently only a slice or a map) and makes sure it's non-nil for
 // JSON serialization. (In particular, JavaScript clients usually want
 // the field to be defined after they decode the JSON.)
-// MakeNonNil takes a pointer to a Go data structure
-// (currently only a slice or a map) and makes sure it's non-nil for
-// JSON serialization. (In particular, JavaScript clients usually want
-// the field to be defined after they decode the JSON.)
+//
+// Deprecated: use NonNilSliceForJSON or NonNilMapForJSON instead.
 func NonNil(ptr interface{}) {
 	if ptr == nil {
 		panic("nil interface")
@@ -51,3 +49,23 @@ func NonNil(ptr interface{}) {
 		rv.Set(reflect.MakeMap(rv.Type()))
 	}
 }
+
+// NonNilSliceForJSON makes sure that *slicePtr is non-nil so it will
+// won't be omitted from JSON serialization and possibly confuse JavaScript
+// clients expecting it to be preesnt.
+func NonNilSliceForJSON[T any, S ~[]T](slicePtr *S) {
+	if *slicePtr != nil {
+		return
+	}
+	*slicePtr = make([]T, 0)
+}
+
+// NonNilMapForJSON makes sure that *slicePtr is non-nil so it will
+// won't be omitted from JSON serialization and possibly confuse JavaScript
+// clients expecting it to be preesnt.
+func NonNilMapForJSON[K comparable, V any, M ~map[K]V](mapPtr *M) {
+	if *mapPtr != nil {
+		return
+	}
+	*mapPtr = make(M)
+}

util/mak/mak_test.go

@@ -69,3 +69,21 @@ func TestNonNil(t *testing.T) {
 		t.Error("map still nil")
 	}
 }
+
+func TestNonNilMapForJSON(t *testing.T) {
+	type M map[string]int
+	var m M
+	NonNilMapForJSON(&m)
+	if m == nil {
+		t.Fatal("still nil")
+	}
+}
+
+func TestNonNilSliceForJSON(t *testing.T) {
+	type S []int
+	var s S
+	NonNilSliceForJSON(&s)
+	if s == nil {
+		t.Fatal("still nil")
+	}
+}

wgengine/magicsock/debughttp.go

@@ -172,6 +172,11 @@ func printEndpointHTML(w io.Writer, ep *endpoint) {
 				break
 			}
 			pos := (int(s.recentPong) - i) % len(s.recentPongs)
+			// If s.recentPongs wraps around pos will be negative, so start
+			// again from the end of the slice.
+			if pos < 0 {
+				pos += len(s.recentPongs)
+			}
 			pr := s.recentPongs[pos]
 			fmt.Fprintf(w, "<li>pong %v ago: in %v, from %v src %v</li>\n",
 				fmtMono(pr.pongAt), pr.latency.Round(time.Millisecond/10),

wgengine/magicsock/magicsock.go

@@ -262,8 +262,8 @@ type Conn struct {
 	// pconn4 and pconn6 are the underlying UDP sockets used to
 	// send/receive packets for wireguard and other magicsock
 	// protocols.
-	pconn4 *RebindingUDPConn
-	pconn6 *RebindingUDPConn
+	pconn4 RebindingUDPConn
+	pconn6 RebindingUDPConn
 
 	// closeDisco4 and closeDisco6 are io.Closers to shut down the raw
 	// disco packet receivers. If nil, no raw disco receiver is
@@ -570,7 +570,7 @@ func NewConn(opts Options) (*Conn, error) {
 	}
 	c.linkMon = opts.LinkMonitor
 
-	if err := c.initialBind(); err != nil {
+	if err := c.rebind(keepCurrentPort); err != nil {
 		return nil, err
 	}
 
@@ -578,15 +578,12 @@ func NewConn(opts Options) (*Conn, error) {
 	c.donec = c.connCtx.Done()
 	c.netChecker = &netcheck.Client{
 		Logf:                logger.WithPrefix(c.logf, "netcheck: "),
-		GetSTUNConn4:        func() netcheck.STUNConn { return c.pconn4 },
+		GetSTUNConn4:        func() netcheck.STUNConn { return &c.pconn4 },
+		GetSTUNConn6:        func() netcheck.STUNConn { return &c.pconn6 },
 		SkipExternalNetwork: inTest(),
 		PortMapper:          c.portMapper,
 	}
 
-	if c.pconn6 != nil {
-		c.netChecker.GetSTUNConn6 = func() netcheck.STUNConn { return c.pconn6 }
-	}
-
 	c.ignoreSTUNPackets()
 
 	if d4, err := c.listenRawDisco("ip4"); err == nil {
@@ -1240,10 +1237,6 @@ func (c *Conn) sendUDPStd(addr netip.AddrPort, b []byte) (sent bool, err error)
 			return false, nil
 		}
 	case addr.Addr().Is6():
-		if c.pconn6 == nil {
-			// ignore IPv6 dest if we don't have an IPv6 address.
-			return false, nil
-		}
 		_, err = c.pconn6.WriteToUDPAddrPort(b, addr)
 		if err != nil && (c.noV6.Load() || neterror.TreatAsLostUDP(err)) {
 			return false, nil
@@ -2660,12 +2653,8 @@ func (c *connBind) Close() error {
 	}
 	c.closed = true
 	// Unblock all outstanding receives.
-	if c.pconn4 != nil {
-		c.pconn4.Close()
-	}
-	if c.pconn6 != nil {
-		c.pconn6.Close()
-	}
+	c.pconn4.Close()
+	c.pconn6.Close()
 	if c.closeDisco4 != nil {
 		c.closeDisco4.Close()
 	}
@@ -2710,12 +2699,8 @@ func (c *Conn) Close() error {
 	c.closeAllDerpLocked("conn-close")
 	// Ignore errors from c.pconnN.Close.
 	// They will frequently have been closed already by a call to connBind.Close.
-	if c.pconn6 != nil {
-		c.pconn6.Close()
-	}
-	if c.pconn4 != nil {
-		c.pconn4.Close()
-	}
+	c.pconn6.Close()
+	c.pconn4.Close()
 
 	// Wait on goroutines updating right at the end, once everything is
 	// already closed. We want everything else in the Conn to be
@@ -2821,20 +2806,6 @@ func (c *Conn) ReSTUN(why string) {
 	}
 }
 
-func (c *Conn) initialBind() error {
-	if runtime.GOOS == "js" {
-		return nil
-	}
-	if err := c.bindSocket(&c.pconn4, "udp4", keepCurrentPort); err != nil {
-		return fmt.Errorf("magicsock: initialBind IPv4 failed: %w", err)
-	}
-	c.portMapper.SetLocalPort(c.LocalPort())
-	if err := c.bindSocket(&c.pconn6, "udp6", keepCurrentPort); err != nil {
-		c.logf("magicsock: ignoring IPv6 bind failure: %v", err)
-	}
-	return nil
-}
-
 // listenPacket opens a packet listener.
 // The network must be "udp4" or "udp6".
 func (c *Conn) listenPacket(network string, port uint16) (nettype.PacketConn, error) {
@@ -2852,17 +2823,17 @@ func (c *Conn) listenPacket(network string, port uint16) (nettype.PacketConn, er
 // The caller is responsible for informing the portMapper of any changes.
 // If curPortFate is set to dropCurrentPort, no attempt is made to reuse
 // the current port.
-func (c *Conn) bindSocket(rucPtr **RebindingUDPConn, network string, curPortFate currentPortFate) error {
-	if *rucPtr == nil {
-		*rucPtr = new(RebindingUDPConn)
-	}
-	ruc := *rucPtr
-
+func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate currentPortFate) error {
 	// Hold the ruc lock the entire time, so that the close+bind is atomic
 	// from the perspective of ruc receive functions.
 	ruc.mu.Lock()
 	defer ruc.mu.Unlock()
 
+	if runtime.GOOS == "js" {
+		ruc.setConnLocked(newBlockForeverConn())
+		return nil
+	}
+
 	if debugAlwaysDERP {
 		c.logf("disabled %v per TS_DEBUG_ALWAYS_USE_DERP", network)
 		ruc.setConnLocked(newBlockForeverConn())
@@ -2927,16 +2898,13 @@ const (
 // rebind closes and re-binds the UDP sockets.
 // We consider it successful if we manage to bind the IPv4 socket.
 func (c *Conn) rebind(curPortFate currentPortFate) error {
-	if runtime.GOOS == "js" {
-		return nil
+	if err := c.bindSocket(&c.pconn6, "udp6", curPortFate); err != nil {
+		c.logf("magicsock: Rebind ignoring IPv6 bind failure: %v", err)
 	}
 	if err := c.bindSocket(&c.pconn4, "udp4", curPortFate); err != nil {
 		return fmt.Errorf("magicsock: Rebind IPv4 failed: %w", err)
 	}
 	c.portMapper.SetLocalPort(c.LocalPort())
-	if err := c.bindSocket(&c.pconn6, "udp6", curPortFate); err != nil {
-		c.logf("magicsock: Rebind ignoring IPv6 bind failure: %v", err)
-	}
 	return nil
 }
 
@@ -3023,11 +2991,13 @@ type RebindingUDPConn struct {
 
 	mu    sync.Mutex // held while changing pconn (and pconnAtomic)
 	pconn nettype.PacketConn
+	port  uint16
 }
 
 func (c *RebindingUDPConn) setConnLocked(p nettype.PacketConn) {
 	c.pconn = p
 	c.pconnAtomic.Store(p)
+	c.port = uint16(c.localAddrLocked().Port)
 }
 
 // currentConn returns c's current pconn, acquiring c.mu in the process.
@@ -3089,6 +3059,12 @@ func (c *RebindingUDPConn) ReadFromNetaddr(b []byte) (n int, ipp netip.AddrPort,
 	}
 }
 
+func (c *RebindingUDPConn) Port() uint16 {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.port
+}
+
 func (c *RebindingUDPConn) LocalAddr() *net.UDPAddr {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -3113,6 +3089,7 @@ func (c *RebindingUDPConn) closeLocked() error {
 	if c.pconn == nil {
 		return errNilPConn
 	}
+	c.port = 0
 	return c.pconn.Close()
 }
 

wgengine/magicsock/magicsock_linux.go

@@ -18,6 +18,7 @@ import (
 	"golang.org/x/net/bpf"
 	"golang.org/x/sys/unix"
 	"tailscale.com/envknob"
+	"tailscale.com/net/netns"
 	"tailscale.com/types/key"
 )
 
@@ -128,6 +129,11 @@ func (c *Conn) listenRawDisco(family string) (io.Closer, error) {
 		return nil, errors.New("raw disco listening disabled by debug flag")
 	}
 
+	// https://github.com/tailscale/tailscale/issues/5607
+	if !netns.UseSocketMark() {
+		return nil, errors.New("raw disco listening disabled, SO_MARK unavailable")
+	}
+
 	var (
 		network  string
 		addr     string
@@ -195,11 +201,11 @@ func (c *Conn) listenRawDisco(family string) (io.Closer, error) {
 	}
 	pc.SetReadDeadline(time.Time{})
 
-	go c.receiveDisco(pc)
+	go c.receiveDisco(pc, family == "ip6")
 	return pc, nil
 }
 
-func (c *Conn) receiveDisco(pc net.PacketConn) {
+func (c *Conn) receiveDisco(pc net.PacketConn, isIPV6 bool) {
 	var buf [1500]byte
 	for {
 		n, src, err := pc.ReadFrom(buf[:])
@@ -213,6 +219,30 @@ func (c *Conn) receiveDisco(pc net.PacketConn) {
 			// Too small to be a valid UDP datagram, drop.
 			continue
 		}
+
+		dstPort := binary.BigEndian.Uint16(buf[2:4])
+		if dstPort == 0 {
+			c.logf("[unexpected] disco raw: received packet for port 0")
+		}
+
+		var acceptPort uint16
+		if isIPV6 {
+			acceptPort = c.pconn6.Port()
+		} else {
+			acceptPort = c.pconn4.Port()
+		}
+		if acceptPort == 0 {
+			// This should only typically happen if the receiving address family
+			// was recently disabled.
+			c.logf("[v1] disco raw: dropping packet for port %d as acceptPort=0", dstPort)
+			continue
+		}
+
+		if dstPort != acceptPort {
+			c.logf("[v1] disco raw: dropping packet for port %d", dstPort)
+			continue
+		}
+
 		srcIP, ok := netip.AddrFromSlice(src.(*net.IPAddr).IP)
 		if !ok {
 			c.logf("[unexpected] PacketConn.ReadFrom returned not-an-IP %v in from", src)

wgengine/netstack/netstack.go

@@ -743,47 +743,64 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 			ns.removeSubnetAddress(dialIP)
 		}
 	}()
+
 	var wq waiter.Queue
-	ep, err := r.CreateEndpoint(&wq)
-	if err != nil {
-		ns.logf("CreateEndpoint error for %s: %v", stringifyTEI(reqDetails), err)
-		r.Complete(true) // sends a RST
-		return
+
+	// We can't actually create the endpoint or complete the inbound
+	// request until we're sure that the connection can be handled by this
+	// endpoint. This function sets up the TCP connection and should be
+	// called immediately before a connection is handled.
+	createConn := func() *gonet.TCPConn {
+		ep, err := r.CreateEndpoint(&wq)
+		if err != nil {
+			ns.logf("CreateEndpoint error for %s: %v", stringifyTEI(reqDetails), err)
+			r.Complete(true) // sends a RST
+			return nil
+		}
+		r.Complete(false)
+
+		// SetKeepAlive so that idle connections to peers that have forgotten about
+		// the connection or gone completely offline eventually time out.
+		// Applications might be setting this on a forwarded connection, but from
+		// userspace we can not see those, so the best we can do is to always
+		// perform them with conservative timing.
+		// TODO(tailscale/tailscale#4522): Netstack defaults match the Linux
+		// defaults, and results in a little over two hours before the socket would
+		// be closed due to keepalive. A shorter default might be better, or seeking
+		// a default from the host IP stack. This also might be a useful
+		// user-tunable, as in userspace mode this can have broad implications such
+		// as lingering connections to fork style daemons. On the other side of the
+		// fence, the long duration timers are low impact values for battery powered
+		// peers.
+		ep.SocketOptions().SetKeepAlive(true)
+
+		// The ForwarderRequest.CreateEndpoint above asynchronously
+		// starts the TCP handshake. Note that the gonet.TCPConn
+		// methods c.RemoteAddr() and c.LocalAddr() will return nil
+		// until the handshake actually completes. But we have the
+		// remote address in reqDetails instead, so we don't use
+		// gonet.TCPConn.RemoteAddr. The byte copies in both
+		// directions to/from the gonet.TCPConn in forwardTCP will
+		// block until the TCP handshake is complete.
+		return gonet.NewTCPConn(&wq, ep)
 	}
-	r.Complete(false)
-
-	// SetKeepAlive so that idle connections to peers that have forgotten about
-	// the connection or gone completely offline eventually time out.
-	// Applications might be setting this on a forwarded connection, but from
-	// userspace we can not see those, so the best we can do is to always
-	// perform them with conservative timing.
-	// TODO(tailscale/tailscale#4522): Netstack defaults match the Linux
-	// defaults, and results in a little over two hours before the socket would
-	// be closed due to keepalive. A shorter default might be better, or seeking
-	// a default from the host IP stack. This also might be a useful
-	// user-tunable, as in userspace mode this can have broad implications such
-	// as lingering connections to fork style daemons. On the other side of the
-	// fence, the long duration timers are low impact values for battery powered
-	// peers.
-	ep.SocketOptions().SetKeepAlive(true)
-
-	// The ForwarderRequest.CreateEndpoint above asynchronously
-	// starts the TCP handshake. Note that the gonet.TCPConn
-	// methods c.RemoteAddr() and c.LocalAddr() will return nil
-	// until the handshake actually completes. But we have the
-	// remote address in reqDetails instead, so we don't use
-	// gonet.TCPConn.RemoteAddr. The byte copies in both
-	// directions to/from the gonet.TCPConn in forwardTCP will
-	// block until the TCP handshake is complete.
-	c := gonet.NewTCPConn(&wq, ep)
 
+	// DNS
 	if reqDetails.LocalPort == 53 && (dialIP == magicDNSIP || dialIP == magicDNSIPv6) {
+		c := createConn()
+		if c == nil {
+			return
+		}
 		go ns.dns.HandleTCPConn(c, netip.AddrPortFrom(clientRemoteIP, reqDetails.RemotePort))
 		return
 	}
 
 	if ns.lb != nil {
 		if reqDetails.LocalPort == 22 && ns.processSSH() && ns.isLocalIP(dialIP) {
+			c := createConn()
+			if c == nil {
+				return
+			}
 			if err := ns.lb.HandleSSHConn(c); err != nil {
 				ns.logf("ssh error: %v", err)
 			}
@@ -791,6 +808,11 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 		}
 		if port, ok := ns.lb.GetPeerAPIPort(dialIP); ok {
 			if reqDetails.LocalPort == port && ns.isLocalIP(dialIP) {
+				c := createConn()
+				if c == nil {
+					return
+				}
+
 				src := netip.AddrPortFrom(clientRemoteIP, reqDetails.RemotePort)
 				dst := netip.AddrPortFrom(dialIP, port)
 				ns.lb.ServePeerAPIConnection(src, dst, c)
@@ -798,12 +820,20 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 			}
 		}
 		if reqDetails.LocalPort == 80 && (dialIP == magicDNSIP || dialIP == magicDNSIPv6) {
+			c := createConn()
+			if c == nil {
+				return
+			}
 			ns.lb.HandleQuad100Port80Conn(c)
 			return
 		}
 	}
 
 	if ns.ForwardTCPIn != nil {
+		c := createConn()
+		if c == nil {
+			return
+		}
 		ns.ForwardTCPIn(c, reqDetails.LocalPort)
 		return
 	}
@@ -811,11 +841,13 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 		dialIP = netaddr.IPv4(127, 0, 0, 1)
 	}
 	dialAddr := netip.AddrPortFrom(dialIP, uint16(reqDetails.LocalPort))
-	ns.forwardTCP(c, clientRemoteIP, &wq, dialAddr)
+
+	if !ns.forwardTCP(createConn, clientRemoteIP, &wq, dialAddr) {
+		r.Complete(true) // sends a RST
+	}
 }
 
-func (ns *Impl) forwardTCP(client *gonet.TCPConn, clientRemoteIP netip.Addr, wq *waiter.Queue, dialAddr netip.AddrPort) {
-	defer client.Close()
+func (ns *Impl) forwardTCP(getClient func() *gonet.TCPConn, clientRemoteIP netip.Addr, wq *waiter.Queue, dialAddr netip.AddrPort) (handled bool) {
 	dialAddrStr := dialAddr.String()
 	if debugNetstack {
 		ns.logf("[v2] netstack: forwarding incoming connection to %s", dialAddrStr)
@@ -823,6 +855,7 @@ func (ns *Impl) forwardTCP(client *gonet.TCPConn, clientRemoteIP netip.Addr, wq
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+
 	waitEntry, notifyCh := waiter.NewChannelEntry(waiter.EventHUp) // TODO(bradfitz): right EventMask?
 	wq.EventRegister(&waitEntry)
 	defer wq.EventUnregister(&waitEntry)
@@ -840,13 +873,29 @@ func (ns *Impl) forwardTCP(client *gonet.TCPConn, clientRemoteIP netip.Addr, wq
 		}
 		cancel()
 	}()
+
+	// Attempt to dial the outbound connection before we accept the inbound one.
 	var stdDialer net.Dialer
 	server, err := stdDialer.DialContext(ctx, "tcp", dialAddrStr)
 	if err != nil {
-		ns.logf("netstack: could not connect to local server at %s: %v", dialAddrStr, err)
+		ns.logf("netstack: could not connect to local server at %s: %v", dialAddr.String(), err)
 		return
 	}
 	defer server.Close()
+
+	// If we get here, either the getClient call below will succeed and
+	// return something we can Close, or it will fail and will properly
+	// respond to the client with a RST. Either way, the caller no longer
+	// needs to clean up the client connection.
+	handled = true
+
+	// We dialed the connection; we can complete the client's TCP handshake.
+	client := getClient()
+	if client == nil {
+		return
+	}
+	defer client.Close()
+
 	backendLocalAddr := server.LocalAddr().(*net.TCPAddr)
 	backendLocalIPPort := netaddr.Unmap(backendLocalAddr.AddrPort())
 	ns.e.RegisterIPPortIdentity(backendLocalIPPort, clientRemoteIP)
@@ -865,6 +914,7 @@ func (ns *Impl) forwardTCP(client *gonet.TCPConn, clientRemoteIP netip.Addr, wq
 		ns.logf("proxy connection closed with error: %v", err)
 	}
 	ns.logf("[v2] netstack: forwarder connection to %s closed", dialAddrStr)
+	return
 }
 
 func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {

wgengine/router/router_linux.go

@@ -50,13 +50,21 @@ const (
 // Empirically, most of the documentation on packet marks on the
 // internet gives the impression that the marks are 16 bits
 // wide. Based on this, we theorize that the upper two bytes are
-// relatively unused in the wild, and so we consume bits starting at
-// the 17th.
+// relatively unused in the wild, and so we consume bits 16:23 (the
+// third byte).
 //
 // The constants are in the iptables/iproute2 string format for
 // matching and setting the bits, so they can be directly embedded in
 // commands.
 const (
+	// The mask for reading/writing the 'firewall mask' bits on a packet.
+	// See the comment on the const block on why we only use the third byte.
+	//
+	// We claim bits 16:23 entirely. For now we only use the lower four
+	// bits, leaving the higher 4 bits for future use.
+	tailscaleFwmarkMask    = "0xff0000"
+	tailscaleFwmarkMaskNum = 0xff0000
+
 	// Packet is from Tailscale and to a subnet route destination, so
 	// is allowed to be routed through this machine.
 	tailscaleSubnetRouteMark = "0x40000"
@@ -104,6 +112,10 @@ type linuxRouter struct {
 	ipRuleAvailable bool // whether kernel was built with IP_MULTIPLE_TABLES
 	v6Available     bool
 	v6NATAvailable  bool
+	fwmaskWorks     bool // whether we can use 'ip rule...fwmark <mark>/<mask>'
+
+	// ipPolicyPrefBase is the base priority at which ip rules are installed.
+	ipPolicyPrefBase int
 
 	ipt4 netfilterRunner
 	ipt6 netfilterRunner
@@ -163,6 +175,7 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monit
 		cmd:  cmd,
 
 		ipRuleFixLimiter: rate.NewLimiter(rate.Every(5*time.Second), 10),
+		ipPolicyPrefBase: 5200,
 	}
 	if r.useIPCommand() {
 		r.ipRuleAvailable = (cmd.run("ip", "rule") == nil)
@@ -176,9 +189,126 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monit
 		}
 	}
 
+	// To be a good denizen of the 4-byte 'fwmark' bitspace on every packet, we try to
+	// only use the third byte. However, support for masking to part of the fwmark bitspace
+	// was only added to busybox in 1.33.0. As such, we want to detect older versions and
+	// not issue such a stanza.
+	var err error
+	if r.fwmaskWorks, err = ipCmdSupportsFwmask(); err != nil {
+		r.logf("failed to determine ip command fwmask support: %v", err)
+	}
+	if r.fwmaskWorks {
+		r.logf("[v1] ip command supports fwmark masks")
+	} else {
+		r.logf("[v1] ip command does NOT support fwmark masks")
+	}
+
+	// A common installation of OpenWRT involves use of the 'mwan3' package.
+	// This package installs ip-tables rules like:
+	//  -A mwan3_fallback_policy -m mark --mark 0x0/0x3f00 -j MARK --set-xmark 0x100/0x3f00
+	//
+	// which coupled with an ip rule:
+	//  2001: from all fwmark 0x100/0x3f00 lookup 1
+	//
+	// has the effect of gobbling tailscale packets, because tailscale by default installs
+	// its policy routing rules at priority 52xx.
+	//
+	// As such, if we are running on openWRT, detect a mwan3 config, AND detect a rule
+	// with a preference 2001 (corresponding to the first interface wman3 manages), we
+	// shift the priority of our policies to 13xx. This effectively puts us betwen mwan3's
+	// permit-by-src-ip rules and mwan3 lookup of its own routing table which would drop
+	// the packet.
+	isMWAN3, err := checkOpenWRTUsingMWAN3()
+	if err != nil {
+		r.logf("error checking mwan3 installation: %v", err)
+	} else if isMWAN3 {
+		r.ipPolicyPrefBase = 1300
+		r.logf("mwan3 on openWRT detected, switching policy base priority to 1300")
+	}
+
 	return r, nil
 }
 
+// ipCmdSupportsFwmask returns true if the system 'ip' binary supports using a
+// fwmark stanza with a mask specified. To our knowledge, everything except busybox
+// pre-1.33 supports this.
+func ipCmdSupportsFwmask() (bool, error) {
+	ipPath, err := exec.LookPath("ip")
+	if err != nil {
+		return false, fmt.Errorf("lookpath: %v", err)
+	}
+	stat, err := os.Lstat(ipPath)
+	if err != nil {
+		return false, fmt.Errorf("lstat: %v", err)
+	}
+	if stat.Mode()&os.ModeSymlink == 0 {
+		// Not a symlink, so can't be busybox. Must be regular ip utility.
+		return true, nil
+	}
+
+	linkDest, err := os.Readlink(ipPath)
+	if err != nil {
+		return false, err
+	}
+	if !strings.Contains(strings.ToLower(linkDest), "busybox") {
+		// Not busybox, presumably supports fwmark masks.
+		return true, nil
+	}
+
+	// If we got this far, the ip utility is a busybox version with an
+	// unknown version.
+	// We run `ip --version` and look for the busybox banner (which
+	// is a stable 'BusyBox vX.Y.Z (<builddate>)' string) to determine
+	// the version.
+	out, err := exec.Command("ip", "--version").CombinedOutput()
+	if err != nil {
+		return false, err
+	}
+	major, minor, _, err := busyboxParseVersion(string(out))
+	if err != nil {
+		return false, nil
+	}
+
+	// Support for masks added in 1.33.0.
+	switch {
+	case major > 1:
+		return true, nil
+	case major == 1 && minor >= 33:
+		return true, nil
+	default:
+		return false, nil
+	}
+}
+
+func busyboxParseVersion(output string) (major, minor, patch int, err error) {
+	bannerStart := strings.Index(output, "BusyBox v")
+	if bannerStart < 0 {
+		return 0, 0, 0, errors.New("missing BusyBox banner")
+	}
+	bannerEnd := bannerStart + len("BusyBox v")
+
+	end := strings.Index(output[bannerEnd:], " ")
+	if end < 0 {
+		return 0, 0, 0, errors.New("missing end delimiter")
+	}
+
+	elements := strings.Split(output[bannerEnd:bannerEnd+end], ".")
+	if len(elements) < 3 {
+		return 0, 0, 0, fmt.Errorf("expected 3 version elements, got %d", len(elements))
+	}
+
+	if major, err = strconv.Atoi(elements[0]); err != nil {
+		return 0, 0, 0, fmt.Errorf("parsing major: %v", err)
+	}
+	if minor, err = strconv.Atoi(elements[1]); err != nil {
+		return 0, 0, 0, fmt.Errorf("parsing minor: %v", err)
+	}
+	if patch, err = strconv.Atoi(elements[2]); err != nil {
+		return 0, 0, 0, fmt.Errorf("parsing patch: %v", err)
+	}
+	return major, minor, patch, nil
+}
+
 func useAmbientCaps() bool {
 	if distro.Get() != distro.Synology {
 		return false
@@ -219,7 +349,7 @@ func (r *linuxRouter) useIPCommand() bool {
 // about the priority number. We could just do this in response to any netlink
 // change. Filtering by known priority ranges cuts back on some logspam.
 func (r *linuxRouter) onIPRuleDeleted(table uint8, priority uint32) {
-	if priority < 5200 || priority >= 5300 {
+	if int(priority) < r.ipPolicyPrefBase || int(priority) >= (r.ipPolicyPrefBase+100) {
 		// Not our rule.
 		return
 	}
@@ -870,6 +1000,8 @@ var (
 )
 
 // ipRules are the policy routing rules that Tailscale uses.
+// The priority is the value represented here added to r.ipPolicyPrefBase,
+// which is usually 5200.
 //
 // NOTE(apenwarr): We leave spaces between each pref number.
 // This is so the sysadmin can override by inserting rules in
@@ -886,14 +1018,14 @@ var ipRules = []netlink.Rule{
 	// Packets from us, tagged with our fwmark, first try the kernel's
 	// main routing table.
 	{
-		Priority: 5210,
+		Priority: 10,
 		Mark:     tailscaleBypassMarkNum,
 		Table:    mainRouteTable.num,
 	},
 	// ...and then we try the 'default' table, for correctness,
 	// even though it's been empty on every Linux system I've ever seen.
 	{
-		Priority: 5230,
+		Priority: 30,
 		Mark:     tailscaleBypassMarkNum,
 		Table:    defaultRouteTable.num,
 	},
@@ -901,7 +1033,7 @@ var ipRules = []netlink.Rule{
 	// then packets from us should be aborted rather than falling through
 	// to the tailscale routes, because that would create routing loops.
 	{
-		Priority: 5250,
+		Priority: 50,
 		Mark:     tailscaleBypassMarkNum,
 		Type:     unix.RTN_UNREACHABLE,
 	},
@@ -911,7 +1043,7 @@ var ipRules = []netlink.Rule{
 	// it takes precedence over all the others, ie. VPN routes always
 	// beat non-VPN routes.
 	{
-		Priority: 5270,
+		Priority: 70,
 		Table:    tailscaleRouteTable.num,
 	},
 	// If that didn't match, then non-fwmark packets fall through to the
@@ -928,14 +1060,18 @@ func (r *linuxRouter) justAddIPRules() error {
 	}
 	var errAcc error
 	for _, family := range r.addrFamilies() {
+
 		for _, ru := range ipRules {
 			// Note: r is a value type here; safe to mutate it.
 			ru.Family = family.netlinkInt()
-			ru.Mask = -1
+			if ru.Mark != 0 {
+				ru.Mask = tailscaleFwmarkMaskNum
+			}
 			ru.Goto = -1
 			ru.SuppressIfgroup = -1
 			ru.SuppressPrefixlen = -1
 			ru.Flow = -1
+			ru.Priority += r.ipPolicyPrefBase
 
 			err := netlink.RuleAdd(&ru)
 			if errors.Is(err, errEEXIST) {
@@ -954,19 +1090,23 @@ func (r *linuxRouter) addIPRulesWithIPCommand() error {
 	rg := newRunGroup(nil, r.cmd)
 
 	for _, family := range r.addrFamilies() {
-		for _, r := range ipRules {
+		for _, rule := range ipRules {
 			args := []string{
 				"ip", family.dashArg(),
 				"rule", "add",
-				"pref", strconv.Itoa(r.Priority),
+				"pref", strconv.Itoa(rule.Priority + r.ipPolicyPrefBase),
 			}
-			if r.Mark != 0 {
-				args = append(args, "fwmark", fmt.Sprintf("0x%x", r.Mark))
+			if rule.Mark != 0 {
+				if r.fwmaskWorks {
+					args = append(args, "fwmark", fmt.Sprintf("0x%x/%s", rule.Mark, tailscaleFwmarkMask))
+				} else {
+					args = append(args, "fwmark", fmt.Sprintf("0x%x", rule.Mark))
+				}
 			}
-			if r.Table != 0 {
-				args = append(args, "table", mustRouteTable(r.Table).ipCmdArg())
+			if rule.Table != 0 {
+				args = append(args, "table", mustRouteTable(rule.Table).ipCmdArg())
 			}
-			if r.Type == unix.RTN_UNREACHABLE {
+			if rule.Type == unix.RTN_UNREACHABLE {
 				args = append(args, "type", "unreachable")
 			}
 			rg.Run(args...)
@@ -1011,6 +1151,7 @@ func (r *linuxRouter) delIPRules() error {
 			ru.Goto = -1
 			ru.SuppressIfgroup = -1
 			ru.SuppressPrefixlen = -1
+			ru.Priority += r.ipPolicyPrefBase
 
 			err := netlink.RuleDel(&ru)
 			if errors.Is(err, errENOENT) {
@@ -1040,14 +1181,14 @@ func (r *linuxRouter) delIPRulesWithIPCommand() error {
 		// That leaves us some flexibility to change these values in later
 		// versions without having ongoing hacks for every possible
 		// combination.
-		for _, r := range ipRules {
+		for _, rule := range ipRules {
 			args := []string{
 				"ip", family.dashArg(),
 				"rule", "del",
-				"pref", strconv.Itoa(r.Priority),
+				"pref", strconv.Itoa(rule.Priority + r.ipPolicyPrefBase),
 			}
-			if r.Table != 0 {
-				args = append(args, "table", mustRouteTable(r.Table).ipCmdArg())
+			if rule.Table != 0 {
+				args = append(args, "table", mustRouteTable(rule.Table).ipCmdArg())
 			} else {
 				args = append(args, "type", "unreachable")
 			}
@@ -1141,11 +1282,11 @@ func (r *linuxRouter) addNetfilterBase4() error {
 	// POSTROUTING. So instead, we match on the inbound interface in
 	// filter/FORWARD, and set a packet mark that nat/POSTROUTING can
 	// use to effectively run that same test again.
-	args = []string{"-i", r.tunname, "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark}
+	args = []string{"-i", r.tunname, "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark + "/" + tailscaleFwmarkMask}
 	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
 		return fmt.Errorf("adding %v in v4/filter/ts-forward: %w", args, err)
 	}
-	args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "ACCEPT"}
+	args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark + "/" + tailscaleFwmarkMask, "-j", "ACCEPT"}
 	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
 		return fmt.Errorf("adding %v in v4/filter/ts-forward: %w", args, err)
 	}
@@ -1167,11 +1308,11 @@ func (r *linuxRouter) addNetfilterBase6() error {
 	// TODO: only allow traffic from Tailscale's ULA range to come
 	// from tailscale0.
 
-	args := []string{"-i", r.tunname, "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark}
+	args := []string{"-i", r.tunname, "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark + "/" + tailscaleFwmarkMask}
 	if err := r.ipt6.Append("filter", "ts-forward", args...); err != nil {
 		return fmt.Errorf("adding %v in v6/filter/ts-forward: %w", args, err)
 	}
-	args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "ACCEPT"}
+	args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark + "/" + tailscaleFwmarkMask, "-j", "ACCEPT"}
 	if err := r.ipt6.Append("filter", "ts-forward", args...); err != nil {
 		return fmt.Errorf("adding %v in v6/filter/ts-forward: %w", args, err)
 	}
@@ -1343,7 +1484,7 @@ func (r *linuxRouter) addSNATRule() error {
 		return nil
 	}
 
-	args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "MASQUERADE"}
+	args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark + "/" + tailscaleFwmarkMask, "-j", "MASQUERADE"}
 	if err := r.ipt4.Append("nat", "ts-postrouting", args...); err != nil {
 		return fmt.Errorf("adding %v in v4/nat/ts-postrouting: %w", args, err)
 	}
@@ -1362,7 +1503,7 @@ func (r *linuxRouter) delSNATRule() error {
 		return nil
 	}
 
-	args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "MASQUERADE"}
+	args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark + "/" + tailscaleFwmarkMask, "-j", "MASQUERADE"}
 	if err := r.ipt4.Delete("nat", "ts-postrouting", args...); err != nil {
 		return fmt.Errorf("deleting %v in v4/nat/ts-postrouting: %w", args, err)
 	}
@@ -1548,6 +1689,39 @@ func checkIPRuleSupportsV6(logf logger.Logf) error {
 	return netlink.RuleAdd(rule)
 }
 
+// Checks if the running openWRT system is using mwan3, based on the heuristic
+// of the config file being present as well as a policy rule with a specific
+// priority (2000 + 1 - first interface mwan3 manages) and non-zero mark.
+func checkOpenWRTUsingMWAN3() (bool, error) {
+	if distro.Get() != distro.OpenWrt {
+		return false, nil
+	}
+
+	if _, err := os.Stat("/etc/config/mwan3"); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	rules, err := netlink.RuleList(netlink.FAMILY_V4)
+	if err != nil {
+		return false, err
+	}
+	for _, r := range rules {
+		// We want to match on a rule like this:
+		//    2001:	from all fwmark 0x100/0x3f00 lookup 1
+		//
+		// We dont match on the mask because it can vary, or the
+		// table because I'm not sure if it can vary.
+		if r.Priority == 2001 && r.Mark != 0 {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
 func nlAddrOfPrefix(p netip.Prefix) *netlink.Addr {
 	return &netlink.Addr{
 		IPNet: netipx.PrefixIPNet(p),

wgengine/router/router_linux_test.go

@@ -25,13 +25,13 @@ import (
 
 func TestRouterStates(t *testing.T) {
 	basic := `
-ip rule add -4 pref 5210 fwmark 0x80000 table main
-ip rule add -4 pref 5230 fwmark 0x80000 table default
-ip rule add -4 pref 5250 fwmark 0x80000 type unreachable
+ip rule add -4 pref 5210 fwmark 0x80000/0xff0000 table main
+ip rule add -4 pref 5230 fwmark 0x80000/0xff0000 table default
+ip rule add -4 pref 5250 fwmark 0x80000/0xff0000 type unreachable
 ip rule add -4 pref 5270 table 52
-ip rule add -6 pref 5210 fwmark 0x80000 table main
-ip rule add -6 pref 5230 fwmark 0x80000 table default
-ip rule add -6 pref 5250 fwmark 0x80000 type unreachable
+ip rule add -6 pref 5210 fwmark 0x80000/0xff0000 table main
+ip rule add -6 pref 5230 fwmark 0x80000/0xff0000 table default
+ip rule add -6 pref 5250 fwmark 0x80000/0xff0000 type unreachable
 ip rule add -6 pref 5270 table 52
 `
 	states := []struct {
@@ -101,22 +101,22 @@ ip route add 10.0.0.0/8 dev tailscale0 table 52
 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`v4/filter/FORWARD -j ts-forward
 v4/filter/INPUT -j ts-input
-v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
 v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
 v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 v4/nat/POSTROUTING -j ts-postrouting
-v4/nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE
+v4/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE
 v6/filter/FORWARD -j ts-forward
 v6/filter/INPUT -j ts-input
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 v6/nat/POSTROUTING -j ts-postrouting
-v6/nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE
+v6/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE
 `,
 		},
 		{
@@ -133,8 +133,8 @@ ip route add 10.0.0.0/8 dev tailscale0 table 52
 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`v4/filter/FORWARD -j ts-forward
 v4/filter/INPUT -j ts-input
-v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -143,8 +143,8 @@ v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 v4/nat/POSTROUTING -j ts-postrouting
 v6/filter/FORWARD -j ts-forward
 v6/filter/INPUT -j ts-input
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 v6/nat/POSTROUTING -j ts-postrouting
 `,
@@ -166,8 +166,8 @@ ip route add 10.0.0.0/8 dev tailscale0 table 52
 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`v4/filter/FORWARD -j ts-forward
 v4/filter/INPUT -j ts-input
-v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -176,8 +176,8 @@ v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 v4/nat/POSTROUTING -j ts-postrouting
 v6/filter/FORWARD -j ts-forward
 v6/filter/INPUT -j ts-input
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 v6/nat/POSTROUTING -j ts-postrouting
 `,
@@ -196,8 +196,8 @@ ip route add 10.0.0.0/8 dev tailscale0 table 52
 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`v4/filter/FORWARD -j ts-forward
 v4/filter/INPUT -j ts-input
-v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -206,8 +206,8 @@ v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 v4/nat/POSTROUTING -j ts-postrouting
 v6/filter/FORWARD -j ts-forward
 v6/filter/INPUT -j ts-input
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 v6/nat/POSTROUTING -j ts-postrouting
 `,
@@ -225,15 +225,15 @@ up
 ip addr add 100.101.102.104/10 dev tailscale0
 ip route add 10.0.0.0/8 dev tailscale0 table 52
 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
-				`v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+				`v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
 v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
 v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 `,
 		},
@@ -251,8 +251,8 @@ ip route add 10.0.0.0/8 dev tailscale0 table 52
 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`v4/filter/FORWARD -j ts-forward
 v4/filter/INPUT -j ts-input
-v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -261,8 +261,8 @@ v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 v4/nat/POSTROUTING -j ts-postrouting
 v6/filter/FORWARD -j ts-forward
 v6/filter/INPUT -j ts-input
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 v6/nat/POSTROUTING -j ts-postrouting
 `,
@@ -283,8 +283,8 @@ ip route add 100.100.100.100/32 dev tailscale0 table 52
 ip route add throw 10.0.0.0/8 table 52` + basic +
 				`v4/filter/FORWARD -j ts-forward
 v4/filter/INPUT -j ts-input
-v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 v4/filter/ts-forward -o tailscale0 -j ACCEPT
 v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -293,8 +293,8 @@ v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 v4/nat/POSTROUTING -j ts-postrouting
 v6/filter/FORWARD -j ts-forward
 v6/filter/INPUT -j ts-input
-v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
-v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
+v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000
+v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT
 v6/filter/ts-forward -o tailscale0 -j ACCEPT
 v6/nat/POSTROUTING -j ts-postrouting
 `,
@@ -811,3 +811,31 @@ func TestCheckIPRuleSupportsV6(t *testing.T) {
 	// Some machines running our tests might not have IPv6.
 	t.Logf("Got: %v", err)
 }
+
+func TestBusyboxParseVersion(t *testing.T) {
+	input := `BusyBox v1.34.1 (2022-09-01 16:10:29 UTC) multi-call binary.
+BusyBox is copyrighted by many authors between 1998-2015.
+Licensed under GPLv2. See source distribution for detailed
+copyright notices.
+
+Usage: busybox [function [arguments]...]
+   or: busybox --list[-full]
+   or: busybox --show SCRIPT
+   or: busybox --install [-s] [DIR]
+   or: function [arguments]...
+
+	BusyBox is a multi-call binary that combines many common Unix
+	utilities into a single executable.  Most people will create a
+	link to busybox for each function they wish to use and BusyBox
+	will act like whatever it was invoked as.
+`
+
+	v1, v2, v3, err := busyboxParseVersion(input)
+	if err != nil {
+		t.Fatalf("busyboxParseVersion() failed: %v", err)
+	}
+
+	if got, want := fmt.Sprintf("%d.%d.%d", v1, v2, v3), "1.34.1"; got != want {
+		t.Errorf("version = %q, want %q", got, want)
+	}
+}

wgengine/wgint/wgint.go

@@ -22,38 +22,28 @@ var (
 
 func getPeerStatsOffset(name string) uintptr {
 	peerType := reflect.TypeOf(device.Peer{})
-	sf, ok := peerType.FieldByName("stats")
+	field, ok := peerType.FieldByName(name)
 	if !ok {
-		panic("no stats field in device.Peer")
+		panic("no " + name + " field in device.Peer")
 	}
-	if sf.Type.Kind() != reflect.Struct {
-		panic("stats field is not a struct")
+	if s := field.Type.String(); s != "atomic.Int64" && s != "atomic.Uint64" {
+		panic("unexpected type " + s + " of field " + name + " in device.Peer")
 	}
-	base := sf.Offset
-
-	st := sf.Type
-	field, ok := st.FieldByName(name)
-	if !ok {
-		panic("no " + name + " field in device.Peer.stats")
-	}
-	if field.Type.Kind() != reflect.Int64 && field.Type.Kind() != reflect.Uint64 {
-		panic("unexpected kind of " + name + " field in device.Peer.stats")
-	}
-	return base + field.Offset
+	return field.Offset
 }
 
 // PeerLastHandshakeNano returns the last handshake time in nanoseconds since the
 // unix epoch.
 func PeerLastHandshakeNano(peer *device.Peer) int64 {
-	return atomic.LoadInt64((*int64)(unsafe.Add(unsafe.Pointer(peer), offHandshake)))
+	return (*atomic.Int64)(unsafe.Add(unsafe.Pointer(peer), offHandshake)).Load()
 }
 
 // PeerRxBytes returns the number of bytes received from this peer.
 func PeerRxBytes(peer *device.Peer) uint64 {
-	return atomic.LoadUint64((*uint64)(unsafe.Add(unsafe.Pointer(peer), offRxBytes)))
+	return (*atomic.Uint64)(unsafe.Add(unsafe.Pointer(peer), offRxBytes)).Load()
 }
 
 // PeerTxBytes returns the number of bytes sent to this peer.
 func PeerTxBytes(peer *device.Peer) uint64 {
-	return atomic.LoadUint64((*uint64)(unsafe.Add(unsafe.Pointer(peer), offTxBytes)))
+	return (*atomic.Uint64)(unsafe.Add(unsafe.Pointer(peer), offTxBytes)).Load()
 }