net/portmapper: fix UPnP probing, work against all ports

Prior to Tailscale 1.12 it detected UPnP on any port.
Starting with Tailscale 1.11.x, it stopped detecting UPnP on all ports.

Then start plumbing its discovered Location header port number to the
code that was assuming port 5000.

Fixes #2109

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/go_generate.yml

@@ -0,0 +1,34 @@
+name: go generate
+
+on:
+  push:
+    branches:
+      - main
+      - "release-branch/*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16
+
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: check 'go generate' is clean
+        run: |
+          mkdir gentools
+          go build -o gentools/stringer golang.org/x/tools/cmd/stringer
+          PATH="$PATH:$(pwd)/gentools" go generate ./...
+          echo
+          echo
+          git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1)

.github/workflows/xe-experimental-vm-test.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     paths:
       - "tstest/integration/vms/**"
-  push:
-    branches: [ main ]
   release:
     types: [ created ]
 

Dockerfile

@@ -42,8 +42,7 @@ FROM golang:1.16-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
-COPY go.mod .
-COPY go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .

VERSION.txt

@@ -1 +1 @@
-1.11.0
+1.13.0

cmd/addlicense/main.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program addlicense adds a license header to a file.
+// It is intended for use with 'go generate',
+// so it has a slightly weird usage.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+var (
+	year = flag.Int("year", 0, "copyright year")
+	file = flag.String("file", "", "file to modify")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `
+usage: addlicense -year YEAR -file FILE <subcommand args...>
+`[1:])
+
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, `
+addlicense adds a Tailscale license to the beginning of file,
+using year as the copyright year.
+
+It is intended for use with 'go generate', so it also runs a subcommand,
+which presumably creates the file.
+
+Sample usage:
+
+addlicense -year 2021 -file pull_strings.go stringer -type=pull
+`[1:])
+	os.Exit(2)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	cmd := exec.Command(flag.Arg(0), flag.Args()[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	check(err)
+	b, err := os.ReadFile(*file)
+	check(err)
+	f, err := os.OpenFile(*file, os.O_TRUNC|os.O_WRONLY, 0644)
+	check(err)
+	_, err = fmt.Fprintf(f, license, *year)
+	check(err)
+	_, err = f.Write(b)
+	check(err)
+	err = f.Close()
+	check(err)
+}
+
+func check(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+var license = `
+// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+`[1:]

cmd/derper/mesh.go

@@ -9,7 +9,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net"
 	"strings"
+	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -39,6 +41,34 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return err
 	}
 	c.MeshKey = s.MeshKey()
+
+	// For meshed peers within a region, connect via VPC addresses.
+	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var d net.Dialer
+		var r net.Resolver
+		if port == "443" && strings.HasSuffix(host, ".tailscale.com") {
+			base := strings.TrimSuffix(host, ".tailscale.com")
+			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+			defer cancel()
+			vpcHost := base + "-vpc.tailscale.com"
+			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
+			if len(ips) > 0 {
+				vpcAddr := net.JoinHostPort(ips[0].String(), port)
+				c, err := d.DialContext(subCtx, network, vpcAddr)
+				if err == nil {
+					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
+					return c, nil
+				}
+				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+			}
+		}
+		return d.DialContext(ctx, network, addr)
+	})
+
 	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
 	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)

cmd/derpprobe/derpprobe.go

@@ -211,6 +211,13 @@ func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.D
 	}
 	defer toc.Close()
 
+	// Wait a bit for from's node to hear about to existing on the
+	// other node in the region, in the case where the two nodes
+	// are different.
+	if from.Name != to.Name {
+		time.Sleep(100 * time.Millisecond) // pretty arbitrary
+	}
+
 	// Make a random packet
 	pkt := make([]byte, 8)
 	crand.Read(pkt)

cmd/tailscale/cli/status.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/util/dnsname"
 )
 
@@ -193,7 +194,7 @@ func runStatus(ctx context.Context, args []string) error {
 //
 // TODO: have the server report this bool instead.
 func peerActive(ps *ipnstate.PeerStatus) bool {
-	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+	return !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 }
 
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {

cmd/tailscale/cli/up.go

@@ -51,7 +51,14 @@ flag is also used.
 	Exec:    runUp,
 }
 
-var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+func effectiveGOOS() string {
+	if v := os.Getenv("TS_DEBUG_UP_FLAG_GOOS"); v != "" {
+		return v
+	}
+	return runtime.GOOS
+}
+
+var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
@@ -63,13 +70,13 @@ func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
@@ -327,7 +334,7 @@ func runUp(ctx context.Context, args []string) error {
 		}
 	}
 
-	prefs, err := prefsFromUpArgs(upArgs, warnf, st, runtime.GOOS)
+	prefs, err := prefsFromUpArgs(upArgs, warnf, st, effectiveGOOS())
 	if err != nil {
 		fatalf("%s", err)
 	}
@@ -344,7 +351,7 @@ func runUp(ctx context.Context, args []string) error {
 	}
 
 	env := upCheckEnv{
-		goos:          runtime.GOOS,
+		goos:          effectiveGOOS(),
 		user:          os.Getenv("USER"),
 		flagSet:       upFlagSet,
 		upArgs:        upArgs,
@@ -384,7 +391,7 @@ func runUp(ctx context.Context, args []string) error {
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
-				switch runtime.GOOS {
+				switch effectiveGOOS() {
 				case "windows":
 					msg += " (Tailscale service in use by other user?)"
 				default:
@@ -458,7 +465,7 @@ func runUp(ctx context.Context, args []string) error {
 		// Windows service (~tailscaled) is the one that computes the
 		// StateKey based on the connection identity. So for now, just
 		// do as the Windows GUI's always done:
-		if runtime.GOOS == "windows" {
+		if effectiveGOOS() == "windows" {
 			// The Windows service will set this as needed based
 			// on our connection's identity.
 			opts.StateKey = ""

cmd/tailscale/depaware.txt

@@ -49,6 +49,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/derp+

cmd/tailscaled/debug.go

@@ -14,18 +14,23 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/url"
 	"os"
+	"strings"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/portmapper"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -33,6 +38,7 @@ var debugArgs struct {
 	monitor   bool
 	getURL    string
 	derpCheck string
+	portmap   bool
 }
 
 var debugModeFunc = debugMode // so it can be addressable
@@ -40,6 +46,7 @@ var debugModeFunc = debugMode // so it can be addressable
 func debugMode(args []string) error {
 	fs := flag.NewFlagSet("debug", flag.ExitOnError)
 	fs.BoolVar(&debugArgs.monitor, "monitor", false, "If true, run link monitor forever. Precludes all other options.")
+	fs.BoolVar(&debugArgs.portmap, "portmap", false, "If true, run portmap debugging. Precludes all other options.")
 	fs.StringVar(&debugArgs.getURL, "get-url", "", "If non-empty, fetch provided URL.")
 	fs.StringVar(&debugArgs.derpCheck, "derp", "", "if non-empty, test a DERP ping via named region code")
 	if err := fs.Parse(args); err != nil {
@@ -55,6 +62,9 @@ func debugMode(args []string) error {
 	if debugArgs.monitor {
 		return runMonitor(ctx)
 	}
+	if debugArgs.portmap {
+		return debugPortmap(ctx)
+	}
 	if debugArgs.getURL != "" {
 		return getURL(ctx, debugArgs.getURL)
 	}
@@ -191,3 +201,83 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 	log.Printf("ok")
 	return err
 }
+
+func debugPortmap(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	portmapper.VerboseLogs = true
+
+	done := make(chan bool, 1)
+
+	var c *portmapper.Client
+	logf := log.Printf
+	c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), func() {
+		logf("portmapping changed.")
+		logf("have mapping: %v", c.HaveMapping())
+
+		if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+			logf("cb: mapping: %v", ext)
+			select {
+			case done <- true:
+			default:
+			}
+			return
+		}
+		logf("cb: no mapping")
+	})
+	linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
+	if err != nil {
+		return err
+	}
+
+	gatewayAndSelfIP := func() (gw, self netaddr.IP, ok bool) {
+		if v := os.Getenv("TS_DEBUG_GW_SELF"); strings.Contains(v, "/") {
+			i := strings.Index(v, "/")
+			gw = netaddr.MustParseIP(v[:i])
+			self = netaddr.MustParseIP(v[i+1:])
+			return gw, self, true
+		}
+		return linkMon.GatewayAndSelfIP()
+	}
+
+	c.SetGatewayLookupFunc(gatewayAndSelfIP)
+
+	gw, selfIP, ok := gatewayAndSelfIP()
+	if !ok {
+		logf("no gateway or self IP; %v", linkMon.InterfaceState())
+		return nil
+	}
+	logf("gw=%v; self=%v", gw, selfIP)
+
+	res, err := c.Probe(ctx)
+	if err != nil {
+		return fmt.Errorf("Probe: %v", err)
+	}
+	logf("Probe: %+v", res)
+
+	if !res.PCP && !res.PMP && !res.UPnP {
+		logf("no portmapping services available")
+		return nil
+	}
+
+	uc, err := net.ListenPacket("udp", "0.0.0.0:0")
+	if err != nil {
+		return err
+	}
+	defer uc.Close()
+	c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
+
+	if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+		logf("mapping: %v", ext)
+	} else {
+		logf("no mapping")
+	}
+
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscaled/depaware.txt

@@ -130,6 +130,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
@@ -143,7 +145,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
-        tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver

cmd/tailscaled/tailscaled.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/go-multierror/multierror"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
@@ -45,15 +46,6 @@ import (
 	"tailscale.com/wgengine/router"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
 // defaultTunName returns the default tun device name for the platform.
 func defaultTunName() string {
 	switch runtime.GOOS {
@@ -167,6 +159,28 @@ func main() {
 	}
 }
 
+func ipnServerOpts() (o ipnserver.Options) {
+	// Allow changing the OS-specific IPN behavior for tests
+	// so we can e.g. test Windows-specific behaviors on Linux.
+	goos := os.Getenv("TS_DEBUG_TAILSCALED_IPN_GOOS")
+	if goos == "" {
+		goos = runtime.GOOS
+	}
+
+	o.Port = 41112
+	o.StatePath = args.statepath
+	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+
+	switch goos {
+	default:
+		o.SurviveDisconnects = true
+		o.AutostartStateKey = ipn.GlobalDaemonStateKey
+	case "windows":
+		// Not those.
+	}
+	return o
+}
+
 func run() error {
 	var err error
 
@@ -196,6 +210,9 @@ func run() error {
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
+		if os.Getenv("TS_PLEASE_PANIC") != "" {
+			panic("TS_PLEASE_PANIC asked us to panic")
+		}
 		dns.Cleanup(logf, args.tunname)
 		router.Cleanup(logf, args.tunname)
 		return nil
@@ -271,14 +288,8 @@ func run() error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		SocketPath:         args.socketpath,
-		Port:               41112,
-		StatePath:          args.statepath,
-		AutostartStateKey:  globalStateKey,
-		SurviveDisconnects: runtime.GOOS != "windows",
-		DebugMux:           debugMux,
-	}
+	opts := ipnServerOpts()
+	opts.DebugMux = debugMux
 	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
@@ -341,7 +352,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			return nil, false, err
 		}
 		conf.Tun = dev
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
 			return nil, false, err

cmd/tailscaled/tailscaled_windows.go

@@ -168,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			return nil, fmt.Errorf("TUN: %w", err)
 		}
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, nil)
 		if err != nil {
 			dev.Close()
 			return nil, fmt.Errorf("router: %w", err)
@@ -233,12 +233,6 @@ func startIPNServer(ctx context.Context, logid string) error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		Port:               41112,
-		SurviveDisconnects: false,
-		StatePath:          args.statepath,
-	}
-
 	// getEngine is called by ipnserver to get the engine. It's
 	// not called concurrently and is not called again once it
 	// successfully returns an engine.
@@ -263,7 +257,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, opts)
+	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -29,8 +29,8 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/creack/pty"
 	"github.com/gliderlabs/ssh"
-	"github.com/kr/pty"
 	gossh "golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"

control/controlclient/debug.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"regexp"
 	"runtime"
 	"strconv"
 	"time"
@@ -42,28 +41,82 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 	}
 }
 
-var reHexArgs = regexp.MustCompile(`\b0x[0-9a-f]+\b`)
-
 // scrubbedGoroutineDump returns the list of all current goroutines, but with the actual
 // values of arguments scrubbed out, lest it contain some private key material.
 func scrubbedGoroutineDump() []byte {
-	buf := make([]byte, 1<<20)
-	buf = buf[:runtime.Stack(buf, true)]
+	var buf []byte
+	// Grab stacks multiple times into increasingly larger buffer sizes
+	// to minimize the risk that we blow past our iOS memory limit.
+	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			// It fit.
+			break
+		}
+	}
+	return scrubHex(buf)
+}
 
+func scrubHex(buf []byte) []byte {
 	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
-	return reHexArgs.ReplaceAllFunc(buf, func(in []byte) []byte {
+
+	foreachHexAddress(buf, func(in []byte) {
 		if string(in) == "0x0" {
-			return in
+			return
 		}
 		if v, ok := saw[string(in)]; ok {
-			return v
+			for i := range in {
+				in[i] = '_'
+			}
+			copy(in, v)
+			return
 		}
+		inStr := string(in)
 		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
+		for i := range in {
+			in[i] = '_'
+		}
 		if err != nil {
-			return []byte("??")
+			in[0] = '?'
+			return
 		}
 		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
-		saw[string(in)] = v
-		return v
+		saw[inStr] = v
+		copy(in, v)
 	})
+	return buf
+}
+
+var ohx = []byte("0x")
+
+// foreachHexAddress calls f with each subslice of b that matches
+// regexp `0x[0-9a-f]*`.
+func foreachHexAddress(b []byte, f func([]byte)) {
+	for len(b) > 0 {
+		i := bytes.Index(b, ohx)
+		if i == -1 {
+			return
+		}
+		b = b[i:]
+		hx := hexPrefix(b)
+		f(hx)
+		b = b[len(hx):]
+	}
+}
+
+func hexPrefix(b []byte) []byte {
+	for i, c := range b {
+		if i < 2 {
+			continue
+		}
+		if !isHexByte(c) {
+			return b[:i]
+		}
+	}
+	return b
+}
+
+func isHexByte(b byte) bool {
+	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
 }

control/controlclient/debug_test.go

@@ -9,3 +9,22 @@ import "testing"
 func TestScrubbedGoroutineDump(t *testing.T) {
 	t.Logf("Got:\n%s\n", scrubbedGoroutineDump())
 }
+
+func TestScrubHex(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"foo", "foo"},
+		{"", ""},
+		{"0x", "?_"},
+		{"0x001 and same 0x001", "v1%1_ and same v1%1_"},
+		{"0x008 and same 0x008", "v1%0_ and same v1%0_"},
+		{"0x001 and diff 0x002", "v1%1_ and diff v2%2_"},
+	}
+	for _, tt := range tests {
+		got := scrubHex([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Errorf("for input:\n%s\n\ngot:\n%s\n\nwant:\n%s\n", tt.in, got, tt.want)
+		}
+	}
+}

control/controlclient/direct.go

@@ -220,6 +220,13 @@ func packageType() string {
 		// Using tailscaled or IPNExtension?
 		exe, _ := os.Executable()
 		return filepath.Base(exe)
+	case "linux":
+		// Report whether this is in a snap.
+		// See https://snapcraft.io/docs/environment-variables
+		// We just look at two somewhat arbitrarily.
+		if os.Getenv("SNAP_NAME") != "" && os.Getenv("SNAP") != "" {
+			return "snap"
+		}
 	}
 	return ""
 }

derp/derp_server.go

@@ -36,6 +36,7 @@ import (
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
@@ -118,6 +119,8 @@ type Server struct {
 	curClients                   expvar.Int
 	curHomeClients               expvar.Int // ones with preferred
 	clientsReplaced              expvar.Int
+	clientsReplaceLimited        expvar.Int
+	clientsReplaceSleeping       expvar.Int
 	unknownFrames                expvar.Int
 	homeMovesIn                  expvar.Int // established clients announce home server moves in
 	homeMovesOut                 expvar.Int // established clients announce home server moves out
@@ -164,7 +167,7 @@ type PacketForwarder interface {
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
 // It is a defined type so that non-net connections can be used.
 type Conn interface {
-	io.Closer
+	io.WriteCloser
 
 	// The *Deadline methods follow the semantics of net.Conn.
 
@@ -346,14 +349,28 @@ func (s *Server) initMetacert() {
 func (s *Server) MetaCert() []byte { return s.metaCert }
 
 // registerClient notes that client c is now authenticated and ready for packets.
-// If c's public key was already connected with a different connection, the prior one is closed.
-func (s *Server) registerClient(c *sclient) {
+//
+// If c's public key was already connected with a different
+// connection, the prior one is closed, unless it's fighting rapidly
+// with another client with the same key, in which case the returned
+// ok is false, and the caller should wait the provided duration
+// before trying again.
+func (s *Server) registerClient(c *sclient) (ok bool, d time.Duration) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	old := s.clients[c.key]
 	if old == nil {
 		c.logf("adding connection")
 	} else {
+		// Take over the old rate limiter, discarding the one
+		// our caller just made.
+		c.replaceLimiter = old.replaceLimiter
+		if rr := c.replaceLimiter.ReserveN(timeNow(), 1); rr.OK() {
+			if d := rr.DelayFrom(timeNow()); d > 0 {
+				s.clientsReplaceLimited.Add(1)
+				return false, d
+			}
+		}
 		s.clientsReplaced.Add(1)
 		c.logf("adding connection, replacing %s", old.remoteAddr)
 		go old.nc.Close()
@@ -365,6 +382,7 @@ func (s *Server) registerClient(c *sclient) {
 	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
+	return true, 0
 }
 
 // broadcastPeerStateChangeLocked enqueues a message to all watchers
@@ -452,8 +470,9 @@ func (s *Server) addWatcher(c *sclient) {
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
-	br, bw := brw.Reader, brw.Writer
+	br := brw.Reader
 	nc.SetDeadline(time.Now().Add(10 * time.Second))
+	bw := &lazyBufioWriter{w: nc, lbw: brw.Writer}
 	if err := s.sendServerKey(bw); err != nil {
 		return fmt.Errorf("send server key: %v", err)
 	}
@@ -490,7 +509,14 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
 		peerGone:       make(chan key.Public),
 		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+
+		// Allow kicking out previous connections once a
+		// minute, with a very high burst of 100. Once a
+		// minute is less than the client's 2 minute
+		// inactivity timeout.
+		replaceLimiter: rate.NewLimiter(rate.Every(time.Minute), 100),
 	}
+
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
 	}
@@ -498,10 +524,18 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		c.info = *clientInfo
 	}
 
-	s.registerClient(c)
+	for {
+		ok, d := s.registerClient(c)
+		if ok {
+			break
+		}
+		s.clientsReplaceSleeping.Add(1)
+		timeSleep(d)
+		s.clientsReplaceSleeping.Add(-1)
+	}
 	defer s.unregisterClient(c)
 
-	err = s.sendServerInfo(bw, clientKey)
+	err = s.sendServerInfo(c.bw, clientKey)
 	if err != nil {
 		return fmt.Errorf("send server info: %v", err)
 	}
@@ -509,6 +543,12 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	return c.run(ctx)
 }
 
+// for testing
+var (
+	timeSleep = time.Sleep
+	timeNow   = time.Now
+)
+
 // run serves the client until there's an error.
 // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error.
 func (c *sclient) run(ctx context.Context) error {
@@ -698,7 +738,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 // dropReason is why we dropped a DERP frame.
 type dropReason int
 
-//go:generate stringer -type=dropReason -trimprefix=dropReason
+//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason
 
 const (
 	dropReasonUnknownDest      dropReason = iota // unknown destination pubkey
@@ -807,18 +847,20 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	return nil
 }
 
-func (s *Server) sendServerKey(bw *bufio.Writer) error {
+func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
 	buf := make([]byte, 0, len(magic)+len(s.publicKey))
 	buf = append(buf, magic...)
 	buf = append(buf, s.publicKey[:]...)
-	return writeFrame(bw, frameServerKey, buf)
+	err := writeFrame(lw.bw(), frameServerKey, buf)
+	lw.Flush() // redundant (no-op) flush to release bufio.Writer
+	return err
 }
 
 type serverInfo struct {
 	Version int `json:"version,omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
+func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
 	var nonce [24]byte
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
@@ -829,7 +871,7 @@ func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 	}
 
 	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw, frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
+	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(nonce[:]); err != nil {
@@ -952,13 +994,18 @@ type sclient struct {
 	meshUpdate     chan struct{}   // write request to write peerStateChange
 	canMesh        bool            // clientInfo had correct mesh token for inter-region routing
 
+	// replaceLimiter controls how quickly two connections with
+	// the same client key can kick each other off the server by
+	// taking over ownership of a key.
+	replaceLimiter *rate.Limiter
+
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
 	connectedAt time.Time
 	preferred   bool
 
 	// Owned by sender, not thread-safe.
-	bw *bufio.Writer
+	bw *lazyBufioWriter
 
 	// Guarded by s.mu
 	//
@@ -1120,14 +1167,14 @@ func (c *sclient) setWriteDeadline() {
 // sendKeepAlive sends a keep-alive frame, without flushing.
 func (c *sclient) sendKeepAlive() error {
 	c.setWriteDeadline()
-	return writeFrameHeader(c.bw, frameKeepAlive, 0)
+	return writeFrameHeader(c.bw.bw(), frameKeepAlive, 0)
 }
 
 // sendPeerGone sends a peerGone frame, without flushing.
 func (c *sclient) sendPeerGone(peer key.Public) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerGone, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1137,7 +1184,7 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 // sendPeerPresent sends a peerPresent frame, without flushing.
 func (c *sclient) sendPeerPresent(peer key.Public) error {
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1210,11 +1257,11 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	if withKey {
 		pktLen += len(srcKey)
 	}
-	if err = writeFrameHeader(c.bw, frameRecvPacket, uint32(pktLen)); err != nil {
+	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw, &srcKey)
+		err := writePublicKey(c.bw.bw(), &srcKey)
 		if err != nil {
 			return err
 		}
@@ -1351,6 +1398,8 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
+	m.Set("clients_replace_limited", &s.clientsReplaceLimited)
+	m.Set("gauge_clients_replace_sleeping", &s.clientsReplaceSleeping)
 	m.Set("bytes_received", &s.bytesRecv)
 	m.Set("bytes_sent", &s.bytesSent)
 	m.Set("packets_dropped", &s.packetsDropped)
@@ -1527,3 +1576,45 @@ func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(minTimeBetweenLogs)
 	}
 }
+
+var bufioWriterPool = &sync.Pool{
+	New: func() interface{} {
+		return bufio.NewWriterSize(ioutil.Discard, 2<<10)
+	},
+}
+
+// lazyBufioWriter is a bufio.Writer-like wrapping writer that lazily
+// allocates its actual bufio.Writer from a sync.Pool, releasing it to
+// the pool upon flush.
+//
+// We do this to reduce memory overhead; most DERP connections are
+// idle and the idle bufio.Writers were 30% of overall memory usage.
+type lazyBufioWriter struct {
+	w   io.Writer     // underlying
+	lbw *bufio.Writer // lazy; nil means it needs an associated buffer
+}
+
+func (w *lazyBufioWriter) bw() *bufio.Writer {
+	if w.lbw == nil {
+		w.lbw = bufioWriterPool.Get().(*bufio.Writer)
+		w.lbw.Reset(w.w)
+	}
+	return w.lbw
+}
+
+func (w *lazyBufioWriter) Available() int { return w.bw().Available() }
+
+func (w *lazyBufioWriter) Write(p []byte) (int, error) { return w.bw().Write(p) }
+
+func (w *lazyBufioWriter) Flush() error {
+	if w.lbw == nil {
+		return nil
+	}
+	err := w.lbw.Flush()
+
+	w.lbw.Reset(ioutil.Discard)
+	bufioWriterPool.Put(w.lbw)
+	w.lbw = nil
+
+	return err
+}

derp/derp_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -849,6 +850,136 @@ func TestClientSendPong(t *testing.T) {
 
 }
 
+func TestServerReplaceClients(t *testing.T) {
+	defer func() {
+		timeSleep = time.Sleep
+		timeNow = time.Now
+	}()
+
+	var (
+		mu     sync.Mutex
+		now    = time.Unix(123, 0)
+		sleeps int
+		slept  time.Duration
+	)
+	timeSleep = func(d time.Duration) {
+		mu.Lock()
+		defer mu.Unlock()
+		sleeps++
+		slept += d
+		now = now.Add(d)
+	}
+	timeNow = func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}
+
+	serverPrivateKey := newPrivateKey(t)
+	var logger logger.Logf = logger.Discard
+	const debug = false
+	if debug {
+		logger = t.Logf
+	}
+
+	s := NewServer(serverPrivateKey, logger)
+	defer s.Close()
+
+	priv := newPrivateKey(t)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	connNum := 0
+	connect := func() *Client {
+		connNum++
+		cout, err := net.Dial("tcp", ln.Addr().String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cin, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		brwServer := bufio.NewReadWriter(bufio.NewReader(cin), bufio.NewWriter(cin))
+		go s.Accept(cin, brwServer, fmt.Sprintf("test-client-%d", connNum))
+
+		brw := bufio.NewReadWriter(bufio.NewReader(cout), bufio.NewWriter(cout))
+		c, err := NewClient(priv, cout, brw, logger)
+		if err != nil {
+			t.Fatalf("client %d: %v", connNum, err)
+		}
+		return c
+	}
+
+	wantVar := func(v *expvar.Int, want int64) {
+		t.Helper()
+		if got := v.Value(); got != want {
+			t.Errorf("got %d; want %d", got, want)
+		}
+	}
+
+	wantClosed := func(c *Client) {
+		t.Helper()
+		for {
+			m, err := c.Recv()
+			if err != nil {
+				t.Logf("got expected error: %v", err)
+				return
+			}
+			switch m.(type) {
+			case ServerInfoMessage:
+				continue
+			default:
+				t.Fatalf("client got %T; wanted an error", m)
+			}
+		}
+	}
+
+	c1 := connect()
+	waitConnect(t, c1)
+	c2 := connect()
+	waitConnect(t, c2)
+	wantVar(&s.clientsReplaced, 1)
+	wantClosed(c1)
+
+	for i := 0; i < 100+5; i++ {
+		c := connect()
+		defer c.nc.Close()
+		if s.clientsReplaceLimited.Value() == 0 && i < 90 {
+			continue
+		}
+		t.Logf("for %d: replaced=%d, limited=%d, sleeping=%d", i,
+			s.clientsReplaced.Value(),
+			s.clientsReplaceLimited.Value(),
+			s.clientsReplaceSleeping.Value(),
+		)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if sleeps == 0 {
+		t.Errorf("no sleeps")
+	}
+	if slept == 0 {
+		t.Errorf("total sleep duration was 0")
+	}
+}
+
+func TestLimiter(t *testing.T) {
+	rl := rate.NewLimiter(rate.Every(time.Minute), 100)
+	for i := 0; i < 200; i++ {
+		r := rl.Reserve()
+		d := r.Delay()
+		t.Logf("i=%d, allow=%v, d=%v", i, r.OK(), d)
+	}
+}
+
 func BenchmarkSendRecv(b *testing.B) {
 	for _, size := range []int{10, 100, 1000, 10000} {
 		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })

derp/derphttp/derphttp_client.go

@@ -54,6 +54,7 @@ type Client struct {
 
 	privateKey key.Private
 	logf       logger.Logf
+	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
 	// Either url or getRegion is non-nil:
 	url       *url.URL
@@ -363,14 +364,26 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	return c.client, c.connGen, nil
 }
 
+// SetURLDialer sets the dialer to use for dialing URLs.
+// This dialer is only use for clients created with NewClient, not NewRegionClient.
+// If unset or nil, the default dialer is used.
+//
+// The primary use for this is the derper mesh mode to connect to each
+// other over a VPC network.
+func (c *Client) SetURLDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
+	c.dialer = dialer
+}
+
 func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 	host := c.url.Hostname()
+	if c.dialer != nil {
+		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
+	}
 	hostOrIP := host
-
 	dialer := netns.NewDialer()
 
 	if c.DNSCache != nil {
-		ip, _, err := c.DNSCache.LookupIP(ctx, host)
+		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
 		if err == nil {
 			hostOrIP = ip.String()
 		}

derp/dropreason_string.go

@@ -1,3 +1,7 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 // Code generated by "stringer -type=dropReason -trimprefix=dropReason"; DO NOT EDIT.
 
 package derp

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
+	github.com/creack/pty v1.1.9
+	github.com/dave/jennifer v1.4.1
 	github.com/frankban/quicktest v1.13.0
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
@@ -16,10 +18,10 @@ require (
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
+	github.com/iancoleman/strcase v0.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
-	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
@@ -29,7 +31,8 @@ require (
 	github.com/pkg/sftp v1.13.0
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 // indirect
+	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
+	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
@@ -43,7 +46,7 @@ require (
 	golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0
 	golang.zx2c4.com/wireguard/windows v0.3.16
 	honnef.co/go/tools v0.1.4
-	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
+	inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1
 	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756

go.sum

@@ -82,12 +82,13 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/daixiang0/gci v0.2.7 h1:bosLNficubzJZICsVzxuyNc6oAbdz0zcqLG2G/RxtY4=
 github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
+github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
+github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -296,6 +297,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -355,8 +358,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -580,12 +581,10 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/tailscale/goupnp v1.0.1-0.20210629174436-7df6c9efe30c h1:F+pROyGPs+9wdB7jBPHr9IZEF8SKj9YUCFFShnyLNZM=
-github.com/tailscale/goupnp v1.0.1-0.20210629174436-7df6c9efe30c/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
-github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683 h1:ZXmZQuVebYllEJL/dpttpIDGx723ezC5GJkHIu0YKrM=
-github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
-github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 h1:AIJ8AF9O7jBmCwilP0ydwJMIzW5dw48Us8f3hLJhYBY=
-github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -692,7 +691,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -739,7 +737,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
@@ -958,8 +955,8 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=

ipn/ipnlocal/local.go

@@ -95,7 +95,7 @@ type LocalBackend struct {
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
 
-	filterHash string
+	filterHash deephash.Sum
 
 	// The mutex protects the following elements.
 	mu             sync.Mutex
@@ -179,6 +179,7 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		gotPortPollRes: make(chan struct{}),
 	}
 	b.statusChanged = sync.NewCond(&b.statusLock)
+	b.e.SetStatusCallback(b.setWgengineStatus)
 
 	linkMon := e.GetLinkMonitor()
 	b.prevIfState = linkMon.InterfaceState()
@@ -214,6 +215,15 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
+// b.mu must be held.
+func (b *LocalBackend) maybePauseControlClientLocked() {
+	if b.cc == nil {
+		return
+	}
+	networkUp := b.prevIfState.AnyInterfaceUp()
+	b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
+}
+
 // linkChange is our link monitor callback, called whenever the network changes.
 // major is whether ifst is different than earlier.
 func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
@@ -222,11 +232,7 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	hadPAC := b.prevIfState.HasPAC()
 	b.prevIfState = ifst
-
-	networkUp := ifst.AnyInterfaceUp()
-	if b.cc != nil {
-		go b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
-	}
+	b.maybePauseControlClientLocked()
 
 	// If the PAC-ness of the network changed, reconfig wireguard+route to
 	// add/remove subnets.
@@ -605,8 +611,8 @@ func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 
 	if cc != nil {
 		cc.UpdateEndpoints(0, s.LocalAddrs)
+		b.stateMachine()
 	}
-	b.stateMachine()
 
 	b.statusLock.Lock()
 	b.statusChanged.Broadcast()
@@ -863,7 +869,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	cc.SetStatusFunc(b.setClientStatus)
-	b.e.SetStatusCallback(b.setWgengineStatus)
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
 	b.mu.Lock()
@@ -939,7 +944,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets, _ := localNetsB.IPSet()
 	logNets, _ := logNetsB.IPSet()
 
-	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.Update(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -979,6 +984,44 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 	tsaddr.TailscaleULARange(),
 }
 
+// internalAndExternalInterfaces splits interface routes into "internal"
+// and "external" sets. Internal routes are those of virtual ethernet
+// network interfaces used by guest VMs and containers, such as WSL and
+// Docker.
+//
+// Given that "internal" routes don't leave the device, we choose to
+// trust them more, allowing access to them when an Exit Node is enabled.
+func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
+	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
+			return
+		}
+		if pfx.IsSingleIP() {
+			return
+		}
+		if runtime.GOOS == "windows" {
+			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
+			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
+			//
+			// This includes WSL2 vEthernet.
+			// Importantly: by default WSL2 /etc/resolv.conf points to
+			// a stub resolver running on the host vEthernet IP.
+			// So enabling exit nodes with the default tailnet
+			// configuration breaks WSL2 DNS without this.
+			mac := iface.Interface.HardwareAddr
+			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
+				internal = append(internal, pfx)
+				return
+			}
+		}
+		external = append(external, pfx)
+	}); err != nil {
+		return nil, nil, err
+	}
+
+	return internal, external, nil
+}
+
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
@@ -2114,18 +2157,21 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
+		internalIPs, externalIPs, err := internalAndExternalInterfaces()
+		if err != nil {
+			b.logf("failed to discover interface ips: %v", err)
+		}
 		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
-			// Only allow local lan access on linux machines for now.
-			ips, _, err := interfaceRoutes()
-			if err != nil {
-				b.logf("failed to discover interface ips: %v", err)
-			}
+			rs.LocalRoutes = internalIPs // unconditionally allow access to guest VM networks
 			if prefs.ExitNodeAllowLANAccess {
-				rs.LocalRoutes = ips.Prefixes()
+				rs.LocalRoutes = append(rs.LocalRoutes, externalIPs...)
+				if len(externalIPs) != 0 {
+					b.logf("allowing exit node access to internal IPs: %v", internalIPs)
+				}
 			} else {
 				// Explicitly add routes to the local network so that we do not
 				// leak any traffic.
-				rs.Routes = append(rs.Routes, ips.Prefixes()...)
+				rs.Routes = append(rs.Routes, externalIPs...)
 			}
 		}
 	}
@@ -2154,6 +2200,18 @@ func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *ipn.Prefs) {
 	}
 	if v := prefs.OSVersion; v != "" {
 		hi.OSVersion = v
+
+		// The Android app annotates when Google Play Services
+		// aren't available by tacking on a string to the
+		// OSVersion. Promote that to the Hostinfo.Package
+		// field instead, rather than adding a new pref, as
+		// this applyPrefsToHostinfo mechanism is mostly
+		// abused currently. TODO(bradfitz): instead let
+		// frontends update Hostinfo, without using Prefs.
+		if runtime.GOOS == "android" && strings.HasSuffix(v, " [nogoogle]") {
+			hi.Package = "nogoogle"
+			hi.OSVersion = strings.TrimSuffix(v, " [nogoogle]")
+		}
 	}
 	if m := prefs.DeviceModel; m != "" {
 		hi.DeviceModel = m
@@ -2173,9 +2231,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	oldState := b.state
 	b.state = newState
 	prefs := b.prefs
-	cc := b.cc
 	netMap := b.netMap
-	networkUp := b.prevIfState.AnyInterfaceUp()
 	activeLogin := b.activeLogin
 	authURL := b.authURL
 	if newState == ipn.Running {
@@ -2185,6 +2241,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		// Transitioning away from running.
 		b.closePeerAPIListenersLocked()
 	}
+	b.maybePauseControlClientLocked()
 	b.mu.Unlock()
 
 	if oldState == newState {
@@ -2195,10 +2252,6 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
-	if cc != nil {
-		cc.SetPaused((newState == ipn.Stopped && netMap != nil) || !networkUp)
-	}
-
 	switch newState {
 	case ipn.NeedsLogin:
 		systemd.Status("Needs login: %s", authURL)
@@ -2459,6 +2512,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		b.logf("active login: %v", login)
 		b.activeLogin = login
 	}
+	b.maybePauseControlClientLocked()
 
 	// Determine if file sharing is enabled
 	fs := hasCapability(nm, tailcfg.CapabilityFileSharing)
@@ -2730,17 +2784,18 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 
+	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
 		}
 		if !on {
-			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
+			return fmt.Errorf("%s is disabled.%s", key, suffix)
 		}
 	}
 	return nil

ipn/ipnlocal/state_test.go

@@ -354,7 +354,7 @@ func TestStateMachine(t *testing.T) {
 	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -389,7 +389,7 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause"})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
@@ -414,7 +414,7 @@ func TestStateMachine(t *testing.T) {
 		// We're still not logged in so there's nothing we can do
 		// with it. (And empirically, it's providing an empty list
 		// of endpoints.)
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
 		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
 	}
@@ -440,7 +440,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, url2, false, nil)
 	{
 		// BUG: UpdateEndpoints again, this is getting silly.
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		// This time, backend should emit it to the UI right away,
 		// because the UI is anxiously awaiting a new URL to visit.
@@ -470,7 +470,7 @@ func TestStateMachine(t *testing.T) {
 		// wait until it gets into Starting.
 		// TODO: (Currently this test doesn't detect that bug, but
 		// it's visible in the logs)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -483,7 +483,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(1)
 	// BUG: the real controlclient sends LoginFinished with every
 	// notification while it's in StateAuthenticated, but not StateSynced.
-	// We should send it exactly once, or every time we're authenticated,
+	// It should send it exactly once, or every time we're authenticated,
 	// but the current code is brittle.
 	// (ie. I suspect it would be better to change false->true in send()
 	// below, and do the same in the real controlclient.)
@@ -492,7 +492,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
@@ -534,7 +534,7 @@ func TestStateMachine(t *testing.T) {
 		nn := notifies.drain(2)
 		// BUG: UpdateEndpoints isn't needed here.
 		// BUG: Login isn't needed here. We never logged out.
-		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -543,7 +543,8 @@ func TestStateMachine(t *testing.T) {
 	}
 
 	// Test the fast-path frontend reconnection.
-	// This one is very finicky, so we have to force State==Running.
+	// This one is very finicky, so we have to force State==Running
+	// or it won't use the fast path.
 	// TODO: actually get to State==Running, rather than cheating.
 	//  That'll require spinning up a fake DERP server and putting it in
 	//  the netmap.
@@ -570,7 +571,7 @@ func TestStateMachine(t *testing.T) {
 	b.Logout()
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"pause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause", "StartLogout", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
@@ -587,7 +588,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
@@ -603,7 +604,7 @@ func TestStateMachine(t *testing.T) {
 		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
-		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"StartLogout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -617,8 +618,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -632,7 +632,7 @@ func TestStateMachine(t *testing.T) {
 	// I guess, since that's supposed to be synchronous.
 	{
 		notifies.drain(0)
-		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Logout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -646,8 +646,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -678,7 +677,7 @@ func TestStateMachine(t *testing.T) {
 		// BUG: We already called Shutdown(), no need to do it again.
 		// BUG: Way too soon for UpdateEndpoints.
 		// BUG: don't unpause because we're not logged in.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -703,7 +702,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -743,9 +742,8 @@ func TestStateMachine(t *testing.T) {
 		//  on startup, otherwise UIs can't show the node list, login
 		//  name, etc when in state ipn.Stopped.
 		//  Arguably they shouldn't try. But they currently do.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
-
 		nn := notifies.drain(2)
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
@@ -754,6 +752,25 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
 	}
 
+	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
+	// the first netmap. Simulate that netmap being received, after which
+	// it should pause us, to avoid wasting CPU retrieving unnecessarily
+	// additional netmap updates.
+	//
+	// TODO: really the various GUIs and prefs should be refactored to
+	//  not require the netmap structure at all when starting while
+	//  !WantRunning. That would remove the need for this (or contacting
+	//  the control server at all when stopped).
+	t.Logf("\n\nStart4 -> netmap")
+	notifies.expect(0)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		notifies.drain(0)
+		c.Assert([]string{"pause", "pause"}, qt.DeepEquals, cc.getCalls())
+	}
+
 	// Request connection.
 	// The state machine didn't call Login() earlier, so now it needs to.
 	t.Logf("\n\nWantRunning4 -> true")
@@ -780,7 +797,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -790,25 +807,27 @@ func TestStateMachine(t *testing.T) {
 	// We want to try logging in as a different user, while Stopped.
 	// First, start the login process (without logging out first).
 	t.Logf("\n\nLoginDifferent")
-	notifies.expect(2)
+	notifies.expect(1)
 	b.StartLoginInteractive()
 	url3 := "http://localhost:1/3"
 	cc.send(nil, url3, false, nil)
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(1)
 		// It might seem like WantRunning should switch to true here,
 		// but that would be risky since we already have a valid
 		// user account. It might try to reconnect to the old account
 		// before the new one is ready. So no change yet.
-		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		//
+		// Because the login hasn't yet completed, the old login
+		// is still valid, so it's correct that we stay paused.
+		c.Assert([]string{"Login", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
-		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 	}
 
-	// Now, let's say the interactive login completed, using a different
-	// user account than before.
+	// Now, let's complete the interactive login, using a different
+	// user account than before. WantRunning changes to true after an
+	// interactive login, so we end up unpaused.
 	t.Logf("\n\nLoginDifferent URL visited")
 	notifies.expect(3)
 	cc.persist.LoginName = "user3"
@@ -817,7 +836,13 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: pause() being called here is a bad sign.
+		//  It means that either the state machine ran at least once
+		//  with the old netmap, or it ran with the new login+netmap
+		//  and !WantRunning. But since it's a fresh and successful
+		//  new login, WantRunning is true, so there was never a
+		//  reason to pause().
+		c.Assert([]string{"pause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -836,7 +861,7 @@ func TestStateMachine(t *testing.T) {
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() ourselves.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(1)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -855,7 +880,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))

ipn/ipnstate/ipnstate.go

@@ -20,6 +20,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/util/dnsname"
 )
@@ -90,7 +91,7 @@ type PeerStatus struct {
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
+	LastWrite     mono.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -320,7 +321,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	f("<tr><th>Peer</th><th>OS</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Connection</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
-	now := time.Now()
+	now := mono.Now()
 
 	var peers []*PeerStatus
 	for _, peer := range st.Peers() {
@@ -378,7 +379,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 		f("<td>")
 
 		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		active := !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 		if active {
 			if ps.Relay != "" && ps.CurAddr == "" {
 				f("relay <b>%s</b>", html.EscapeString(ps.Relay))

logpolicy/logpolicy.go

@@ -128,6 +128,13 @@ func (l logWriter) Write(buf []byte) (int, error) {
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
 func logsDir(logf logger.Logf) string {
+	if d := os.Getenv("TS_LOGS_DIR"); d != "" {
+		fi, err := os.Stat(d)
+		if err == nil && fi.IsDir() {
+			return d
+		}
+	}
+
 	// STATE_DIRECTORY is set by systemd 240+ but we support older
 	// systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237.
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")
@@ -180,6 +187,10 @@ func runningUnderSystemd() bool {
 	return false
 }
 
+func redirectStderrToLogPanics() bool {
+	return runningUnderSystemd() || os.Getenv("TS_PLEASE_PANIC") != ""
+}
+
 // tryFixLogStateLocation is a temporary fixup for
 // https://github.com/tailscale/tailscale/issues/247 . We accidentally
 // wrote logging state files to /, and then later to $CACHE_DIRECTORY
@@ -428,9 +439,14 @@ func New(collection string) *Policy {
 		c.HTTPC = &http.Client{Transport: newLogtailTransport(u.Host)}
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{
+		ReplaceStderr: redirectStderrToLogPanics(),
+	})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
+		if filchBuf.OrigStderr != nil {
+			c.Stderr = filchBuf.OrigStderr
+		}
 	}
 	lw := logtail.NewLogger(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here

logtail/logtail.go

@@ -118,9 +118,10 @@ type Logger struct {
 	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
+	explainedRaw   bool
 
 	shutdownStart chan struct{} // closed when shutdown begins
-	shutdownDone  chan struct{} // closd when shutdown complete
+	shutdownDone  chan struct{} // closed when shutdown complete
 }
 
 // SetVerbosityLevel controls the verbosity level that should be
@@ -230,6 +231,14 @@ func (l *Logger) drainPending() (res []byte) {
 			// outside of the logtail logger. Encode it.
 			// Do not add a client time, as it could have been
 			// been written a long time ago.
+			if !l.explainedRaw {
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: *** Lines prefixed with RAW-STDERR below bypassed logtail and probably come from a previous run of the program\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR:\n")
+				l.explainedRaw = true
+			}
+			fmt.Fprintf(l.stderr, "RAW-STDERR: %s", b)
 			b = l.encodeText(b, true)
 		}
 

net/dns/config.go

@@ -5,9 +5,12 @@
 package dns
 
 import (
+	"bufio"
+	"fmt"
 	"sort"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/util/dnsname"
 )
 
@@ -38,6 +41,20 @@ type Config struct {
 	Hosts map[dnsname.FQDN][]netaddr.IP
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{DefaultResolvers:")
+	resolver.WriteIPPorts(w, c.DefaultResolvers)
+
+	w.WriteString(" Routes:")
+	resolver.WriteRoutes(w, c.Routes)
+
+	fmt.Fprintf(w, " SearchDomains:%v", c.SearchDomains)
+	fmt.Fprintf(w, " Hosts:%v", len(c.Hosts))
+	w.WriteString("}")
+}
+
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {

net/dns/ini.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build windows
+
 package dns
 
 import (

net/dns/ini_test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build windows
+
 package dns
 
 import (

net/dns/manager.go

@@ -5,6 +5,7 @@
 package dns
 
 import (
+	"bufio"
 	"runtime"
 	"time"
 
@@ -50,14 +51,18 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, li
 }
 
 func (m *Manager) Set(cfg Config) error {
-	m.logf("Set: %+v", cfg)
+	m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		cfg.WriteToBufioWriter(w)
+	}))
 
 	rcfg, ocfg, err := m.compileConfig(cfg)
 	if err != nil {
 		return err
 	}
 
-	m.logf("Resolvercfg: %+v", rcfg)
+	m.logf("Resolvercfg: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		rcfg.WriteToBufioWriter(w)
+	}))
 	m.logf("OScfg: %+v", ocfg)
 
 	if err := m.resolver.SetConfig(rcfg); err != nil {

net/dns/manager_windows.go

@@ -293,7 +293,7 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 
 		t0 = time.Now()
-		m.logf("running ipconfig /registerdns ...")
+		m.logf("running ipconfig /flushdns ...")
 		cmd = exec.Command("ipconfig", "/flushdns")
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
 		err = cmd.Run()

net/dns/resolver/doh_test.go

@@ -44,7 +44,9 @@ func TestDoH(t *testing.T) {
 		t.Fatal("no known DoH")
 	}
 
-	f := new(forwarder)
+	f := &forwarder{
+		dohSem: make(chan struct{}, 10),
+	}
 
 	for ip := range knownDoH {
 		t.Run(ip.String(), func(t *testing.T) {
@@ -81,3 +83,16 @@ func TestDoH(t *testing.T) {
 		})
 	}
 }
+
+func TestDoHV6Fallback(t *testing.T) {
+	for ip, base := range knownDoH {
+		if ip.Is4() {
+			ip6, ok := dohV6(base)
+			if !ok {
+				t.Errorf("no v6 DoH known for %v", ip)
+			} else if !ip6.Is6() {
+				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+			}
+		}
+	}
+}

net/dns/resolver/forwarder.go

@@ -16,6 +16,8 @@ import (
 	"math/rand"
 	"net"
 	"net/http"
+	"runtime"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -39,6 +41,11 @@ const (
 	// connections open to DNS-over-HTTPs servers. This is pretty
 	// arbitrary.
 	dohTransportTimeout = 30 * time.Second
+
+	// wellKnownHostBackupDelay is how long to artificially delay upstream
+	// DNS queries to the "fallback" DNS server IP for a known provider
+	// (e.g. how long to wait to query Google's 8.8.4.4 after 8.8.8.8).
+	wellKnownHostBackupDelay = 200 * time.Millisecond
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
@@ -118,6 +125,7 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		return
 	}
 
+	// https://datatracker.ietf.org/doc/html/rfc6891#section-6.1.2
 	opt := packet[len(packet)-optFixedBytes:]
 
 	if opt[0] != 0 {
@@ -134,8 +142,8 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		// Be conservative and don't touch unknown versions.
 		return
 	}
-	// Ignore flags in opt[7:9]
-	if binary.BigEndian.Uint16(opt[10:12]) != 0 {
+	// Ignore flags in opt[6:9]
+	if binary.BigEndian.Uint16(opt[9:11]) != 0 {
 		// RDLEN must be 0 (no variable length data). We're at the end of the
 		// packet so this should be 0 anyway)..
 		return
@@ -151,7 +159,20 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 
 type route struct {
 	Suffix    dnsname.FQDN
-	Resolvers []netaddr.IPPort
+	Resolvers []resolverAndDelay
+}
+
+// resolverAndDelay is an upstream DNS resolver and a delay for how
+// long to wait before querying it.
+type resolverAndDelay struct {
+	// ipp is the upstream resolver.
+	ipp netaddr.IPPort
+
+	// startDelay is an amount to delay this resolver at
+	// start. It's used when, say, there are four Google or
+	// Cloudflare DNS IPs (two IPv4 + two IPv6) and we don't want
+	// to race all four at once.
+	startDelay time.Duration
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
@@ -159,6 +180,7 @@ type forwarder struct {
 	logf    logger.Logf
 	linkMon *monitor.Mon
 	linkSel ForwardLinkSelector
+	dohSem  chan struct{}
 
 	ctx       context.Context    // good until Close
 	ctxCancel context.CancelFunc // closes ctx
@@ -180,11 +202,18 @@ func init() {
 }
 
 func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+	maxDoHInFlight := 1000 // effectively unlimited
+	if runtime.GOOS == "ios" {
+		// No HTTP/2 on iOS yet (for size reasons), so DoH is
+		// pricier.
+		maxDoHInFlight = 10
+	}
 	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		linkMon:   linkMon,
 		linkSel:   linkSel,
 		responses: responses,
+		dohSem:    make(chan struct{}, maxDoHInFlight),
 	}
 	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
 	return f
@@ -195,7 +224,84 @@ func (f *forwarder) Close() error {
 	return nil
 }
 
-func (f *forwarder) setRoutes(routes []route) {
+// resolversWithDelays maps from a set of DNS server ip:ports (currently
+// the port is always 53) to a slice of a type that included a
+// startDelay. So if ipps contains e.g. four Google DNS IPs (two IPv4
+// + twoIPv6), this function partition adds delays to some.
+func resolversWithDelays(ipps []netaddr.IPPort) []resolverAndDelay {
+	type hostAndFam struct {
+		host string // some arbitrary string representing DNS host (currently the DoH base)
+		bits uint8  // either 32 or 128 for IPv4 vs IPv6s address family
+	}
+
+	// Track how many of each known resolver host are in the list,
+	// per address family.
+	total := map[hostAndFam]int{}
+
+	rr := make([]resolverAndDelay, len(ipps))
+	for _, ipp := range ipps {
+		ip := ipp.IP()
+		if host, ok := knownDoH[ip]; ok {
+			total[hostAndFam{host, ip.BitLen()}]++
+		}
+	}
+
+	done := map[hostAndFam]int{}
+	for i, ipp := range ipps {
+		ip := ipp.IP()
+		var startDelay time.Duration
+		if host, ok := knownDoH[ip]; ok {
+			key4 := hostAndFam{host, 32}
+			key6 := hostAndFam{host, 128}
+			switch {
+			case ip.Is4():
+				if done[key4] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			case ip.Is6():
+				total4 := total[key4]
+				if total4 >= 2 {
+					// If we have two IPv4 IPs of the same provider
+					// already in the set, delay the IPv6 queries
+					// until halfway through the timeout (so wait
+					// 2.5 seconds). Even the network is IPv6-only,
+					// the DoH dialer will fallback to IPv6
+					// immediately anyway.
+					startDelay = responseTimeout / 2
+				} else if total4 == 1 {
+					startDelay += wellKnownHostBackupDelay
+				}
+				if done[key6] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			}
+			done[hostAndFam{host, ip.BitLen()}]++
+		}
+		rr[i] = resolverAndDelay{
+			ipp:        ipp,
+			startDelay: startDelay,
+		}
+	}
+	return rr
+}
+
+// setRoutes sets the routes to use for DNS forwarding. It's called by
+// Resolver.SetConfig on reconfig.
+//
+// The memory referenced by routesBySuffix should not be modified.
+func (f *forwarder) setRoutes(routesBySuffix map[dnsname.FQDN][]netaddr.IPPort) {
+	routes := make([]route, 0, len(routesBySuffix))
+	for suffix, ipps := range routesBySuffix {
+		routes = append(routes, route{
+			Suffix:    suffix,
+			Resolvers: resolversWithDelays(ipps),
+		})
+	}
+	// Sort from longest prefix to shortest.
+	sort.Slice(routes, func(i, j int) bool {
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
+	})
+
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.routes = routes
@@ -243,7 +349,16 @@ func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client,
 				if !strings.HasPrefix(netw, "tcp") {
 					return nil, fmt.Errorf("unexpected network %q", netw)
 				}
-				return nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
+				c, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
+				// If v4 failed, try an equivalent v6 also in the time remaining.
+				if err != nil && ctx.Err() == nil {
+					if ip6, ok := dohV6(urlBase); ok && ip.Is4() {
+						if c6, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip6.String(), "443")); err == nil {
+							return c6, nil
+						}
+					}
+				}
+				return c, err
 			},
 		},
 	}
@@ -253,7 +368,22 @@ func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client,
 
 const dohType = "application/dns-message"
 
+func (f *forwarder) releaseDoHSem() { <-f.dohSem }
+
 func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client, packet []byte) ([]byte, error) {
+	// Bound the number of HTTP requests in flight. This primarily
+	// matters for iOS where we're very memory constrained and
+	// HTTP requests are heavier on iOS where we don't include
+	// HTTP/2 for binary size reasons (as binaries on iOS linked
+	// with Go code cost memory proportional to the binary size,
+	// for reasons not fully understood).
+	select {
+	case f.dohSem <- struct{}{}:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	defer f.releaseDoHSem()
+
 	req, err := http.NewRequestWithContext(ctx, "POST", urlBase, bytes.NewReader(packet))
 	if err != nil {
 		return nil, err
@@ -283,22 +413,19 @@ func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client,
 //
 // send expects the reply to have the same txid as txidOut.
 //
-// The provided closeOnCtxDone lets send register values to Close if
-// the caller's ctx expires. This avoids send from allocating its own
-// waiting goroutine to interrupt the ReadFrom, as memory is tight on
-// iOS and we want the number of pending DNS lookups to be bursty
-// without too much associated goroutine/memory cost.
-func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
+func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPort) ([]byte, error) {
+	ip := dst.IP()
+
 	// Upgrade known DNS IPs to DoH (DNS-over-HTTPs).
-	if urlBase, dc, ok := f.getDoHClient(dst.IP()); ok {
-		res, err := f.sendDoH(ctx, urlBase, dc, packet)
+	if urlBase, dc, ok := f.getDoHClient(ip); ok {
+		res, err := f.sendDoH(ctx, urlBase, dc, fq.packet)
 		if err == nil || ctx.Err() != nil {
 			return res, err
 		}
-		f.logf("DoH error from %v: %v", dst.IP, err)
+		f.logf("DoH error from %v: %v", ip, err)
 	}
 
-	ln, err := f.packetListener(dst.IP())
+	ln, err := f.packetListener(ip)
 	if err != nil {
 		return nil, err
 	}
@@ -309,10 +436,10 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	defer conn.Close()
 
-	closeOnCtxDone.Add(conn)
-	defer closeOnCtxDone.Remove(conn)
+	fq.closeOnCtxDone.Add(conn)
+	defer fq.closeOnCtxDone.Remove(conn)
 
-	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
+	if _, err := conn.WriteTo(fq.packet, dst.UDPAddr()); err != nil {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
@@ -341,7 +468,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	out = out[:n]
 	txid := getTxID(out)
-	if txid != txidOut {
+	if txid != fq.txid {
 		return nil, errors.New("txid doesn't match")
 	}
 
@@ -364,7 +491,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 }
 
 // resolvers returns the resolvers to use for domain.
-func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
+func (f *forwarder) resolvers(domain dnsname.FQDN) []resolverAndDelay {
 	f.mu.Lock()
 	routes := f.routes
 	f.mu.Unlock()
@@ -376,6 +503,30 @@ func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
 	return nil
 }
 
+// forwardQuery is information and state about a forwarded DNS query that's
+// being sent to 1 or more upstreams.
+//
+// In the case of racing against multiple equivalent upstreams
+// (e.g. Google or CloudFlare's 4 DNS IPs: 2 IPv4 + 2 IPv6), this type
+// handles racing them more intelligently than just blasting away 4
+// queries at once.
+type forwardQuery struct {
+	txid   txid
+	packet []byte
+
+	// closeOnCtxDone lets send register values to Close if the
+	// caller's ctx expires. This avoids send from allocating its
+	// own waiting goroutine to interrupt the ReadFrom, as memory
+	// is tight on iOS and we want the number of pending DNS
+	// lookups to be bursty without too much associated
+	// goroutine/memory cost.
+	closeOnCtxDone *closePool
+
+	// TODO(bradfitz): add race delay state:
+	// mu sync.Mutex
+	// ...
+}
+
 // forward forwards the query to all upstream nameservers and returns the first response.
 func (f *forwarder) forward(query packet) error {
 	domain, err := nameFromQuery(query.bs)
@@ -383,7 +534,6 @@ func (f *forwarder) forward(query packet) error {
 		return err
 	}
 
-	txid := getTxID(query.bs)
 	clampEDNSSize(query.bs, maxResponseBytes)
 
 	resolvers := f.resolvers(domain)
@@ -391,8 +541,12 @@ func (f *forwarder) forward(query packet) error {
 		return errNoUpstreams
 	}
 
-	closeOnCtxDone := new(closePool)
-	defer closeOnCtxDone.Close()
+	fq := &forwardQuery{
+		txid:           getTxID(query.bs),
+		packet:         query.bs,
+		closeOnCtxDone: new(closePool),
+	}
+	defer fq.closeOnCtxDone.Close()
 
 	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
 	defer cancel()
@@ -403,9 +557,18 @@ func (f *forwarder) forward(query packet) error {
 		firstErr error
 	)
 
-	for _, ipp := range resolvers {
-		go func(ipp netaddr.IPPort) {
-			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
+	for _, rr := range resolvers {
+		go func(rr resolverAndDelay) {
+			if rr.startDelay > 0 {
+				timer := time.NewTimer(rr.startDelay)
+				select {
+				case <-timer.C:
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			resb, err := f.send(ctx, fq, rr.ipp)
 			if err != nil {
 				mu.Lock()
 				defer mu.Unlock()
@@ -418,7 +581,7 @@ func (f *forwarder) forward(query packet) error {
 			case resc <- resb:
 			default:
 			}
-		}(ipp)
+		}(rr)
 	}
 
 	select {
@@ -509,7 +672,22 @@ func (p *closePool) Close() error {
 
 var knownDoH = map[netaddr.IP]string{}
 
-func addDoH(ip, base string) { knownDoH[netaddr.MustParseIP(ip)] = base }
+var dohIPsOfBase = map[string][]netaddr.IP{}
+
+func addDoH(ipStr, base string) {
+	ip := netaddr.MustParseIP(ipStr)
+	knownDoH[ip] = base
+	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
+}
+
+func dohV6(base string) (ip netaddr.IP, ok bool) {
+	for _, ip := range dohIPsOfBase[base] {
+		if ip.Is6() {
+			return ip, true
+		}
+	}
+	return ip, false
+}
 
 func init() {
 	// Cloudflare
@@ -519,16 +697,16 @@ func init() {
 	addDoH("2606:4700:4700::1001", "https://cloudflare-dns.com/dns-query")
 
 	// Cloudflare -Malware
-	addDoH("1.1.1.2", "https://cloudflare-dns.com/dns-query")
-	addDoH("1.0.0.2", "https://cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1112", "https://cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1002", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.1.1.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1112", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1002", "https://security.cloudflare-dns.com/dns-query")
 
 	// Cloudflare -Malware -Adult
-	addDoH("1.1.1.3", "https://cloudflare-dns.com/dns-query")
-	addDoH("1.0.0.3", "https://cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1113", "https://cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1003", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.1.1.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1113", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1003", "https://family.cloudflare-dns.com/dns-query")
 
 	// Google
 	addDoH("8.8.8.8", "https://dns.google/dns-query")

net/dns/resolver/forwarder_test.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+func (rr resolverAndDelay) String() string {
+	return fmt.Sprintf("%v+%v", rr.ipp, rr.startDelay)
+}
+
+func TestResolversWithDelays(t *testing.T) {
+	// query
+	q := func(ss ...string) (ipps []netaddr.IPPort) {
+		for _, s := range ss {
+			ipps = append(ipps, netaddr.MustParseIPPort(s))
+		}
+		return
+	}
+	// output
+	o := func(ss ...string) (rr []resolverAndDelay) {
+		for _, s := range ss {
+			var d time.Duration
+			if i := strings.Index(s, "+"); i != -1 {
+				var err error
+				d, err = time.ParseDuration(s[i+1:])
+				if err != nil {
+					panic(fmt.Sprintf("parsing duration in %q: %v", s, err))
+				}
+				s = s[:i]
+			}
+			rr = append(rr, resolverAndDelay{
+				ipp:        netaddr.MustParseIPPort(s),
+				startDelay: d,
+			})
+		}
+		return
+	}
+
+	tests := []struct {
+		name string
+		in   []netaddr.IPPort
+		want []resolverAndDelay
+	}{
+		{
+			name: "unknown-no-delays",
+			in:   q("1.2.3.4:53", "2.3.4.5:53"),
+			want: o("1.2.3.4:53", "2.3.4.5:53"),
+		},
+		{
+			name: "google-all-ipv4",
+			in:   q("8.8.8.8:53", "8.8.4.4:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms"),
+		},
+		{
+			name: "google-only-ipv6",
+			in:   q("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53+200ms"),
+		},
+		{
+			name: "google-all-four",
+			in:   q("8.8.8.8:53", "8.8.4.4:53", "[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms", "[2001:4860:4860::8888]:53+2.5s", "[2001:4860:4860::8844]:53+2.7s"),
+		},
+		{
+			name: "quad9-one-v4-one-v6",
+			in:   q("9.9.9.9:53", "[2620:fe::fe]:53"),
+			want: o("9.9.9.9:53", "[2620:fe::fe]:53+200ms"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := resolversWithDelays(tt.in)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+
+}

net/dns/resolver/tsdns.go

@@ -7,8 +7,10 @@
 package resolver
 
 import (
+	"bufio"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"runtime"
 	"sort"
 	"strings"
@@ -79,6 +81,74 @@ type Config struct {
 	LocalDomains []dnsname.FQDN
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{Routes:")
+	WriteRoutes(w, c.Routes)
+	fmt.Fprintf(w, " Hosts:%v LocalDomains:[", len(c.Hosts))
+	space := false
+	arpa := 0
+	for _, d := range c.LocalDomains {
+		if strings.HasSuffix(string(d), ".arpa.") {
+			arpa++
+			continue
+		}
+		if space {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(d))
+		space = true
+	}
+	w.WriteString("]")
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+	w.WriteString("}")
+}
+
+// WriteIPPorts writes vv to w.
+func WriteIPPorts(w *bufio.Writer, vv []netaddr.IPPort) {
+	w.WriteByte('[')
+	var b []byte
+	for i, v := range vv {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		b = v.AppendTo(b[:0])
+		w.Write(b)
+	}
+	w.WriteByte(']')
+}
+
+// WriteRoutes writes routes to w, omitting *.arpa routes and instead
+// summarizing how many of them there were.
+func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]netaddr.IPPort) {
+	var kk []dnsname.FQDN
+	arpa := 0
+	for k := range routes {
+		if strings.HasSuffix(string(k), ".arpa.") {
+			arpa++
+			continue
+		}
+		kk = append(kk, k)
+	}
+	sort.Slice(kk, func(i, j int) bool { return kk[i] < kk[j] })
+	w.WriteByte('{')
+	for i, k := range kk {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(k))
+		w.WriteByte(':')
+		WriteIPPorts(w, routes[k])
+	}
+	w.WriteByte('}')
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+}
+
 // Resolver is a DNS resolver for nodes on the Tailscale network,
 // associating them with domain names of the form <mynode>.<mydomain>.<root>.
 // If it is asked to resolve a domain that is not of that form,
@@ -138,7 +208,6 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		r.saveConfigForTests(cfg)
 	}
 
-	routes := make([]route, 0, len(cfg.Routes))
 	reverse := make(map[netaddr.IP]dnsname.FQDN, len(cfg.Hosts))
 
 	for host, ips := range cfg.Hosts {
@@ -147,18 +216,7 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		}
 	}
 
-	for suffix, ips := range cfg.Routes {
-		routes = append(routes, route{
-			Suffix:    suffix,
-			Resolvers: ips,
-		})
-	}
-	// Sort from longest prefix to shortest.
-	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
-	})
-
-	r.forwarder.setRoutes(routes)
+	r.forwarder.setRoutes(cfg.Routes)
 
 	r.mu.Lock()
 	defer r.mu.Unlock()

net/dns/resolver/tsdns_test.go

@@ -931,8 +931,11 @@ func TestAllocs(t *testing.T) {
 		query []byte
 		want  int
 	}{
-		// Name lowercasing and response slice created by dns.NewBuilder.
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
+		// Name lowercasing, response slice created by dns.NewBuilder,
+		// and closure allocation from go call.
+		// (Closure allocation only happens when using new register ABI,
+		// which is amd64 with Go 1.17, and probably more platforms later.)
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 3},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
 		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}

net/dnscache/dnscache.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// TODO(bradfitz): update this code to use netaddr more
+
 // Package dnscache contains a minimal DNS cache that makes a bunch of
 // assumptions that are only valid for us. Not recommended for general use.
 package dnscache
@@ -78,8 +80,9 @@ type Resolver struct {
 }
 
 type ipCacheEntry struct {
-	ip      net.IP // either v4 or v6
-	ip6     net.IP // nil if no v4 or no v6
+	ip      net.IP       // either v4 or v6
+	ip6     net.IP       // nil if no v4 or no v6
+	allIPs  []net.IPAddr // 1+ v4 and/or v6
 	expires time.Time
 }
 
@@ -105,81 +108,82 @@ var debug, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DNS_CACHE"))
 //
 // If err is nil, ip will be non-nil. The v6 address may be nil even
 // with a nil error.
-func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, err error) {
+func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, allIPs []net.IPAddr, err error) {
 	if ip := net.ParseIP(host); ip != nil {
 		if ip4 := ip.To4(); ip4 != nil {
-			return ip4, nil, nil
+			return ip4, nil, []net.IPAddr{{IP: ip4}}, nil
 		}
 		if debug {
 			log.Printf("dnscache: %q is an IP", host)
 		}
-		return ip, nil, nil
+		return ip, nil, []net.IPAddr{{IP: ip}}, nil
 	}
 
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q = %v (cached)", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
-	type ipPair struct {
+	type ipRes struct {
 		ip, ip6 net.IP
+		allIPs  []net.IPAddr
 	}
 	ch := r.sf.DoChan(host, func() (interface{}, error) {
-		ip, ip6, err := r.lookupIP(host)
+		ip, ip6, allIPs, err := r.lookupIP(host)
 		if err != nil {
 			return nil, err
 		}
-		return ipPair{ip, ip6}, nil
+		return ipRes{ip, ip6, allIPs}, nil
 	})
 	select {
 	case res := <-ch:
 		if res.Err != nil {
 			if r.UseLastGood {
-				if ip, ip6, ok := r.lookupIPCacheExpired(host); ok {
+				if ip, ip6, allIPs, ok := r.lookupIPCacheExpired(host); ok {
 					if debug {
 						log.Printf("dnscache: %q using %v after error", host, ip)
 					}
-					return ip, ip6, nil
+					return ip, ip6, allIPs, nil
 				}
 			}
 			if debug {
 				log.Printf("dnscache: error resolving %q: %v", host, res.Err)
 			}
-			return nil, nil, res.Err
+			return nil, nil, nil, res.Err
 		}
-		pair := res.Val.(ipPair)
-		return pair.ip, pair.ip6, nil
+		r := res.Val.(ipRes)
+		return r.ip, r.ip6, r.allIPs, nil
 	case <-ctx.Done():
 		if debug {
 			log.Printf("dnscache: context done while resolving %q: %v", host, ctx.Err())
 		}
-		return nil, nil, ctx.Err()
+		return nil, nil, nil, ctx.Err()
 	}
 }
 
-func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok && ent.expires.After(time.Now()) {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
-func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
 func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	if r.UseLastGood {
-		if _, _, ok := r.lookupIPCacheExpired(host); ok {
+		if _, _, _, ok := r.lookupIPCacheExpired(host); ok {
 			// If we have some previous good value for this host,
 			// don't give this DNS lookup much time. If we're in a
 			// situation where the user's DNS server is unreachable
@@ -194,12 +198,12 @@ func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	return 10 * time.Second
 }
 
-func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, err error) {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q found in cache as %v", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), r.lookupTimeoutForHost(host))
@@ -218,10 +222,10 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 		}
 	}
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	if len(ips) == 0 {
-		return nil, nil, fmt.Errorf("no IPs for %q found", host)
+		return nil, nil, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
 	have4 := false
@@ -240,12 +244,12 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 			}
 		}
 	}
-	r.addIPCache(host, ip, ip6, r.ttl())
-	return ip, ip6, nil
+	r.addIPCache(host, ip, ip6, ips, r.ttl())
+	return ip, ip6, ips, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
-	if isPrivateIP(ip) {
+func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr, d time.Duration) {
+	if naIP, _ := netaddr.FromStdIP(ip); naIP.IsPrivate() {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
 		if debug {
@@ -263,27 +267,14 @@ func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
 	if r.ipCache == nil {
 		r.ipCache = make(map[string]ipCacheEntry)
 	}
-	r.ipCache[host] = ipCacheEntry{ip: ip, ip6: ip6, expires: time.Now().Add(d)}
-}
-
-func mustCIDR(s string) *net.IPNet {
-	_, ipNet, err := net.ParseCIDR(s)
-	if err != nil {
-		panic(err)
+	r.ipCache[host] = ipCacheEntry{
+		ip:      ip,
+		ip6:     ip6,
+		allIPs:  allIPs,
+		expires: time.Now().Add(d),
 	}
-	return ipNet
 }
 
-func isPrivateIP(ip net.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
-var (
-	private1 = mustCIDR("10.0.0.0/8")
-	private2 = mustCIDR("172.16.0.0/12")
-	private3 = mustCIDR("192.168.0.0/16")
-)
-
 type DialContextFunc func(ctx context.Context, network, address string) (net.Conn, error)
 
 // Dialer returns a wrapped DialContext func that uses the provided dnsCache.
@@ -305,41 +296,131 @@ func Dialer(fwd DialContextFunc, dnsCache *Resolver) DialContextFunc {
 				// Return with original error
 				return
 			}
-			for _, ip := range ips {
-				dst := net.JoinHostPort(ip.String(), port)
-				if c, err := fwd(ctx, network, dst); err == nil {
-					retConn = c
-					ret = nil
-					return
-				}
+			if c, err := raceDial(ctx, fwd, network, ips, port); err == nil {
+				retConn = c
+				ret = nil
+				return
 			}
 		}()
 
-		ip, ip6, err := dnsCache.LookupIP(ctx, host)
+		ip, ip6, allIPs, err := dnsCache.LookupIP(ctx, host)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve %q: %w", host, err)
 		}
-		dst := net.JoinHostPort(ip.String(), port)
-		if debug {
-			log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+		i4s := v4addrs(allIPs)
+		if len(i4s) < 2 {
+			dst := net.JoinHostPort(ip.String(), port)
+			if debug {
+				log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+			}
+			c, err := fwd(ctx, network, dst)
+			if err == nil || ctx.Err() != nil || ip6 == nil {
+				return c, err
+			}
+			// Fall back to trying IPv6.
+			dst = net.JoinHostPort(ip6.String(), port)
+			return fwd(ctx, network, dst)
 		}
-		c, err := fwd(ctx, network, dst)
-		if err == nil || ctx.Err() != nil || ip6 == nil {
-			return c, err
-		}
-		// Fall back to trying IPv6.
-		// TODO(bradfitz): this is a primarily for IPv6-only
-		// hosts; it's not supposed to be a real Happy
-		// Eyeballs implementation. We should use the net
-		// package's implementation of that by plumbing this
-		// dnscache impl into net.Dialer.Resolver.Dial and
-		// unmarshal/marshal DNS queries/responses to the net
-		// package. This works for v6-only hosts for now.
-		dst = net.JoinHostPort(ip6.String(), port)
-		return fwd(ctx, network, dst)
+
+		// Multiple IPv4 candidates, and 0+ IPv6.
+		ipsToTry := append(i4s, v6addrs(allIPs)...)
+		return raceDial(ctx, fwd, network, ipsToTry, port)
 	}
 }
 
+// fallbackDelay is how long to wait between trying subsequent
+// addresses when multiple options are available.
+// 300ms is the same as Go's Happy Eyeballs fallbackDelay value.
+const fallbackDelay = 300 * time.Millisecond
+
+// raceDial tries to dial port on each ip in ips, starting a new race
+// dial every fallbackDelay apart, returning whichever completes first.
+func raceDial(ctx context.Context, fwd DialContextFunc, network string, ips []netaddr.IP, port string) (net.Conn, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	type res struct {
+		c   net.Conn
+		err error
+	}
+	resc := make(chan res)           // must be unbuffered
+	failBoost := make(chan struct{}) // best effort send on dial failure
+
+	go func() {
+		for i, ip := range ips {
+			if i != 0 {
+				timer := time.NewTimer(fallbackDelay)
+				select {
+				case <-timer.C:
+				case <-failBoost:
+					timer.Stop()
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			go func(ip netaddr.IP) {
+				c, err := fwd(ctx, network, net.JoinHostPort(ip.String(), port))
+				if err != nil {
+					// Best effort wake-up a pending dial.
+					// e.g. IPv4 dials failing quickly on an IPv6-only system.
+					// In that case we don't want to wait 300ms per IPv4 before
+					// we get to the IPv6 addresses.
+					select {
+					case failBoost <- struct{}{}:
+					default:
+					}
+				}
+				select {
+				case resc <- res{c, err}:
+				case <-ctx.Done():
+					if c != nil {
+						c.Close()
+					}
+				}
+			}(ip)
+		}
+	}()
+
+	var firstErr error
+	var fails int
+	for {
+		select {
+		case r := <-resc:
+			if r.c != nil {
+				return r.c, nil
+			}
+			fails++
+			if firstErr == nil {
+				firstErr = r.err
+			}
+			if fails == len(ips) {
+				return nil, firstErr
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+func v4addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is4() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
+func v6addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is6() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
 var errTLSHandshakeTimeout = errors.New("timeout doing TLS handshake")
 
 // TLSDialer is like Dialer but returns a func suitable for using with net/http.Transport.DialTLSContext.

net/dnscache/dnscache_test.go

@@ -5,24 +5,29 @@
 package dnscache
 
 import (
+	"context"
+	"flag"
 	"net"
 	"testing"
+	"time"
 )
 
-func TestIsPrivateIP(t *testing.T) {
-	tests := []struct {
-		ip   string
-		want bool
-	}{
-		{"10.1.2.3", true},
-		{"172.16.1.100", true},
-		{"192.168.1.1", true},
-		{"1.2.3.4", false},
-	}
+var dialTest = flag.String("dial-test", "", "if non-empty, addr:port to test dial")
 
-	for _, test := range tests {
-		if got := isPrivateIP(net.ParseIP(test.ip)); got != test.want {
-			t.Errorf("isPrivateIP(%q)=%v, want %v", test.ip, got, test.want)
-		}
+func TestDialer(t *testing.T) {
+	if *dialTest == "" {
+		t.Skip("skipping; --dial-test is blank")
 	}
+	r := new(Resolver)
+	var std net.Dialer
+	dialer := Dialer(std.DialContext, r)
+	t0 := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	c, err := dialer(ctx, "tcp", *dialTest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("dialed in %v", time.Since(t0))
+	c.Close()
 }

net/dnsfallback/dns-fallback-servers.json

@@ -90,18 +90,25 @@
 			"RegionName": "r4",
 			"Nodes": [
 				{
-					"Name": "4a",
+					"Name": "4c",
 					"RegionID": 4,
-					"HostName": "derp4.tailscale.com",
-					"IPv4": "167.172.182.26",
-					"IPv6": "2a03:b0c0:3:e0::36e:9001"
+					"HostName": "derp4c.tailscale.com",
+					"IPv4": "134.122.77.138",
+					"IPv6": "2a03:b0c0:3:d0::1501:6001"
 				},
 				{
-					"Name": "4b",
+					"Name": "4d",
 					"RegionID": 4,
-					"HostName": "derp4b.tailscale.com",
-					"IPv4": "157.230.25.0",
-					"IPv6": "2a03:b0c0:3:e0::58f:3001"
+					"HostName": "derp4d.tailscale.com",
+					"IPv4": "134.122.94.167",
+					"IPv6": "2a03:b0c0:3:d0::1501:b001"
+				},
+				{
+					"Name": "4e",
+					"RegionID": 4,
+					"HostName": "derp4e.tailscale.com",
+					"IPv4": "134.122.74.153",
+					"IPv6": "2a03:b0c0:3:d0::29:9001"
 				}
 			]
 		},
@@ -153,11 +160,25 @@
 			"RegionName": "r8",
 			"Nodes": [
 				{
-					"Name": "8a",
+					"Name": "8b",
 					"RegionID": 8,
-					"HostName": "derp8.tailscale.com",
-					"IPv4": "167.71.139.179",
-					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
+					"HostName": "derp8b.tailscale.com",
+					"IPv4": "46.101.74.201",
+					"IPv6": "2a03:b0c0:1:d0::ec1:e001"
+				},
+				{
+					"Name": "8c",
+					"RegionID": 8,
+					"HostName": "derp8c.tailscale.com",
+					"IPv4": "206.189.16.32",
+					"IPv6": "2a03:b0c0:1:d0::e1f:4001"
+				},
+				{
+					"Name": "8d",
+					"RegionID": 8,
+					"HostName": "derp8d.tailscale.com",
+					"IPv4": "178.62.44.132",
+					"IPv6": "2a03:b0c0:1:d0::e08:e001"
 				}
 			]
 		},

net/interfaces/interfaces.go

@@ -134,7 +134,7 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 					// but their OS supports IPv6 so they have an fe80::
 					// address. We don't want to report all of those
 					// IPv6 LL to Control.
-				} else if ip.Is6() && tsaddr.IsULA(ip) {
+				} else if ip.Is6() && ip.IsPrivate() {
 					// Google Cloud Run uses NAT with IPv6 Unique
 					// Local Addresses to provide IPv6 connectivity.
 					ula6 = append(ula6, ip)
@@ -479,7 +479,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
 		ip := pfx.IP()
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			if privateIP == "" {
 				privateIP = ip.String()
 			}
@@ -519,21 +519,15 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
-		for _, prefix := range privatev4s {
-			if prefix.Contains(gateway) && prefix.Contains(ip) {
-				myIP = ip
-				ok = true
-				return
-			}
+		if gateway.IsPrivate() && ip.IsPrivate() {
+			myIP = ip
+			ok = true
+			return
 		}
 	})
 	return gateway, myIP, !myIP.IsZero()
 }
 
-func isPrivateIP(ip netaddr.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
 // isUsableV4 reports whether ip is a usable IPv4 address which could
 // conceivably be used to get Internet connectivity. Globally routable and
 // private IPv4 addresses are always Usable, and link local 169.254.x.x
@@ -554,23 +548,11 @@ func isUsableV4(ip netaddr.IP) bool {
 // (fc00::/7) are in some environments used with address translation.
 func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
-		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
-}
-
-func mustCIDR(s string) netaddr.IPPrefix {
-	prefix, err := netaddr.ParseIPPrefix(s)
-	if err != nil {
-		panic(err)
-	}
-	return prefix
+		(ip.Is6() && ip.IsPrivate() && !tsaddr.TailscaleULARange().Contains(ip))
 }
 
 var (
-	private1   = mustCIDR("10.0.0.0/8")
-	private2   = mustCIDR("172.16.0.0/12")
-	private3   = mustCIDR("192.168.0.0/16")
-	privatev4s = []netaddr.IPPrefix{private1, private2, private3}
-	v6Global1  = mustCIDR("2000::/3")
+	v6Global1 = netaddr.MustParseIPPrefix("2000::/3")
 )
 
 // anyInterestingIP reports whether pfxs contains any IP that matches

net/interfaces/interfaces_darwin_test.go

@@ -73,7 +73,7 @@ func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
 			return nil
 		}
 		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
-		if err == nil && isPrivateIP(ip) {
+		if err == nil && ip.IsPrivate() {
 			ret = ip
 			// We've found what we're looking for.
 			return errStopReadingNetstatTable

net/interfaces/interfaces_linux.go

@@ -72,7 +72,7 @@ func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
 			return nil // ignore error, skip line and keep going
 		}
 		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			ret = ip
 		}
 		return nil

net/interfaces/interfaces_test.go

@@ -58,6 +58,8 @@ func TestIsUsableV6(t *testing.T) {
 		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
 		{"Link Local", "fe80::1", false},
 		{"Global", "2602::1", true},
+		{"IPv4 public", "192.0.2.1", false},
+		{"IPv4 private", "192.168.1.1", false},
 	}
 
 	for _, test := range tests {

net/interfaces/interfaces_windows.go

@@ -93,7 +93,7 @@ func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
 		}
 	}
 
-	if !ret.IsZero() && !isPrivateIP(ret) {
+	if !ret.IsZero() && !ret.IsPrivate() {
 		// Default route has a non-private gateway
 		return netaddr.IP{}, false
 	}

net/portmapper/disabled_stubs.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"context"
+
+	"inet.af/netaddr"
+)
+
+type upnpClient interface{}
+
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	return netaddr.IPPort{}, false
+}

net/portmapper/portmapper.go

@@ -14,9 +14,11 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"sync"
 	"time"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netns"
@@ -62,8 +64,11 @@ type Client struct {
 	pmpPubIPTime time.Time  // time pmpPubIP last verified
 	pmpLastEpoch uint32
 
-	pcpSawTime  time.Time // time we last saw PCP was available
-	uPnPSawTime time.Time // time we last saw UPnP was available
+	pcpSawTime time.Time // time we last saw PCP was available
+
+	uPnPSawTime    time.Time         // time we last saw UPnP was available
+	uPnPMeta       uPnPDiscoResponse // Location header from UPnP UDP discovery response
+	uPnPHTTPClient *http.Client      // nil until needed
 
 	localPort uint16
 
@@ -270,7 +275,7 @@ func IsNoMappingError(err error) bool {
 
 var (
 	ErrNoPortMappingServices = errors.New("no port mapping services were found")
-	ErrGatewayNotFound       = errors.New("failed to look up gateway address")
+	ErrGatewayRange          = errors.New("skipping portmap; gateway range likely lacks support")
 )
 
 // GetCachedMappingOrStartCreatingOne quickly returns with our current cached portmapping, if any.
@@ -331,7 +336,7 @@ func (c *Client) createMapping() {
 func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return netaddr.IPPort{}, NoMappingError{ErrGatewayNotFound}
+		return netaddr.IPPort{}, NoMappingError{ErrGatewayRange}
 	}
 
 	c.mu.Lock()
@@ -540,7 +545,7 @@ type ProbeResult struct {
 func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return res, ErrGatewayNotFound
+		return res, ErrGatewayRange
 	}
 	defer func() {
 		if err == nil {
@@ -560,27 +565,9 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	defer cancel()
 	defer closeCloserOnContextDone(ctx, uc)()
 
-	if c.sawUPnPRecently() {
-		res.UPnP = true
-	} else {
-		hasUPnP := make(chan bool, 1)
-		defer func() {
-			res.UPnP = <-hasUPnP
-		}()
-		go func() {
-			client, err := getUPnPClient(ctx, gw)
-			if err == nil && client != nil {
-				hasUPnP <- true
-				c.mu.Lock()
-				c.uPnPSawTime = time.Now()
-				c.mu.Unlock()
-			}
-			close(hasUPnP)
-		}()
-	}
-
 	pcpAddr := netaddr.IPPortFrom(gw, pcpPort).UDPAddr()
 	pmpAddr := netaddr.IPPortFrom(gw, pmpPort).UDPAddr()
+	upnpAddr := netaddr.IPPortFrom(gw, upnpPort).UDPAddr()
 
 	// Don't send probes to services that we recently learned (for
 	// the same gw/myIP) are available. See
@@ -595,11 +582,16 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	} else {
 		uc.WriteTo(pcpAnnounceRequest(myIP), pcpAddr)
 	}
+	if c.sawUPnPRecently() {
+		res.UPnP = true
+	} else {
+		uc.WriteTo(uPnPPacket, upnpAddr)
+	}
 
 	buf := make([]byte, 1500)
 	pcpHeard := false // true when we get any PCP response
 	for {
-		if pcpHeard && res.PMP {
+		if pcpHeard && res.PMP && res.UPnP {
 			// Nothing more to discover.
 			return res, nil
 		}
@@ -612,6 +604,19 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 		}
 		port := addr.(*net.UDPAddr).Port
 		switch port {
+		case upnpPort:
+			if mem.Contains(mem.B(buf[:n]), mem.S(":InternetGatewayDevice:")) {
+				meta, err := parseUPnPDiscoResponse(buf[:n])
+				if err != nil {
+					c.logf("unrecognized UPnP discovery response; ignoring")
+				}
+				// log.Printf("UPnP reply %+v, %q", meta, buf[:n])
+				res.UPnP = true
+				c.mu.Lock()
+				c.uPnPSawTime = time.Now()
+				c.uPnPMeta = meta
+				c.mu.Unlock()
+			}
 		case pcpPort: // same as pmpPort
 			if pres, ok := parsePCPResponse(buf[:n]); ok {
 				if pres.OpCode == pcpOpReply|pcpOpAnnounce {
@@ -724,3 +729,13 @@ func parsePCPResponse(b []byte) (res pcpResponse, ok bool) {
 }
 
 var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
+const (
+	upnpPort = 1900
+)
+
+var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
+	"HOST: 239.255.255.250:1900\r\n" +
+	"ST: ssdp:all\r\n" +
+	"MAN: \"ssdp:discover\"\r\n" +
+	"MX: 2\r\n\r\n")

net/portmapper/upnp.go

@@ -2,19 +2,33 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build !ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
 package portmapper
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
 	"net/url"
 	"time"
 
+	"github.com/tailscale/goupnp"
 	"github.com/tailscale/goupnp/dcps/internetgateway2"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlknobs"
+	"tailscale.com/net/netns"
 )
 
+// VerboseLogs controls verbose debug logging.
+// It exists for use by "tailscaled debug --portmap".
+var VerboseLogs bool
+
 // References:
 //
 // WANIP Connection v2: http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v2-Service.pdf
@@ -40,7 +54,8 @@ func (u *upnpMapping) Release(ctx context.Context) {
 }
 
 // upnpClient is an interface over the multiple different clients exported by goupnp,
-// exposing the functions we need for portmapping. They are auto-generated from XML-specs.
+// exposing the functions we need for portmapping. Those clients are auto-generated from XML-specs,
+// which is why they're not very idiomatic.
 type upnpClient interface {
 	AddPortMapping(
 		ctx context.Context,
@@ -73,7 +88,7 @@ type upnpClient interface {
 		// greater than 0. From the spec, it appears if it is set to 0, it will switch to using
 		// 604800 seconds, but not sure why this is desired. The recommended time is 3600 seconds.
 		leaseDurationSec uint32,
-	) (err error)
+	) error
 
 	DeletePortMapping(ctx context.Context, remoteHost string, externalPort uint16, protocol string) error
 	GetExternalIPAddress(ctx context.Context) (externalIPAddress string, err error)
@@ -86,6 +101,10 @@ const tsPortMappingDesc = "tailscale-portmap"
 // addAnyPortMapping abstracts over different UPnP client connections, calling the available
 // AddAnyPortMapping call if available for WAN IP connection v2, otherwise defaulting to the old
 // behavior of calling AddPortMapping with port = 0 to specify a wildcard port.
+// It returns the new external port (which may not be identical to the external port specified),
+// or an error.
+//
+// TODO(bradfitz): also returned the actual lease duration obtained. and check it regularly.
 func addAnyPortMapping(
 	ctx context.Context,
 	upnp upnpClient,
@@ -107,6 +126,9 @@ func addAnyPortMapping(
 			uint32(leaseDuration.Seconds()),
 		)
 	}
+	for externalPort == 0 {
+		externalPort = uint16(rand.Intn(65535))
+	}
 	err = upnp.AddPortMapping(
 		ctx,
 		"",
@@ -118,54 +140,79 @@ func addAnyPortMapping(
 		tsPortMappingDesc,
 		uint32(leaseDuration.Seconds()),
 	)
-	return internalPort, err
+	return externalPort, err
 }
 
-// getUPnPClients gets a client for interfacing with UPnP, ignoring the underlying protocol for
+// getUPnPClient gets a client for interfacing with UPnP, ignoring the underlying protocol for
 // now.
 // Adapted from https://github.com/huin/goupnp/blob/master/GUIDE.md.
-func getUPnPClient(ctx context.Context, gw netaddr.IP) (upnpClient, error) {
+//
+// The gw is the detected gateway.
+//
+// The meta is the most recently parsed UDP discovery packet response
+// from the Internet Gateway Device.
+//
+// The provided ctx is not retained in the returned upnpClient, but
+// its associated HTTP client is (if set via goupnp.WithHTTPClient).
+func getUPnPClient(ctx context.Context, gw netaddr.IP, meta uPnPDiscoResponse) (upnpClient, error) {
 	if controlknobs.DisableUPnP() {
 		return nil, nil
 	}
-	ctx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
-	defer cancel()
-	// Attempt to connect over the multiple available connection types concurrently,
-	// returning the fastest.
 
-	// TODO(jknodt): this url seems super brittle? maybe discovery is better but this is faster
-	u, err := url.Parse(fmt.Sprintf("http://%s:5000/rootDesc.xml", gw))
+	if meta.Location == "" {
+		return nil, nil
+	}
+
+	if VerboseLogs {
+		log.Printf("fetching %v", meta.Location)
+	}
+	u, err := url.Parse(meta.Location)
 	if err != nil {
 		return nil, err
 	}
 
-	clients := make(chan upnpClient, 3)
-	go func() {
-		var err error
-		ip1Clients, err := internetgateway2.NewWANIPConnection1ClientsByURL(ctx, u)
-		if err == nil && len(ip1Clients) > 0 {
-			clients <- ip1Clients[0]
-		}
-	}()
-	go func() {
-		ip2Clients, err := internetgateway2.NewWANIPConnection2ClientsByURL(ctx, u)
-		if err == nil && len(ip2Clients) > 0 {
-			clients <- ip2Clients[0]
-		}
-	}()
-	go func() {
-		ppp1Clients, err := internetgateway2.NewWANPPPConnection1ClientsByURL(ctx, u)
-		if err == nil && len(ppp1Clients) > 0 {
-			clients <- ppp1Clients[0]
-		}
-	}()
-
-	select {
-	case client := <-clients:
-		return client, nil
-	case <-ctx.Done():
-		return nil, ctx.Err()
+	ipp, err := netaddr.ParseIPPort(u.Host)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected host %q in %q", u.Host, meta.Location)
 	}
+	if ipp.IP() != gw {
+		return nil, fmt.Errorf("UPnP discovered root %q does not match gateway IP %v; ignoring UPnP",
+			meta.Location, gw)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+	defer cancel()
+
+	// This part does a network fetch.
+	root, err := goupnp.DeviceByURL(ctx, u)
+	if err != nil {
+		return nil, err
+	}
+
+	// These parts don't do a network fetch.
+	// Pick the best service type available.
+	if cc, _ := internetgateway2.NewWANIPConnection2ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANIPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANPPPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	return nil, nil
+}
+
+func (c *Client) upnpHTTPClientLocked() *http.Client {
+	if c.uPnPHTTPClient == nil {
+		c.uPnPHTTPClient = &http.Client{
+			Transport: &http.Transport{
+				DialContext:     netns.NewDialer().DialContext,
+				IdleConnTimeout: 2 * time.Second, // LAN is cheap
+			},
+		}
+	}
+	return c.uPnPHTTPClient
 }
 
 // getUPnPPortMapping attempts to create a port-mapping over the UPnP protocol. On success,
@@ -190,11 +237,17 @@ func (c *Client) getUPnPPortMapping(
 	var err error
 	c.mu.Lock()
 	oldMapping, ok := c.mapping.(*upnpMapping)
+	meta := c.uPnPMeta
+	httpClient := c.upnpHTTPClientLocked()
 	c.mu.Unlock()
 	if ok && oldMapping != nil {
 		client = oldMapping.client
 	} else {
-		client, err = getUPnPClient(ctx, gw)
+		ctx := goupnp.WithHTTPClient(ctx, httpClient)
+		client, err = getUPnPClient(ctx, gw, meta)
+		if VerboseLogs {
+			log.Printf("getUPnPClient: %T, %v", client, err)
+		}
 		if err != nil {
 			return netaddr.IPPort{}, false
 		}
@@ -212,11 +265,17 @@ func (c *Client) getUPnPPortMapping(
 		internal.IP().String(),
 		time.Second*pmpMapLifetimeSec,
 	)
+	if VerboseLogs {
+		log.Printf("addAnyPortMapping: %v, %v", newPort, err)
+	}
 	if err != nil {
 		return netaddr.IPPort{}, false
 	}
 	// TODO cache this ip somewhere?
 	extIP, err := client.GetExternalIPAddress(ctx)
+	if VerboseLogs {
+		log.Printf("client.GetExternalIPAddress: %v, %v", extIP, err)
+	}
 	if err != nil {
 		// TODO this doesn't seem right
 		return netaddr.IPPort{}, false
@@ -237,3 +296,18 @@ func (c *Client) getUPnPPortMapping(
 	c.localPort = newPort
 	return upnp.external, true
 }
+
+type uPnPDiscoResponse struct {
+	Location string
+}
+
+// parseUPnPDiscoResponse parses a UPnP HTTP-over-UDP discovery response.
+func parseUPnPDiscoResponse(body []byte) (uPnPDiscoResponse, error) {
+	var r uPnPDiscoResponse
+	res, err := http.ReadResponse(bufio.NewReaderSize(bytes.NewReader(body), 128), nil)
+	if err != nil {
+		return r, err
+	}
+	r.Location = res.Header.Get("Location")
+	return r, nil
+}

net/portmapper/upnp_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package portmapper
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"inet.af/netaddr"
+)
+
+// Google Wifi
+const (
+	googleWifiUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nUSN: uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece::urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nEXT:\r\nSERVER: Linux/5.4.0-1034-gcp UPnP/1.1 MiniUPnPd/1.9\r\nLOCATION: http://192.168.86.1:5000/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1\r\nBOOTID.UPNP.ORG: 1\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	googleWifiRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0"><specVersion><major>1</major><minor>0</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:2</deviceType><friendlyName>OnHub</friendlyName><manufacturer>Google</manufacturer><manufacturerURL>http://google.com/</manufacturerURL><modelDescription>Wireless Router</modelDescription><modelName>OnHub</modelName><modelNumber>1</modelNumber><modelURL>https://on.google.com/hub/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:Layer3Forwarding1</serviceId><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL><SCPDURL>/L3F.xml</SCPDURL></service><service><serviceType>urn:schemas-upnp-org:service:DeviceProtection:1</serviceType><serviceId>urn:upnp-org:serviceId:DeviceProtection1</serviceId><controlURL>/ctl/DP</controlURL><eventSubURL>/evt/DP</eventSubURL><SCPDURL>/DP.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:2</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ecf</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL><SCPDURL>/WANCfg.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:2</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ec0</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:2</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL><SCPDURL>/WANIPCn.xml</SCPDURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>http://testwifi.here/</presentationURL></device></root>`
+)
+
+// pfSense 2.5.0-RELEASE / FreeBSD 12.2-STABLE
+const (
+	pfSenseUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nUSN: uuid:bee7052b-49e8-3597-b545-55a1e38ac11::urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nEXT:\r\nSERVER: FreeBSD/12.2-STABLE UPnP/1.1 MiniUPnPd/2.2.1\r\nLOCATION: http://192.168.1.1:2189/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1627958564\r\nBOOTID.UPNP.ORG: 1627958564\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	pfSenseRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0" configId="1337"><specVersion><major>1</major><minor>1</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:1</deviceType><friendlyName>FreeBSD router</friendlyName><manufacturer>FreeBSD</manufacturer><manufacturerURL>http://www.freebsd.org/</manufacturerURL><modelDescription>FreeBSD router</modelDescription><modelName>FreeBSD router</modelName><modelNumber>2.5.0-RELEASE</modelNumber><modelURL>http://www.freebsd.org/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac11</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:L3Forwarding1</serviceId><SCPDURL>/L3F.xml</SCPDURL><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:1</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac12</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><SCPDURL>/WANCfg.xml</SCPDURL><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:1</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac13</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:1</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><SCPDURL>/WANIPCn.xml</SCPDURL><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>https://192.168.1.1/</presentationURL></device></root>`
+)
+
+func TestParseUPnPDiscoResponse(t *testing.T) {
+	tests := []struct {
+		name    string
+		headers string
+		want    uPnPDiscoResponse
+	}{
+		{"google", googleWifiUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.86.1:5000/rootDesc.xml",
+		}},
+		{"pfsense", pfSenseUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.1.1:2189/rootDesc.xml",
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseUPnPDiscoResponse([]byte(tt.headers))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unexpected result:\n got: %+v\nwant: %+v\n", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetUPnPClient(t *testing.T) {
+	tests := []struct {
+		name    string
+		xmlBody string
+		want    string
+	}{
+		{"google", googleWifiRootDescXML, "*internetgateway2.WANIPConnection2"},
+		{"pfsense", pfSenseRootDescXML, "*internetgateway2.WANIPConnection1"},
+		// TODO(bradfitz): find a PPP one in the wild
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.RequestURI == "/rootDesc.xml" {
+					io.WriteString(w, tt.xmlBody)
+					return
+				}
+				http.NotFound(w, r)
+			}))
+			defer ts.Close()
+			gw, _ := netaddr.FromStdIP(ts.Listener.Addr().(*net.TCPAddr).IP)
+			c, err := getUPnPClient(context.Background(), gw, uPnPDiscoResponse{
+				Location: ts.URL + "/rootDesc.xml",
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := fmt.Sprintf("%T", c)
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

net/tsaddr/tsaddr.go

@@ -105,11 +105,6 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	return netaddr.IPFrom16(ret)
 }
 
-func IsULA(ip netaddr.IP) bool {
-	ulaRange.Do(func() { mustPrefix(&ulaRange.v, "fc00::/7") })
-	return ulaRange.v.Contains(ip)
-}
-
 func mustPrefix(v *netaddr.IPPrefix, prefix string) {
 	var err error
 	*v, err = netaddr.ParseIPPrefix(prefix)

net/tsaddr/tsaddr_test.go

@@ -43,28 +43,6 @@ func TestCGNATRange(t *testing.T) {
 	}
 }
 
-func TestIsUla(t *testing.T) {
-	tests := []struct {
-		name string
-		ip   string
-		want bool
-	}{
-		{"first ULA", "fc00::1", true},
-		{"not ULA", "fb00::1", false},
-		{"Tailscale", "fd7a:115c:a1e0::1", true},
-		{"Cloud Run", "fddf:3978:feb1:d745::1", true},
-		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
-		{"Link Local", "fe80::1", false},
-		{"Global", "2602::1", false},
-	}
-
-	for _, test := range tests {
-		if got := IsULA(netaddr.MustParseIP(test.ip)); got != test.want {
-			t.Errorf("IsULA(%s) = %v, want %v", test.name, got, test.want)
-		}
-	}
-}
-
 func TestNewContainsIPFunc(t *testing.T) {
 	f := NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.0.0.0/8")})
 	if f(netaddr.MustParseIP("8.8.8.8")) {

net/tstun/wrap.go

@@ -18,6 +18,7 @@ import (
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -64,7 +65,7 @@ type Wrapper struct {
 
 	closeOnce sync.Once
 
-	lastActivityAtomic int64 // unix seconds of last send or receive
+	lastActivityAtomic mono.Time // time of last send or receive
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 
@@ -153,9 +154,10 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		// a goroutine should not block when setting it, even with no listeners.
 		bufferConsumed: make(chan struct{}, 1),
 		closed:         make(chan struct{}),
-		outbound:       make(chan tunReadResult),
-		eventsUpDown:   make(chan tun.Event),
-		eventsOther:    make(chan tun.Event),
+		// outbound can be unbuffered; the buffer is an optimization.
+		outbound:     make(chan tunReadResult, 1),
+		eventsUpDown: make(chan tun.Event),
+		eventsOther:  make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
@@ -164,6 +166,7 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 	go tun.pumpEvents()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
+	tun.noteActivity()
 
 	return tun
 }
@@ -363,16 +366,15 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 
 // noteActivity records that there was a read or write at the current time.
 func (t *Wrapper) noteActivity() {
-	atomic.StoreInt64(&t.lastActivityAtomic, time.Now().Unix())
+	t.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 // IdleDuration reports how long it's been since the last read or write to this device.
 //
-// Its value is only accurate to roughly second granularity.
-// If there's never been activity, the duration is since 1970.
+// Its value should only be presumed accurate to roughly 10ms granularity.
+// If there's never been activity, the duration is since the wrapper was created.
 func (t *Wrapper) IdleDuration() time.Duration {
-	sec := atomic.LoadInt64(&t.lastActivityAtomic)
-	return time.Since(time.Unix(sec, 0))
+	return mono.Since(t.lastActivityAtomic.LoadAtomic())
 }
 
 func (t *Wrapper) Read(buf []byte, offset int) (int, error) {

net/tstun/wrap_test.go

@@ -10,13 +10,13 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"unsafe"
 
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -335,9 +335,9 @@ func TestFilter(t *testing.T) {
 				// data was actually filtered.
 				// If it stays zero, nothing made it through
 				// to the wrapped TUN.
-				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
+				tun.lastActivityAtomic.StoreAtomic(0)
 				_, err = tun.Write(tt.data, 0)
-				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
+				filtered = tun.lastActivityAtomic.LoadAtomic() == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)
@@ -416,7 +416,7 @@ func TestAtomic64Alignment(t *testing.T) {
 	}
 
 	c := new(Wrapper)
-	atomic.StoreInt64(&c.lastActivityAtomic, 123)
+	c.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 func TestPeerAPIBypass(t *testing.T) {

safesocket/safesocket_darwin.go

@@ -53,15 +53,16 @@ func localTCPPortAndTokenDarwin() (port int, token string, err error) {
 	// The current process is running outside the sandbox, so use
 	// lsof to find the IPNExtension:
 
-	out, err := exec.Command("lsof",
+	cmd := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
 		fmt.Sprintf("-u%d", os.Getuid()), // process of same user only
 		"-c", "IPNExtension",             // starting with IPNExtension
 		"-F", // machine-readable output
-	).Output()
+	)
+	out, err := cmd.Output()
 	if err != nil {
-		return 0, "", fmt.Errorf("failed to run lsof looking for IPNExtension: %w", err)
+		return 0, "", fmt.Errorf("failed to run '%s' looking for IPNExtension: %w", cmd, err)
 	}
 	bs := bufio.NewScanner(bytes.NewReader(out))
 	subStr := []byte(".tailscale.ipn.macos/sameuserproof-")

safesocket/unixsocket.go

@@ -28,16 +28,15 @@ func connect(path string, port uint16) (net.Conn, error) {
 	pipe, err := net.Dial("unix", path)
 	if err != nil {
 		if runtime.GOOS == "darwin" {
-			extConn, err := connectMacOSAppSandbox()
-			if err != nil {
-				log.Printf("safesocket: failed to connect to Tailscale IPNExtension: %v", err)
-			} else {
-				return extConn, nil
+			extConn, extErr := connectMacOSAppSandbox()
+			if extErr != nil {
+				return nil, fmt.Errorf("safesocket: failed to connect to %v: %v; failed to connect to Tailscale IPNExtension: %v", path, err, extErr)
 			}
+			return extConn, nil
 		}
 		return nil, err
 	}
-	return pipe, err
+	return pipe, nil
 }
 
 // TODO(apenwarr): handle magic cookie auth

scripts/check_license_headers.sh

@@ -38,9 +38,6 @@ for file in $(find $1 -name '*.go' -not -path '*/.git/*'); do
         $1/wgengine/router/ifconfig_windows.go)
             # WireGuard copyright.
         ;;
-		*_string.go)
-			# Generated file from go:generate stringer
-		;;
         *)
             header="$(head -3 $file)"
             if ! check_file "$header"; then

syncs/syncs.go

@@ -79,6 +79,16 @@ func (b *AtomicBool) Set(v bool) {
 	atomic.StoreInt32((*int32)(b), n)
 }
 
+// Swap sets b to v and reports whether it changed.
+func (b *AtomicBool) Swap(v bool) (changed bool) {
+	var n int32
+	if v {
+		n = 1
+	}
+	old := atomic.SwapInt32((*int32)(b), n)
+	return old != n
+}
+
 func (b *AtomicBool) Get() bool {
 	return atomic.LoadInt32((*int32)(b)) != 0
 }

tailcfg/tailcfg.go

@@ -164,6 +164,12 @@ type Node struct {
 	Hostinfo   Hostinfo
 	Created    time.Time
 
+	// PrimaryRoutes are the routes from AllowedIPs that this node
+	// is currently the primary subnet router for, as determined
+	// by the control plane. It does not include the self address
+	// values from Addresses that are in AllowedIPs.
+	PrimaryRoutes []netaddr.IPPrefix `json:",omitempty"`
+
 	// LastSeen is when the node was last online. It is not
 	// updated when Online is true. It is nil if the current
 	// node doesn't have permission to know, or the node
@@ -1142,6 +1148,7 @@ func (n *Node) Equal(n2 *Node) bool {
 		eqBoolPtr(n.Online, n2.Online) &&
 		eqCIDRs(n.Addresses, n2.Addresses) &&
 		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqCIDRs(n.PrimaryRoutes, n2.PrimaryRoutes) &&
 		eqStrings(n.Endpoints, n2.Endpoints) &&
 		n.DERP == n2.DERP &&
 		n.Hostinfo.Equal(&n2.Hostinfo) &&

tailcfg/tailcfg_clone.go

@@ -49,6 +49,7 @@ func (src *Node) Clone() *Node {
 	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
 	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
 	dst.Hostinfo = *src.Hostinfo.Clone()
+	dst.PrimaryRoutes = append(src.PrimaryRoutes[:0:0], src.PrimaryRoutes...)
 	if dst.LastSeen != nil {
 		dst.LastSeen = new(time.Time)
 		*dst.LastSeen = *src.LastSeen
@@ -79,6 +80,7 @@ var _NodeNeedsRegeneration = Node(struct {
 	DERP                    string
 	Hostinfo                Hostinfo
 	Created                 time.Time
+	PrimaryRoutes           []netaddr.IPPrefix
 	LastSeen                *time.Time
 	Online                  *bool
 	KeepAlive               bool

tailcfg/tailcfg_test.go

@@ -194,7 +194,8 @@ func TestNodeEqual(t *testing.T) {
 		"ID", "StableID", "Name", "User", "Sharer",
 		"Key", "KeyExpiry", "Machine", "DiscoKey",
 		"Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo",
-		"Created", "LastSeen", "Online", "KeepAlive", "MachineAuthorized",
+		"Created", "PrimaryRoutes",
+		"LastSeen", "Online", "KeepAlive", "MachineAuthorized",
 		"Capabilities",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
 	}

tempfork/wireguard-windows/firewall/zsyscall_windows.go

@@ -1,5 +1,3 @@
-// +build windows
-
 // Code generated by 'go generate'; DO NOT EDIT.
 
 package firewall

tstest/integration/gen_deps.go

@@ -12,16 +12,25 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"os/exec"
 )
 
 func main() {
+	for _, goos := range []string{"windows", "linux", "darwin", "freebsd", "openbsd"} {
+		generate(goos)
+	}
+}
+
+func generate(goos string) {
 	var x struct {
 		Imports []string
 	}
-	j, err := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled").Output()
+	cmd := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled")
+	cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH=amd64")
+	j, err := cmd.Output()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("GOOS=%s GOARCH=amd64 %s: %v", goos, cmd, err)
 	}
 	if err := json.Unmarshal(j, &x); err != nil {
 		log.Fatal(err)
@@ -46,7 +55,8 @@ import (
 	}
 	fmt.Fprintf(&out, ")\n")
 
-	err = ioutil.WriteFile("tailscaled_deps_test.go", out.Bytes(), 0644)
+	filename := fmt.Sprintf("tailscaled_deps_test_%s.go", goos)
+	err = ioutil.WriteFile(filename, out.Bytes(), 0644)
 	if err != nil {
 		log.Fatal(err)
 	}

tstest/integration/integration.go

@@ -205,6 +205,13 @@ func (lc *LogCatcher) logsString() string {
 	return lc.buf.String()
 }
 
+// Reset clears the buffered logs from memory.
+func (lc *LogCatcher) Reset() {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	lc.buf.Reset()
+}
+
 func (lc *LogCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var body io.Reader = r.Body
 	if r.Header.Get("Content-Encoding") == "zstd" {

tstest/integration/integration_test.go

@@ -42,6 +42,7 @@ import (
 
 var (
 	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
+	verboseTailscale  = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging")
 )
 
 var mainError atomic.Value // of error
@@ -72,29 +73,9 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-
-	n1.AwaitListening(t)
-
-	st := n1.MustStatus(t)
-	t.Logf("Status: %s", st.BackendState)
-
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `Program starting: `
-		if !env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Error(err)
-	}
-
+	n1.AwaitResponding(t)
 	n1.MustUp()
 
-	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
-		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
-		time.Sleep(d)
-	}
-
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
 	n1.AwaitRunning(t)
 
@@ -103,6 +84,40 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+func TestCollectPanic(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n := newTestNode(t, env)
+
+	cmd := exec.Command(n.env.Binaries.Daemon, "--cleanup")
+	cmd.Env = append(os.Environ(),
+		"TS_PLEASE_PANIC=1",
+		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
+	)
+	got, _ := cmd.CombinedOutput() // we expect it to fail, ignore err
+	t.Logf("initial run: %s", got)
+
+	// Now we run it again, and on start, it will upload the logs to logcatcher.
+	cmd = exec.Command(n.env.Binaries.Daemon, "--cleanup")
+	cmd.Env = append(os.Environ(), "TS_LOG_TARGET="+n.env.LogCatcherServer.URL)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("cleanup failed: %v: %q", err, out)
+	}
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `panic`
+		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // test Issue 2321: Start with UpdatePrefs should save prefs to disk
 func TestStateSavedOnStart(t *testing.T) {
 	t.Parallel()
@@ -115,22 +130,7 @@ func TestStateSavedOnStart(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-
-	n1.AwaitListening(t)
-
-	st := n1.MustStatus(t)
-	t.Logf("Status: %s", st.BackendState)
-
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `Program starting: `
-		if !env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Error(err)
-	}
-
+	n1.AwaitResponding(t)
 	n1.MustUp()
 
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
@@ -346,6 +346,76 @@ func TestAddPingRequest(t *testing.T) {
 	t.Error("all ping attempts failed")
 }
 
+// Issue 2434: when "down" (WantRunning false), tailscaled shouldn't
+// be connected to control.
+func TestNoControlConnWhenDown(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+
+	// Come up the first time.
+	n1.MustUp()
+	ip1 := n1.AwaitIP(t)
+	n1.AwaitRunning(t)
+
+	// Then bring it down and stop the daemon.
+	n1.MustDown()
+	d1.MustCleanShutdown(t)
+
+	env.LogCatcher.Reset()
+	d2 := n1.StartDaemon(t)
+	defer d2.Kill()
+	n1.AwaitResponding(t)
+
+	st := n1.MustStatus(t)
+	if got, want := st.BackendState, "Stopped"; got != want {
+		t.Fatalf("after restart, state = %q; want %q", got, want)
+	}
+
+	ip2 := n1.AwaitIP(t)
+	if ip1 != ip2 {
+		t.Errorf("IPs different: %q vs %q", ip1, ip2)
+	}
+
+	// The real test: verify our daemon doesn't have an HTTP request open.:
+	if n := env.Control.InServeMap(); n != 0 {
+		t.Errorf("in serve map = %d; want 0", n)
+	}
+
+	d2.MustCleanShutdown(t)
+}
+
+// Issue 2137: make sure Windows tailscaled works with the CLI alone,
+// without the GUI to kick off a Start.
+func TestOneNodeUpWindowsStyle(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+	n1.upFlagGOOS = "windows"
+
+	d1 := n1.StartDaemonAsIPNGOOS(t, "windows")
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp("--unattended")
+
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+}
+
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
@@ -422,9 +492,10 @@ func (e *testEnv) Close() error {
 type testNode struct {
 	env *testEnv
 
-	dir       string // temp dir for sock & state
-	sockFile  string
-	stateFile string
+	dir        string // temp dir for sock & state
+	sockFile   string
+	stateFile  string
+	upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI
 
 	mu        sync.Mutex
 	onLogLine []func([]byte)
@@ -434,10 +505,14 @@ type testNode struct {
 // The node is not started automatically.
 func newTestNode(t *testing.T, env *testEnv) *testNode {
 	dir := t.TempDir()
+	sockFile := filepath.Join(dir, "tailscale.sock")
+	if len(sockFile) >= 104 {
+		t.Fatalf("sockFile path %q (len %v) is too long, must be < 104", sockFile, len(sockFile))
+	}
 	return &testNode{
 		env:       env,
 		dir:       dir,
-		sockFile:  filepath.Join(dir, "tailscale.sock"),
+		sockFile:  sockFile,
 		stateFile: filepath.Join(dir, "tailscale.state"),
 	}
 }
@@ -462,6 +537,26 @@ func (n *testNode) diskPrefs(t testing.TB) *ipn.Prefs {
 	return p
 }
 
+// AwaitResponding waits for n's tailscaled to be up enough to be
+// responding, but doesn't wait for any particular state.
+func (n *testNode) AwaitResponding(t testing.TB) {
+	t.Helper()
+	n.AwaitListening(t)
+
+	st := n.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `Program starting: `
+		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // addLogLineHook registers a hook f to be called on each tailscaled
 // log line output.
 func (n *testNode) addLogLineHook(f func([]byte)) {
@@ -563,6 +658,10 @@ func (d *Daemon) MustCleanShutdown(t testing.TB) {
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
 func (n *testNode) StartDaemon(t testing.TB) *Daemon {
+	return n.StartDaemonAsIPNGOOS(t, runtime.GOOS)
+}
+
+func (n *testNode) StartDaemonAsIPNGOOS(t testing.TB, ipnGOOS string) *Daemon {
 	cmd := exec.Command(n.env.Binaries.Daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
@@ -573,6 +672,8 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
 		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
 		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
+		"TS_DEBUG_TAILSCALED_IPN_GOOS="+ipnGOOS,
+		"TS_LOGS_DIR="+t.TempDir(),
 	)
 	cmd.Stderr = &nodeOutputParser{n: n}
 	if *verboseTailscaled {
@@ -587,10 +688,15 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 	}
 }
 
-func (n *testNode) MustUp() {
+func (n *testNode) MustUp(extraArgs ...string) {
 	t := n.env.t
-	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
-	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
+	args := []string{
+		"up",
+		"--login-server=" + n.env.ControlServer.URL,
+	}
+	args = append(args, extraArgs...)
+	t.Logf("Running %v ...", args)
+	if err := n.Tailscale(args...).Run(); err != nil {
 		t.Fatalf("up: %v", err)
 	}
 }
@@ -622,7 +728,10 @@ func (n *testNode) AwaitIPs(t testing.TB) []netaddr.IP {
 	t.Helper()
 	var addrs []netaddr.IP
 	if err := tstest.WaitFor(20*time.Second, func() error {
-		out, err := n.Tailscale("ip").Output()
+		cmd := n.Tailscale("ip")
+		cmd.Stdout = nil // in case --verbose-tailscale was set
+		cmd.Stderr = nil // in case --verbose-tailscale was set
+		out, err := cmd.Output()
 		if err != nil {
 			return err
 		}
@@ -654,6 +763,7 @@ func (n *testNode) AwaitIP(t testing.TB) netaddr.IP {
 	return ips[0]
 }
 
+// AwaitRunning waits for n to reach the IPN state "Running".
 func (n *testNode) AwaitRunning(t testing.TB) {
 	t.Helper()
 	if err := tstest.WaitFor(20*time.Second, func() error {
@@ -676,11 +786,22 @@ func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
 	cmd := exec.Command(n.env.Binaries.CLI, "--socket="+n.sockFile)
 	cmd.Args = append(cmd.Args, arg...)
 	cmd.Dir = n.dir
+	cmd.Env = append(os.Environ(),
+		"TS_DEBUG_UP_FLAG_GOOS="+n.upFlagGOOS,
+		"TS_LOGS_DIR="+n.env.t.TempDir(),
+	)
+	if *verboseTailscale {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
 	return cmd
 }
 
 func (n *testNode) Status() (*ipnstate.Status, error) {
-	out, err := n.Tailscale("status", "--json").CombinedOutput()
+	cmd := n.Tailscale("status", "--json")
+	cmd.Stdout = nil // in case --verbose-tailscale was set
+	cmd.Stderr = nil // in case --verbose-tailscale was set
+	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
 	}

tstest/integration/tailscaled_deps_test_darwin.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/exec"
+	_ "os/signal"
+	_ "path/filepath"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -39,6 +39,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_linux.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_windows.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "golang.org/x/sys/windows"
+	_ "golang.org/x/sys/windows/svc"
+	_ "golang.org/x/sys/windows/svc/mgr"
+	_ "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	_ "inet.af/netaddr"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail/backoff"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wf"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/testcontrol/testcontrol.go

@@ -51,6 +51,7 @@ type Server struct {
 	mux         *http.ServeMux
 
 	mu            sync.Mutex
+	inServeMap    int
 	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        wgkey.Key
 	privKey       wgkey.Private
@@ -509,7 +510,22 @@ func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
 	return peersToUpdate
 }
 
+func (s *Server) incrInServeMap(delta int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inServeMap += delta
+}
+
+// InServeMap returns the number of clients currently in a MapRequest HTTP handler.
+func (s *Server) InServeMap() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.inServeMap
+}
+
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	s.incrInServeMap(1)
+	defer s.incrInServeMap(-1)
 	ctx := r.Context()
 
 	req := new(tailcfg.MapRequest)

tstest/integration/vms/distros.go

@@ -2,24 +2,33 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux
-
 package vms
 
+import (
+	_ "embed"
+	"log"
+
+	"github.com/tailscale/hujson"
+)
+
+// go:generate go run ./gen
+
 type Distro struct {
-	name           string // amazon-linux
-	url            string // URL to a qcow2 image
-	sha256sum      string // hex-encoded sha256 sum of contents of URL
-	mem            int    // VM memory in megabytes
-	packageManager string // yum/apt/dnf/zypper
-	initSystem     string // systemd/openrc
+	Name           string // amazon-linux
+	URL            string // URL to a qcow2 image
+	SHA256Sum      string // hex-encoded sha256 sum of contents of URL
+	MemoryMegs     int    // VM memory in megabytes
+	PackageManager string // yum/apt/dnf/zypper
+	InitSystem     string // systemd/openrc
 }
 
 func (d *Distro) InstallPre() string {
-	switch d.packageManager {
+	switch d.PackageManager {
 	case "yum":
 		return ` - [ yum, update, gnupg2 ]
- - [ yum, "-y", install, iptables ]`
+ - [ yum, "-y", install, iptables ]
+ - [ sh, "-c", "printf '\n\nUseDNS no\n\n' | tee -a /etc/ssh/sshd_config" ]
+ - [ systemctl, restart, "sshd.service" ]`
 	case "zypper":
 		return ` - [ zypper, in, "-y", iptables ]`
 
@@ -38,58 +47,15 @@ func (d *Distro) InstallPre() string {
 	return ""
 }
 
-var distros = []Distro{
-	// NOTE(Xe): If you run into issues getting the autoconfig to work, run
-	// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
-	// client with a command like this:
-	//
-	//    $ vncviewer :0
-	//
-	// On NixOS you can get away with something like this:
-	//
-	//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
-	//
-	// Login as root with the password root. Then look in
-	// /var/log/cloud-init-output.log for what you messed up.
+//go:embed distros.hujson
+var distroData string
 
-	// NOTE(Xe): These images are not official images created by the Alpine Linux
-	// cloud team because the cloud team hasn't created any official images yet.
-	// These images were created under the guidance of the cloud team and contain
-	// few notable differences from what they would end up shipping. The Alpine
-	// Linux cloud team probably won't have official images up until a year or so
-	// after this comment is written (2021-06-11), but overall they will be
-	// compatible with these images. These images were created using the setup in
-	// this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
-	// these links.
-	{"alpine-3-13-5", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", 256, "apk", "openrc"},
-	{"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk", "openrc"},
+var Distros []Distro = func() []Distro {
+	var result []Distro
+	err := hujson.Unmarshal([]byte(distroData), &result)
+	if err != nil {
+		log.Fatalf("error decoding distros: %v", err)
+	}
 
-	// NOTE(Xe): All of the following images are official images straight from each
-	// distribution's official documentation.
-	{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum", "systemd"},
-	{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman", "systemd"},
-	{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum", "systemd"},
-	{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf", "systemd"},
-	{"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt", "systemd"},
-	{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt", "systemd"},
-	{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf", "systemd"},
-	{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper", "systemd"},
-	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"},
-	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"},
-	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"},
-	{"oracle-linux-7", "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2", "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb", 512, "yum", "systemd"},
-	{"oracle-linux-8", "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2", "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b", 768, "dnf", "systemd"},
-	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"},
-	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"},
-	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
-	{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"},
-	{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"},
-
-	// NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
-	// used here is actually the URL of the NixOS channel being built from and the
-	// shasum is meaningless. This `channel:name` syntax is documented at [1].
-	//
-	// [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
-	{"nixos-21-05", "channel:nixos-21.05", "lolfakesha", 512, "nix", "systemd"},
-	{"nixos-unstable", "channel:nixos-unstable", "lolfakesha", 512, "nix", "systemd"},
-}
+	return result
+}()

tstest/integration/vms/distros.hujson

@@ -0,0 +1,208 @@
+// NOTE(Xe): If you run into issues getting the autoconfig to work, run
+// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
+// client with a command like this:
+//
+//    $ vncviewer :0
+//
+// On NixOS you can get away with something like this:
+//
+//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
+//
+// Login as root with the password root. Then look in
+// /var/log/cloud-init-output.log for what you messed up.
+[
+	  // NOTE(Xe): These images are not official images created by the Alpine Linux
+	  // cloud team because the cloud team hasn't created any official images yet.
+	  // These images were created under the guidance of the cloud team and contain
+	  // few notable differences from what they would end up shipping. The Alpine
+	  // Linux cloud team probably won't have official images up until a year or so
+	  // after this comment is written (2021-06-11), but overall they will be
+	  // compatible with these images. These images were created using the setup in
+	  // this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
+	  // these links.
+    {
+        "Name": "alpine-3-13-5",
+        "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2",
+        "SHA256Sum": "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287",
+        "MemoryMegs": 256,
+        "PackageManager": "apk",
+        "InitSystem": "openrc"
+    },
+    {
+        "Name": "alpine-edge",
+        "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2",
+        "SHA256Sum": "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0",
+        "MemoryMegs": 256,
+        "PackageManager": "apk",
+        "InitSystem": "openrc"
+    },
+
+	  // NOTE(Xe): All of the following images are official images straight from each
+	  // distribution's official documentation.
+    {
+        "Name": "amazon-linux",
+        "URL": "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2",
+        "SHA256Sum": "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b",
+        "MemoryMegs": 512,
+        "PackageManager": "yum",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "arch",
+        "URL": "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2",
+        "SHA256Sum": "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27",
+        "MemoryMegs": 512,
+        "PackageManager": "pacman",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "centos-7",
+        "URL": "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c",
+        "SHA256Sum": "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87",
+        "MemoryMegs": 512,
+        "PackageManager": "yum",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "centos-8",
+        "URL": "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2",
+        "SHA256Sum": "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0",
+        "MemoryMegs": 768,
+        "PackageManager": "dnf",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "debian-9",
+        "URL": "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2",
+        "SHA256Sum": "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "debian-10",
+        "URL": "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2",
+        "SHA256Sum": "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0",
+        "MemoryMegs": 768,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "fedora-34",
+        "URL": "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2",
+        "SHA256Sum": "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea",
+        "MemoryMegs": 768,
+        "PackageManager": "dnf",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-leap-15-1",
+        "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2",
+        "SHA256Sum": "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-leap-15-2",
+        "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2",
+        "SHA256Sum": "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-leap-15-3",
+        "URL": "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2",
+        "SHA256Sum": "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-tumbleweed",
+        "URL": "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2",
+        "SHA256Sum": "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "oracle-linux-7",
+        "URL": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2",
+        "SHA256Sum": "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb",
+        "MemoryMegs": 512,
+        "PackageManager": "yum",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "oracle-linux-8",
+        "URL": "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2",
+        "SHA256Sum": "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b",
+        "MemoryMegs": 768,
+        "PackageManager": "dnf",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-16-04",
+        "URL": "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img",
+        "SHA256Sum": "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-18-04",
+        "URL": "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img",
+        "SHA256Sum": "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-20-04",
+        "URL": "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img",
+        "SHA256Sum": "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-20-10",
+        "URL": "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img",
+        "SHA256Sum": "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-21-04",
+        "URL": "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img",
+        "SHA256Sum": "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+
+	  // NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
+	  // used here is actually the URL of the NixOS channel being built from and the
+	  // shasum is meaningless. This `channel:name` syntax is documented at [1].
+	  //
+	  // [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
+    {
+        "Name": "nixos-21-05",
+        "URL": "channel:nixos-21.05",
+        "SHA256Sum": "lolfakesha",
+        "MemoryMegs": 512,
+        "PackageManager": "nix",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "nixos-unstable",
+        "URL": "channel:nixos-unstable",
+        "SHA256Sum": "lolfakesha",
+        "MemoryMegs": 512,
+        "PackageManager": "nix",
+        "InitSystem": "systemd"
+    }
+]

tstest/integration/vms/distros_test.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"testing"
+)
+
+func TestDistrosGotLoaded(t *testing.T) {
+	if len(Distros) == 0 {
+		t.Fatal("no distros were loaded")
+	}
+}

tstest/integration/vms/gen/test_codegen.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// build ignore
+
+package main
+
+import (
+	_ "embed"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/dave/jennifer/jen"
+	"github.com/iancoleman/strcase"
+	"tailscale.com/tstest/integration/vms"
+)
+
+func main() {
+	f := jen.NewFile("vms")
+	f.Comment("Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.")
+
+	ptr := jen.Op("*")
+
+	for i, d := range vms.Distros {
+		f.Func().
+			Id("TestRun" + strcase.ToCamel(d.Name)).
+			Params(jen.Id("t").Add(ptr).Qual("testing", "T")).
+			BlockFunc(func(g *jen.Group) {
+				g.Id("t").Dot("Parallel").Call()
+				g.Id("setupTests").Call(jen.Id("t"))
+				g.Id("testOneDistribution").Call(jen.Id("t"), jen.Lit(i), jen.Id("Distros").Index(jen.Lit(i)))
+			})
+	}
+
+	os.Remove("top_level_test.go")
+	fout, err := os.Create("top_level_test.go")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fout.Close()
+
+	fmt.Fprintf(fout, "// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.\n", time.Now().Year())
+	fout.WriteString("// Use of this source code is governed by a BSD-style\n")
+	fout.WriteString("// license that can be found in the LICENSE file.\n")
+	fout.WriteString("\n")
+	fout.WriteString("// +build linux\n\n")
+
+	err = f.Render(fout)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

tstest/integration/vms/nixos_test.go

@@ -167,9 +167,13 @@ func copyUnit(t *testing.T, bins *integration.Binaries) {
 }
 
 func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
+	if d.Name == "nixos-unstable" {
+		t.Skip("https://github.com/NixOS/nixpkgs/issues/131098")
+	}
+
 	copyUnit(t, h.bins)
 	dir := t.TempDir()
-	fname := filepath.Join(dir, d.name+".nix")
+	fname := filepath.Join(dir, d.Name+".nix")
 	fout, err := os.Create(fname)
 	if err != nil {
 		t.Fatal(err)
@@ -196,10 +200,10 @@ func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
 	os.MkdirAll(outpath, 0755)
 
 	t.Cleanup(func() {
-		os.RemoveAll(filepath.Join(outpath, d.name)) // makes the disk image a candidate for GC
+		os.RemoveAll(filepath.Join(outpath, d.Name)) // makes the disk image a candidate for GC
 	})
 
-	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.name), "-c", fname)
+	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.Name), "-c", fname)
 	if *verboseNixOutput {
 		cmd.Stdout = logger.FuncWriter(t.Logf)
 		cmd.Stderr = logger.FuncWriter(t.Logf)
@@ -214,16 +218,16 @@ func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
 		cmd.Stderr = fout
 		defer fout.Close()
 	}
-	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.url)
+	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.URL)
 	cmd.Dir = outpath
 	t.Logf("running %s %#v", "nixos-generate", cmd.Args)
 	if err := cmd.Run(); err != nil {
-		t.Fatalf("error while making NixOS image for %s: %v", d.name, err)
+		t.Fatalf("error while making NixOS image for %s: %v", d.Name, err)
 	}
 
 	if !*verboseNixOutput {
 		t.Log("done")
 	}
 
-	return filepath.Join(outpath, d.name, "nixos.qcow2")
+	return filepath.Join(outpath, d.Name, "nixos.qcow2")
 }

tstest/integration/vms/opensuse_leap_15_1_test.go

@@ -41,7 +41,7 @@ type openSUSELeap151MetaDataMeta struct {
 }
 
 func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
-	if d.name != "opensuse-leap-15-1" {
+	if d.Name != "opensuse-leap-15-1" {
 		return false
 	}
 
@@ -54,14 +54,14 @@ func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
 
 	metadata, err := json.Marshal(openSUSELeap151MetaData{
 		Zone:        "nova",
-		Hostname:    d.name,
+		Hostname:    d.Name,
 		LaunchIndex: "0",
 		Meta: openSUSELeap151MetaDataMeta{
 			Role:      "server",
 			DSMode:    "local",
 			Essential: "false",
 		},
-		Name: d.name,
+		Name: d.Name,
 		UUID: uuid.New().String(),
 	})
 	if err != nil {

tstest/integration/vms/top_level_test.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import "testing"
+
+// Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.
+func TestRunAlpine3135(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 0, Distros[0])
+}
+func TestRunAlpineEdge(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 1, Distros[1])
+}
+func TestRunAmazonLinux(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 2, Distros[2])
+}
+func TestRunArch(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 3, Distros[3])
+}
+func TestRunCentos7(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 4, Distros[4])
+}
+func TestRunCentos8(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 5, Distros[5])
+}
+func TestRunDebian9(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 6, Distros[6])
+}
+func TestRunDebian10(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 7, Distros[7])
+}
+func TestRunFedora34(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 8, Distros[8])
+}
+func TestRunOpensuseLeap151(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 9, Distros[9])
+}
+func TestRunOpensuseLeap152(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 10, Distros[10])
+}
+func TestRunOpensuseLeap153(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 11, Distros[11])
+}
+func TestRunOpensuseTumbleweed(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 12, Distros[12])
+}
+func TestRunOracleLinux7(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 13, Distros[13])
+}
+func TestRunOracleLinux8(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 14, Distros[14])
+}
+func TestRunUbuntu1604(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 15, Distros[15])
+}
+func TestRunUbuntu1804(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 16, Distros[16])
+}
+func TestRunUbuntu2004(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 17, Distros[17])
+}
+func TestRunUbuntu2010(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 18, Distros[18])
+}
+func TestRunUbuntu2104(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 19, Distros[19])
+}
+func TestRunNixos2105(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 20, Distros[20])
+}
+func TestRunNixosUnstable(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 21, Distros[21])
+}

tstest/integration/vms/vm_setup_test.go

@@ -53,17 +53,17 @@ func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir stri
 	mkLayeredQcow(t, tdir, d, h.fetchDistro(t, d))
 	mkSeed(t, d, sshKey, hostURL, tdir, port)
 
-	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
+	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.Name+".qcow2"))
 
 	args := []string{
 		"-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off",
 		"-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port),
 		"-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25",
-		"-m", fmt.Sprint(d.mem),
+		"-m", fmt.Sprint(d.MemoryMegs),
 		"-boot", "c",
 		"-drive", driveArg,
-		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
-		"-smbios", "type=1,serial=ds=nocloud;h=" + d.name,
+		"-cdrom", filepath.Join(tdir, d.Name, "seed", "seed.iso"),
+		"-smbios", "type=1,serial=ds=nocloud;h=" + d.Name,
 	}
 
 	if *useVNC {
@@ -101,7 +101,7 @@ func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir stri
 	t.Cleanup(func() {
 		err := cmd.Process.Kill()
 		if err != nil {
-			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
+			t.Errorf("can't kill %s (%d): %v", d.Name, cmd.Process.Pid, err)
 		}
 
 		cmd.Wait()
@@ -139,15 +139,15 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
 		d.PartSize = 64 * 1024 * 1024 // 64MB per part
 	})
 
-	t.Logf("fetching s3://%s/%s", bucketName, d.sha256sum)
+	t.Logf("fetching s3://%s/%s", bucketName, d.SHA256Sum)
 
 	_, err = dler.Download(fout, &s3.GetObjectInput{
 		Bucket: aws.String(bucketName),
-		Key:    aws.String(d.sha256sum),
+		Key:    aws.String(d.SHA256Sum),
 	})
 	if err != nil {
 		fout.Close()
-		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.sha256sum, err)
+		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.SHA256Sum, err)
 	}
 
 	err = fout.Close()
@@ -169,17 +169,17 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	}
 	cdir = filepath.Join(cdir, "tailscale", "vm-test")
 
-	if strings.HasPrefix(resultDistro.name, "nixos") {
+	if strings.HasPrefix(resultDistro.Name, "nixos") {
 		return h.makeNixOSImage(t, resultDistro, cdir)
 	}
 
-	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
+	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.SHA256Sum)
 
 	_, err = os.Stat(qcowPath)
 	if err == nil {
 		hash := checkCachedImageHash(t, resultDistro, cdir)
-		if hash != resultDistro.sha256sum {
-			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.name, qcowPath, resultDistro.sha256sum)
+		if hash != resultDistro.SHA256Sum {
+			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.Name, qcowPath, resultDistro.SHA256Sum)
 			err = errors.New("some fake non-nil error to force a redownload")
 
 			if err := os.Remove(qcowPath); err != nil {
@@ -189,26 +189,26 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	}
 
 	if err != nil {
-		t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
+		t.Logf("downloading distro image %s to %s", resultDistro.URL, qcowPath)
 		fout, err := os.Create(qcowPath)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		if !fetchFromS3(t, fout, resultDistro) {
-			resp, err := http.Get(resultDistro.url)
+			resp, err := http.Get(resultDistro.URL)
 			if err != nil {
-				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
+				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.Name, resultDistro.URL, err)
 			}
 
 			if resp.StatusCode != http.StatusOK {
 				resp.Body.Close()
-				t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
+				t.Fatalf("%s replied %s", resultDistro.URL, resp.Status)
 			}
 
 			_, err = io.Copy(fout, resp.Body)
 			if err != nil {
-				t.Fatalf("download of %s failed: %v", resultDistro.url, err)
+				t.Fatalf("download of %s failed: %v", resultDistro.URL, err)
 			}
 
 			resp.Body.Close()
@@ -219,8 +219,8 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 
 			hash := checkCachedImageHash(t, resultDistro, cdir)
 
-			if hash != resultDistro.sha256sum {
-				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.sha256sum, hash)
+			if hash != resultDistro.SHA256Sum {
+				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.SHA256Sum, hash)
 			}
 		}
 	}
@@ -231,7 +231,7 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) {
 	t.Helper()
 
-	qcowPath := filepath.Join(cacheDir, "qcow2", d.sha256sum)
+	qcowPath := filepath.Join(cacheDir, "qcow2", d.SHA256Sum)
 
 	fin, err := os.Open(qcowPath)
 	if err != nil {
@@ -244,8 +244,8 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri
 	}
 	hash := hex.EncodeToString(hasher.Sum(nil))
 
-	if hash != d.sha256sum {
-		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.sha256sum)
+	if hash != d.SHA256Sum {
+		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.SHA256Sum)
 	}
 
 	gotHash = hash
@@ -255,7 +255,7 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri
 
 func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
 	bins := h.bins
-	if strings.HasPrefix(d.name, "nixos") {
+	if strings.HasPrefix(d.Name, "nixos") {
 		return
 	}
 
@@ -275,7 +275,7 @@ func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
 	// TODO(Xe): revisit this assumption before it breaks the test.
 	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
 
-	switch d.initSystem {
+	switch d.InitSystem {
 	case "openrc":
 		mkdir(t, cli, "/etc/init.d")
 		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.openrc", "/etc/init.d/tailscaled")

tstest/integration/vms/vms_test.go

@@ -18,6 +18,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"text/template"
 	"time"
@@ -57,14 +58,14 @@ func TestDownloadImages(t *testing.T) {
 
 	bins := integration.BuildTestBinaries(t)
 
-	for _, d := range distros {
+	for _, d := range Distros {
 		distro := d
-		t.Run(distro.name, func(t *testing.T) {
-			if !distroRex.Unwrap().MatchString(distro.name) {
-				t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex)
+		t.Run(distro.Name, func(t *testing.T) {
+			if !distroRex.Unwrap().MatchString(distro.Name) {
+				t.Skipf("distro name %q doesn't match regex: %s", distro.Name, distroRex)
 			}
 
-			if strings.HasPrefix(distro.name, "nixos") {
+			if strings.HasPrefix(distro.Name, "nixos") {
 				t.Skip("NixOS is built on the fly, no need to download it")
 			}
 
@@ -98,7 +99,7 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro, qcowBase string) {
 	run(t, tdir, "qemu-img", "create",
 		"-f", "qcow2",
 		"-o", "backing_file="+qcowBase,
-		filepath.Join(tdir, d.name+".qcow2"),
+		filepath.Join(tdir, d.Name+".qcow2"),
 	)
 }
 
@@ -112,7 +113,7 @@ var (
 func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 	t.Helper()
 
-	dir := filepath.Join(tdir, d.name, "seed")
+	dir := filepath.Join(tdir, d.Name, "seed")
 	os.MkdirAll(dir, 0700)
 
 	// make meta-data
@@ -127,7 +128,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			Hostname string
 		}{
 			ID:       "31337",
-			Hostname: d.name,
+			Hostname: d.Name,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -156,7 +157,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 		}{
 			SSHKey:     strings.TrimSpace(sshKey),
 			HostURL:    hostURL,
-			Hostname:   d.name,
+			Hostname:   d.Name,
 			Port:       port,
 			InstallPre: d.InstallPre(),
 			Password:   securePassword,
@@ -220,10 +221,11 @@ func getProbablyFreePortNumber() (int, error) {
 	return portNum, nil
 }
 
-// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
-// tailscale on it and then ensures that it connects to the network
-// successfully.
-func TestVMIntegrationEndToEnd(t *testing.T) {
+func setupTests(t *testing.T) {
+	ramsem.once.Do(func() {
+		ramsem.sem = semaphore.NewWeighted(int64(*vmRamLimit))
+	})
+
 	if !*runVMTests {
 		t.Skip("not running integration tests (need --run-vm-tests)")
 	}
@@ -239,56 +241,53 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test --v --timeout=60m --run-vm-tests'")
 		t.Fatalf("missing dependency: %v", err)
 	}
+}
 
-	ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
-	rex := distroRex.Unwrap()
+var ramsem struct {
+	once sync.Once
+	sem  *semaphore.Weighted
+}
 
-	t.Run("do", func(t *testing.T) {
-		for n, distro := range distros {
-			n, distro := n, distro
-			if rex.MatchString(distro.name) {
-				t.Logf("%s matches %s", distro.name, rex)
-			} else {
-				continue
+func testOneDistribution(t *testing.T, n int, distro Distro) {
+	setupTests(t)
+
+	if distroRex.Unwrap().MatchString(distro.Name) {
+		t.Logf("%s matches %s", distro.Name, distroRex.Unwrap())
+	} else {
+		t.Skip("regex not matched")
+	}
+
+	ctx, done := context.WithCancel(context.Background())
+	t.Cleanup(done)
+
+	h := newHarness(t)
+	dir := t.TempDir()
+
+	err := ramsem.sem.Acquire(ctx, int64(distro.MemoryMegs))
+	if err != nil {
+		t.Fatalf("can't acquire ram semaphore: %v", err)
+	}
+	t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) })
+
+	h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir)
+	var ipm ipMapping
+
+	t.Run("wait-for-start", func(t *testing.T) {
+		waiter := time.NewTicker(time.Second)
+		defer waiter.Stop()
+		var ok bool
+		for {
+			<-waiter.C
+			h.ipMu.Lock()
+			if ipm, ok = h.ipMap[distro.Name]; ok {
+				h.ipMu.Unlock()
+				break
 			}
-
-			t.Run(distro.name, func(t *testing.T) {
-				ctx, done := context.WithCancel(context.Background())
-				t.Cleanup(done)
-
-				t.Parallel()
-
-				h := newHarness(t)
-				dir := t.TempDir()
-
-				err := ramsem.Acquire(ctx, int64(distro.mem))
-				if err != nil {
-					t.Fatalf("can't acquire ram semaphore: %v", err)
-				}
-				defer ramsem.Release(int64(distro.mem))
-
-				h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir)
-				var ipm ipMapping
-
-				t.Run("wait-for-start", func(t *testing.T) {
-					waiter := time.NewTicker(time.Second)
-					defer waiter.Stop()
-					var ok bool
-					for {
-						<-waiter.C
-						h.ipMu.Lock()
-						if ipm, ok = h.ipMap[distro.name]; ok {
-							h.ipMu.Unlock()
-							break
-						}
-						h.ipMu.Unlock()
-					}
-				})
-
-				h.testDistro(t, distro, ipm)
-			})
+			h.ipMu.Unlock()
 		}
 	})
+
+	h.testDistro(t, distro, ipm)
 }
 
 func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
@@ -339,7 +338,7 @@ func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 			&expect.BExp{R: `(\#)`},
 		}
 
-		switch d.initSystem {
+		switch d.InitSystem {
 		case "openrc":
 			// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
 			// of service readiness. If this sleep is removed then tailscale will not be

tstime/mono/mono.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package mono provides fast monotonic time.
+// On most platforms, mono.Now is about 2x faster than time.Now.
+// However, time.Now is really fast, and nicer to use.
+//
+// For almost all purposes, you should use time.Now.
+//
+// Package mono exists because we get the current time multiple
+// times per network packet, at which point it makes a
+// measurable difference.
+package mono
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+	_ "unsafe" // for go:linkname
+)
+
+// Time is the number of nanoseconds elapsed since an unspecified reference start time.
+type Time int64
+
+// Now returns the current monotonic time.
+func Now() Time {
+	// On a newly started machine, the monotonic clock might be very near zero.
+	// Thus mono.Time(0).Before(mono.Now.Add(-time.Minute)) might yield true.
+	// The corresponding package time expression never does, if the wall clock is correct.
+	// Preserve this correspondence by increasing the "base" monotonic clock by a fair amount.
+	const baseOffset int64 = 1 << 55 // approximately 10,000 hours in nanoseconds
+	return Time(now() + baseOffset)
+}
+
+// Since returns the time elapsed since t.
+func Since(t Time) time.Duration {
+	return time.Duration(Now() - t)
+}
+
+// Sub returns t-n, the duration from n to t.
+func (t Time) Sub(n Time) time.Duration {
+	return time.Duration(t - n)
+}
+
+// Add returns t+d.
+func (t Time) Add(d time.Duration) Time {
+	return t + Time(d)
+}
+
+// After reports t > n, whether t is after n.
+func (t Time) After(n Time) bool {
+	return t > n
+}
+
+// After reports t < n, whether t is before n.
+func (t Time) Before(n Time) bool {
+	return t < n
+}
+
+// IsZero reports whether t == 0.
+func (t Time) IsZero() bool {
+	return t == 0
+}
+
+// StoreAtomic does an atomic store *t = new.
+func (t *Time) StoreAtomic(new Time) {
+	atomic.StoreInt64((*int64)(t), int64(new))
+}
+
+// LoadAtomic does an atomic load *t.
+func (t *Time) LoadAtomic() Time {
+	return Time(atomic.LoadInt64((*int64)(t)))
+}
+
+//go:linkname now runtime.nanotime1
+func now() int64
+
+// baseWall and baseMono are a pair of almost-identical times used to correlate a Time with a wall time.
+var (
+	baseWall time.Time
+	baseMono Time
+)
+
+func init() {
+	baseWall = time.Now()
+	baseMono = Now()
+}
+
+// String prints t, including an estimated equivalent wall clock.
+// This is best-effort only, for rough debugging purposes only.
+// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
+// Even in the best of circumstances, it may vary by a few milliseconds.
+func (t Time) String() string {
+	return fmt.Sprintf("mono.Time(ns=%d, estimated wall=%v)", int64(t), baseWall.Add(t.Sub(baseMono)).Truncate(0))
+}
+
+// MarshalJSON formats t for JSON as if it were a time.Time.
+// We format Time this way for backwards-compatibility.
+// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
+// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
+// Even in the best of circumstances, it may vary by a few milliseconds.
+func (t Time) MarshalJSON() ([]byte, error) {
+	var tt time.Time
+	if !t.IsZero() {
+		tt = baseWall.Add(t.Sub(baseMono)).Truncate(0)
+	}
+	return tt.MarshalJSON()
+}
+
+// UnmarshalJSON sets t according to data.
+// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	var tt time.Time
+	err := tt.UnmarshalJSON(data)
+	if err != nil {
+		return err
+	}
+	*t = Now().Add(-time.Since(tt))
+	return nil
+}

tstime/mono/mono_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mono
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNow(t *testing.T) {
+	start := Now()
+	time.Sleep(100 * time.Millisecond)
+	if elapsed := Since(start); elapsed < 100*time.Millisecond {
+		t.Errorf("short sleep: %v elapsed, want min %v", elapsed, 100*time.Millisecond)
+	}
+}
+
+func BenchmarkMonoNow(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Now()
+	}
+}
+
+func BenchmarkTimeNow(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		time.Now()
+	}
+}

tstime/rate/rate.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a modified, simplified version of code from golang.org/x/time/rate.
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package rate provides a rate limiter.
+package rate
+
+import (
+	"sync"
+	"time"
+
+	"tailscale.com/tstime/mono"
+)
+
+// Limit defines the maximum frequency of some events.
+// Limit is represented as number of events per second.
+// A zero Limit is invalid.
+type Limit float64
+
+// Every converts a minimum time interval between events to a Limit.
+func Every(interval time.Duration) Limit {
+	if interval <= 0 {
+		panic("invalid interval")
+	}
+	return 1 / Limit(interval.Seconds())
+}
+
+// A Limiter controls how frequently events are allowed to happen.
+// It implements a "token bucket" of size b, initially full and refilled
+// at rate r tokens per second.
+// Informally, in any large enough time interval, the Limiter limits the
+// rate to r tokens per second, with a maximum burst size of b events.
+// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
+// Use NewLimiter to create non-zero Limiters.
+type Limiter struct {
+	limit  Limit
+	burst  float64
+	mu     sync.Mutex // protects following fields
+	tokens float64    // number of tokens currently in bucket
+	last   mono.Time  // the last time the limiter's tokens field was updated
+}
+
+// NewLimiter returns a new Limiter that allows events up to rate r and permits
+// bursts of at most b tokens.
+func NewLimiter(r Limit, b int) *Limiter {
+	if b < 1 {
+		panic("bad burst, must be at least 1")
+	}
+	return &Limiter{limit: r, burst: float64(b)}
+}
+
+// AllowN reports whether an event may happen now.
+func (lim *Limiter) Allow() bool {
+	return lim.allow(mono.Now())
+}
+
+func (lim *Limiter) allow(now mono.Time) bool {
+	lim.mu.Lock()
+	defer lim.mu.Unlock()
+
+	// If time has moved backwards, look around awkwardly and pretend nothing happened.
+	if now.Before(lim.last) {
+		lim.last = now
+	}
+
+	// Calculate the new number of tokens available due to the passage of time.
+	elapsed := now.Sub(lim.last)
+	tokens := lim.tokens + float64(lim.limit)*elapsed.Seconds()
+	if tokens > lim.burst {
+		tokens = lim.burst
+	}
+
+	// Consume a token.
+	tokens--
+
+	// Update state.
+	ok := tokens >= 0
+	if ok {
+		lim.last = now
+		lim.tokens = tokens
+	}
+	return ok
+}

tstime/rate/rate_test.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a modified, simplified version of code from golang.org/x/time/rate.
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.7
+// +build go1.7
+
+package rate
+
+import (
+	"context"
+	"math"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"tailscale.com/tstime/mono"
+)
+
+func closeEnough(a, b Limit) bool {
+	return (math.Abs(float64(a)/float64(b)) - 1.0) < 1e-9
+}
+
+func TestEvery(t *testing.T) {
+	cases := []struct {
+		interval time.Duration
+		lim      Limit
+	}{
+		{1 * time.Nanosecond, Limit(1e9)},
+		{1 * time.Microsecond, Limit(1e6)},
+		{1 * time.Millisecond, Limit(1e3)},
+		{10 * time.Millisecond, Limit(100)},
+		{100 * time.Millisecond, Limit(10)},
+		{1 * time.Second, Limit(1)},
+		{2 * time.Second, Limit(0.5)},
+		{time.Duration(2.5 * float64(time.Second)), Limit(0.4)},
+		{4 * time.Second, Limit(0.25)},
+		{10 * time.Second, Limit(0.1)},
+		{time.Duration(math.MaxInt64), Limit(1e9 / float64(math.MaxInt64))},
+	}
+	for _, tc := range cases {
+		lim := Every(tc.interval)
+		if !closeEnough(lim, tc.lim) {
+			t.Errorf("Every(%v) = %v want %v", tc.interval, lim, tc.lim)
+		}
+	}
+}
+
+const (
+	d = 100 * time.Millisecond
+)
+
+var (
+	t0 = mono.Now()
+	t1 = t0.Add(time.Duration(1) * d)
+	t2 = t0.Add(time.Duration(2) * d)
+	t3 = t0.Add(time.Duration(3) * d)
+	t4 = t0.Add(time.Duration(4) * d)
+	t5 = t0.Add(time.Duration(5) * d)
+	t9 = t0.Add(time.Duration(9) * d)
+)
+
+type allow struct {
+	t  mono.Time
+	ok bool
+}
+
+func run(t *testing.T, lim *Limiter, allows []allow) {
+	t.Helper()
+	for i, allow := range allows {
+		ok := lim.allow(allow.t)
+		if ok != allow.ok {
+			t.Errorf("step %d: lim.AllowN(%v) = %v want %v",
+				i, allow.t, ok, allow.ok)
+		}
+	}
+}
+
+func TestLimiterBurst1(t *testing.T) {
+	run(t, NewLimiter(10, 1), []allow{
+		{t0, true},
+		{t0, false},
+		{t0, false},
+		{t1, true},
+		{t1, false},
+		{t1, false},
+		{t2, true},
+		{t2, false},
+	})
+}
+
+func TestLimiterJumpBackwards(t *testing.T) {
+	run(t, NewLimiter(10, 3), []allow{
+		{t1, true}, // start at t1
+		{t0, true}, // jump back to t0, two tokens remain
+		{t0, true},
+		{t0, false},
+		{t0, false},
+		{t1, true}, // got a token
+		{t1, false},
+		{t1, false},
+		{t2, true}, // got another token
+		{t2, false},
+		{t2, false},
+	})
+}
+
+// Ensure that tokensFromDuration doesn't produce
+// rounding errors by truncating nanoseconds.
+// See golang.org/issues/34861.
+func TestLimiter_noTruncationErrors(t *testing.T) {
+	if !NewLimiter(0.7692307692307693, 1).Allow() {
+		t.Fatal("expected true")
+	}
+}
+
+func TestSimultaneousRequests(t *testing.T) {
+	const (
+		limit       = 1
+		burst       = 5
+		numRequests = 15
+	)
+	var (
+		wg    sync.WaitGroup
+		numOK = uint32(0)
+	)
+
+	// Very slow replenishing bucket.
+	lim := NewLimiter(limit, burst)
+
+	// Tries to take a token, atomically updates the counter and decreases the wait
+	// group counter.
+	f := func() {
+		defer wg.Done()
+		if ok := lim.Allow(); ok {
+			atomic.AddUint32(&numOK, 1)
+		}
+	}
+
+	wg.Add(numRequests)
+	for i := 0; i < numRequests; i++ {
+		go f()
+	}
+	wg.Wait()
+	if numOK != burst {
+		t.Errorf("numOK = %d, want %d", numOK, burst)
+	}
+}
+
+func TestLongRunningQPS(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	if runtime.GOOS == "openbsd" {
+		t.Skip("low resolution time.Sleep invalidates test (golang.org/issue/14183)")
+		return
+	}
+
+	// The test runs for a few seconds executing many requests and then checks
+	// that overall number of requests is reasonable.
+	const (
+		limit = 100
+		burst = 100
+	)
+	var numOK = int32(0)
+
+	lim := NewLimiter(limit, burst)
+
+	var wg sync.WaitGroup
+	f := func() {
+		if ok := lim.Allow(); ok {
+			atomic.AddInt32(&numOK, 1)
+		}
+		wg.Done()
+	}
+
+	start := time.Now()
+	end := start.Add(5 * time.Second)
+	for time.Now().Before(end) {
+		wg.Add(1)
+		go f()
+
+		// This will still offer ~500 requests per second, but won't consume
+		// outrageous amount of CPU.
+		time.Sleep(2 * time.Millisecond)
+	}
+	wg.Wait()
+	elapsed := time.Since(start)
+	ideal := burst + (limit * float64(elapsed) / float64(time.Second))
+
+	// We should never get more requests than allowed.
+	if want := int32(ideal + 1); numOK > want {
+		t.Errorf("numOK = %d, want %d (ideal %f)", numOK, want, ideal)
+	}
+	// We should get very close to the number of requests allowed.
+	if want := int32(0.999 * ideal); numOK < want {
+		t.Errorf("numOK = %d, want %d (ideal %f)", numOK, want, ideal)
+	}
+}
+
+type request struct {
+	t   time.Time
+	n   int
+	act time.Time
+	ok  bool
+}
+
+// dFromDuration converts a duration to a multiple of the global constant d
+func dFromDuration(dur time.Duration) int {
+	// Adding a millisecond to be swallowed by the integer division
+	// because we don't care about small inaccuracies
+	return int((dur + time.Millisecond) / d)
+}
+
+// dSince returns multiples of d since t0
+func dSince(t mono.Time) int {
+	return dFromDuration(t.Sub(t0))
+}
+
+type wait struct {
+	name   string
+	ctx    context.Context
+	n      int
+	delay  int // in multiples of d
+	nilErr bool
+}
+
+func BenchmarkAllowN(b *testing.B) {
+	lim := NewLimiter(Every(1*time.Second), 1)
+	now := mono.Now()
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			lim.allow(now)
+		}
+	})
+}

tsweb/tsweb.go

@@ -409,7 +409,7 @@ func VarzHandler(w http.ResponseWriter, r *http.Request) {
 		case expvar.Func:
 			val := v()
 			switch val.(type) {
-			case int64, int:
+			case float64, int64, int:
 				fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
 			default:
 				fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)

util/deephash/deephash.go

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package deephash hashes a Go value recursively, in a predictable
-// order, without looping.
+// Package deephash hashes a Go value recursively, in a predictable order,
+// without looping. The hash is only valid within the lifetime of a program.
+// Users should not store the hash on disk or send it over the network.
+// The hash is sufficiently strong and unique such that
+// Hash(x) == Hash(y) is an appropriate replacement for x == y.
 //
 // This package, like most of the tailscale.com Go module, should be
 // considered Tailscale-internal; we make no API promises.
@@ -18,8 +21,9 @@ import (
 	"hash"
 	"math"
 	"reflect"
-	"strconv"
 	"sync"
+	"time"
+	"unsafe"
 )
 
 const scratchSize = 128
@@ -27,85 +31,78 @@ const scratchSize = 128
 // hasher is reusable state for hashing a value.
 // Get one via hasherPool.
 type hasher struct {
-	h       hash.Hash
-	bw      *bufio.Writer
-	scratch [scratchSize]byte
-	visited map[uintptr]bool
+	h          hash.Hash
+	bw         *bufio.Writer
+	scratch    [scratchSize]byte
+	visitStack visitStack
 }
 
-// newHasher initializes a new hasher, for use by hasherPool.
-func newHasher() *hasher {
-	h := &hasher{
-		h:       sha256.New(),
-		visited: map[uintptr]bool{},
+func (h *hasher) reset() {
+	if h.h == nil {
+		h.h = sha256.New()
+	}
+	if h.bw == nil {
+		h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
 	}
-	h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
-	return h
-}
-
-// setBufioWriter switches the bufio writer to w after flushing
-// any output to the old one. It then also returns the old one, so
-// the caller can switch back to it.
-func (h *hasher) setBufioWriter(w *bufio.Writer) (old *bufio.Writer) {
-	old = h.bw
-	old.Flush()
-	h.bw = w
-	return old
-}
-
-// Hash returns the raw SHA-256 (not hex) of v.
-func (h *hasher) Hash(v interface{}) (hash [sha256.Size]byte) {
 	h.bw.Flush()
 	h.h.Reset()
-	h.print(reflect.ValueOf(v))
+}
+
+// Sum is an opaque checksum type that is comparable.
+type Sum struct {
+	sum [sha256.Size]byte
+}
+
+func (s1 *Sum) xor(s2 Sum) {
+	for i := 0; i < sha256.Size; i++ {
+		s1.sum[i] ^= s2.sum[i]
+	}
+}
+
+func (s Sum) String() string {
+	return hex.EncodeToString(s.sum[:])
+}
+
+var (
+	once sync.Once
+	seed uint64
+)
+
+func (h *hasher) sum() (s Sum) {
 	h.bw.Flush()
 	// Sum into scratch & copy out, as hash.Hash is an interface
 	// so the slice necessarily escapes, and there's no sha256
 	// concrete type exported and we don't want the 'hash' result
 	// parameter to escape to the heap:
-	h.h.Sum(h.scratch[:0])
-	copy(hash[:], h.scratch[:])
-	return
+	copy(s.sum[:], h.h.Sum(h.scratch[:0]))
+	return s
 }
 
 var hasherPool = &sync.Pool{
-	New: func() interface{} { return newHasher() },
+	New: func() interface{} { return new(hasher) },
 }
 
-// Hash returns the raw SHA-256 hash of v.
-func Hash(v interface{}) [sha256.Size]byte {
+// Hash returns the hash of v.
+func Hash(v interface{}) (s Sum) {
 	h := hasherPool.Get().(*hasher)
 	defer hasherPool.Put(h)
-	for k := range h.visited {
-		delete(h.visited, k)
-	}
-	return h.Hash(v)
+	h.reset()
+	once.Do(func() {
+		seed = uint64(time.Now().UnixNano())
+	})
+	h.hashUint64(seed)
+	h.hashValue(reflect.ValueOf(v))
+	return h.sum()
 }
 
-// UpdateHash sets last to the hex-encoded hash of v and reports whether its value changed.
-func UpdateHash(last *string, v ...interface{}) (changed bool) {
+// Update sets last to the hash of v and reports whether its value changed.
+func Update(last *Sum, v ...interface{}) (changed bool) {
 	sum := Hash(v)
-	if sha256EqualHex(sum, *last) {
+	if sum == *last {
 		// unchanged.
 		return false
 	}
-	*last = hex.EncodeToString(sum[:])
-	return true
-}
-
-// sha256EqualHex reports whether hx is the hex encoding of sum.
-func sha256EqualHex(sum [sha256.Size]byte, hx string) bool {
-	if len(hx) != len(sum)*2 {
-		return false
-	}
-	const hextable = "0123456789abcdef"
-	j := 0
-	for _, v := range sum {
-		if hx[j] != hextable[v>>4] || hx[j+1] != hextable[v&0x0f] {
-			return false
-		}
-		j += 2
-	}
+	*last = sum
 	return true
 }
 
@@ -115,64 +112,78 @@ type appenderTo interface {
 	AppendTo([]byte) []byte
 }
 
-func (h *hasher) uint(i uint64) {
-	binary.BigEndian.PutUint64(h.scratch[:8], i)
-	h.bw.Write(h.scratch[:8])
+func (h *hasher) hashUint8(i uint8) {
+	h.bw.WriteByte(i)
 }
-
-func (h *hasher) int(i int) {
-	binary.BigEndian.PutUint64(h.scratch[:8], uint64(i))
+func (h *hasher) hashUint16(i uint16) {
+	binary.LittleEndian.PutUint16(h.scratch[:2], i)
+	h.bw.Write(h.scratch[:2])
+}
+func (h *hasher) hashUint32(i uint32) {
+	binary.LittleEndian.PutUint32(h.scratch[:4], i)
+	h.bw.Write(h.scratch[:4])
+}
+func (h *hasher) hashUint64(i uint64) {
+	binary.LittleEndian.PutUint64(h.scratch[:8], i)
 	h.bw.Write(h.scratch[:8])
 }
 
 var uint8Type = reflect.TypeOf(byte(0))
 
-// print hashes v into w.
-// It reports whether it was able to do so without hitting a cycle.
-func (h *hasher) print(v reflect.Value) (acyclic bool) {
+func (h *hasher) hashValue(v reflect.Value) {
 	if !v.IsValid() {
-		return true
+		return
 	}
 
 	w := h.bw
-	visited := h.visited
 
 	if v.CanInterface() {
 		// Use AppendTo methods, if available and cheap.
 		if v.CanAddr() && v.Type().Implements(appenderToType) {
 			a := v.Addr().Interface().(appenderTo)
-			scratch := a.AppendTo(h.scratch[:0])
-			w.Write(scratch)
-			return true
+			size := h.scratch[:8]
+			record := a.AppendTo(size)
+			binary.LittleEndian.PutUint64(record, uint64(len(record)-len(size)))
+			w.Write(record)
+			return
 		}
 	}
 
+	// TODO(dsnet): Avoid cycle detection for types that cannot have cycles.
+
 	// Generic handling.
 	switch v.Kind() {
 	default:
 		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
 	case reflect.Ptr:
-		ptr := v.Pointer()
-		if visited[ptr] {
-			return false
+		if v.IsNil() {
+			h.hashUint8(0) // indicates nil
+			return
 		}
-		visited[ptr] = true
-		return h.print(v.Elem())
+
+		// Check for cycle.
+		ptr := pointerOf(v)
+		if idx, ok := h.visitStack.seen(ptr); ok {
+			h.hashUint8(2) // indicates cycle
+			h.hashUint64(uint64(idx))
+			return
+		}
+		h.visitStack.push(ptr)
+		defer h.visitStack.pop(ptr)
+
+		h.hashUint8(1) // indicates visiting a pointer
+		h.hashValue(v.Elem())
 	case reflect.Struct:
-		acyclic = true
 		w.WriteString("struct")
-		h.int(v.NumField())
+		h.hashUint64(uint64(v.NumField()))
 		for i, n := 0, v.NumField(); i < n; i++ {
-			h.int(i)
-			if !h.print(v.Field(i)) {
-				acyclic = false
-			}
+			h.hashUint64(uint64(i))
+			h.hashValue(v.Field(i))
 		}
-		return acyclic
 	case reflect.Slice, reflect.Array:
 		vLen := v.Len()
 		if v.Kind() == reflect.Slice {
-			h.int(vLen)
+			h.hashUint64(uint64(vLen))
 		}
 		if v.Type().Elem() == uint8Type && v.CanInterface() {
 			if vLen > 0 && vLen <= scratchSize {
@@ -182,164 +193,167 @@ func (h *hasher) print(v reflect.Value) (acyclic bool) {
 				// to allocate, so it's not a win.
 				n := reflect.Copy(reflect.ValueOf(&h.scratch).Elem(), v)
 				w.Write(h.scratch[:n])
-				return true
+				return
 			}
 			fmt.Fprintf(w, "%s", v.Interface())
-			return true
+			return
 		}
-		acyclic = true
 		for i := 0; i < vLen; i++ {
-			h.int(i)
-			if !h.print(v.Index(i)) {
-				acyclic = false
-			}
+			// TODO(dsnet): Perform cycle detection for slices,
+			// which is functionally a list of pointers.
+			// See https://github.com/google/go-cmp/blob/402949e8139bb890c71a707b6faf6dd05c92f4e5/cmp/compare.go#L438-L450
+			h.hashUint64(uint64(i))
+			h.hashValue(v.Index(i))
 		}
-		return acyclic
 	case reflect.Interface:
-		return h.print(v.Elem())
-	case reflect.Map:
-		// TODO(bradfitz): ideally we'd avoid these map
-		// operations to detect cycles if we knew from the map
-		// element type that there no way to form a cycle,
-		// which is the common case. Notably, we don't care
-		// about hashing the same map+contents twice in
-		// different parts of the tree. In fact, we should
-		// ideally. (And this prevents it) We should only stop
-		// hashing when there's a cycle.  What we should
-		// probably do is make sure we enumerate the data
-		// structure tree is a fixed order and then give each
-		// pointer an increasing number, and when we hit a
-		// dup, rather than emitting nothing, we should emit a
-		// "value #12" reference. Which implies that all things
-		// emit to the bufio.Writer should be type-tagged so
-		// we can distinguish loop references without risk of
-		// collisions.
-		ptr := v.Pointer()
-		if visited[ptr] {
-			return false
+		if v.IsNil() {
+			h.hashUint8(0) // indicates nil
+			return
 		}
-		visited[ptr] = true
+		v = v.Elem()
 
-		if h.hashMapAcyclic(v) {
-			return true
+		h.hashUint8(1) // indicates visiting interface value
+		h.hashType(v.Type())
+		h.hashValue(v)
+	case reflect.Map:
+		// Check for cycle.
+		ptr := pointerOf(v)
+		if idx, ok := h.visitStack.seen(ptr); ok {
+			h.hashUint8(2) // indicates cycle
+			h.hashUint64(uint64(idx))
+			return
 		}
-		return h.hashMapFallback(v)
+		h.visitStack.push(ptr)
+		defer h.visitStack.pop(ptr)
+
+		h.hashUint8(1) // indicates visiting a map
+		h.hashMap(v)
 	case reflect.String:
-		h.int(v.Len())
-		w.WriteString(v.String())
+		s := v.String()
+		h.hashUint64(uint64(len(s)))
+		w.WriteString(s)
 	case reflect.Bool:
-		w.Write(strconv.AppendBool(h.scratch[:0], v.Bool()))
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		w.Write(strconv.AppendInt(h.scratch[:0], v.Int(), 10))
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		h.uint(v.Uint())
-	case reflect.Float32, reflect.Float64:
-		w.Write(strconv.AppendUint(h.scratch[:0], math.Float64bits(v.Float()), 10))
-	case reflect.Complex64, reflect.Complex128:
-		fmt.Fprintf(w, "%v", v.Complex())
+		if v.Bool() {
+			h.hashUint8(1)
+		} else {
+			h.hashUint8(0)
+		}
+	case reflect.Int8:
+		h.hashUint8(uint8(v.Int()))
+	case reflect.Int16:
+		h.hashUint16(uint16(v.Int()))
+	case reflect.Int32:
+		h.hashUint32(uint32(v.Int()))
+	case reflect.Int64, reflect.Int:
+		h.hashUint64(uint64(v.Int()))
+	case reflect.Uint8:
+		h.hashUint8(uint8(v.Uint()))
+	case reflect.Uint16:
+		h.hashUint16(uint16(v.Uint()))
+	case reflect.Uint32:
+		h.hashUint32(uint32(v.Uint()))
+	case reflect.Uint64, reflect.Uint, reflect.Uintptr:
+		h.hashUint64(uint64(v.Uint()))
+	case reflect.Float32:
+		h.hashUint32(math.Float32bits(float32(v.Float())))
+	case reflect.Float64:
+		h.hashUint64(math.Float64bits(float64(v.Float())))
+	case reflect.Complex64:
+		h.hashUint32(math.Float32bits(real(complex64(v.Complex()))))
+		h.hashUint32(math.Float32bits(imag(complex64(v.Complex()))))
+	case reflect.Complex128:
+		h.hashUint64(math.Float64bits(real(complex128(v.Complex()))))
+		h.hashUint64(math.Float64bits(imag(complex128(v.Complex()))))
 	}
-	return true
 }
 
 type mapHasher struct {
-	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
-	ebuf [sha256.Size]byte // scratch buffer
-	s256 hash.Hash         // sha256 hash.Hash
-	bw   *bufio.Writer     // to hasher into ebuf
-	val  valueCache        // re-usable values for map iteration
-	iter *reflect.MapIter  // re-usable map iterator
-}
-
-func (mh *mapHasher) Reset() {
-	for i := range mh.xbuf {
-		mh.xbuf[i] = 0
-	}
-}
-
-func (mh *mapHasher) startEntry() {
-	for i := range mh.ebuf {
-		mh.ebuf[i] = 0
-	}
-	mh.bw.Flush()
-	mh.s256.Reset()
-}
-
-func (mh *mapHasher) endEntry() {
-	mh.bw.Flush()
-	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
-		mh.xbuf[i] ^= b
-	}
+	h    hasher
+	val  valueCache      // re-usable values for map iteration
+	iter reflect.MapIter // re-usable map iterator
 }
 
 var mapHasherPool = &sync.Pool{
-	New: func() interface{} {
-		mh := new(mapHasher)
-		mh.s256 = sha256.New()
-		mh.bw = bufio.NewWriter(mh.s256)
-		mh.val = make(valueCache)
-		mh.iter = new(reflect.MapIter)
-		return mh
-	},
+	New: func() interface{} { return new(mapHasher) },
 }
 
 type valueCache map[reflect.Type]reflect.Value
 
-func (c valueCache) get(t reflect.Type) reflect.Value {
-	v, ok := c[t]
+func (c *valueCache) get(t reflect.Type) reflect.Value {
+	v, ok := (*c)[t]
 	if !ok {
 		v = reflect.New(t).Elem()
-		c[t] = v
+		if *c == nil {
+			*c = make(valueCache)
+		}
+		(*c)[t] = v
 	}
 	return v
 }
 
-// hashMapAcyclic is the faster sort-free version of map hashing. If
-// it detects a cycle it returns false and guarantees that nothing was
-// written to w.
-func (h *hasher) hashMapAcyclic(v reflect.Value) (acyclic bool) {
+// hashMap hashes a map in a sort-free manner.
+// It relies on a map being a functionally an unordered set of KV entries.
+// So long as we hash each KV entry together, we can XOR all
+// of the individual hashes to produce a unique hash for the entire map.
+func (h *hasher) hashMap(v reflect.Value) {
 	mh := mapHasherPool.Get().(*mapHasher)
 	defer mapHasherPool.Put(mh)
-	mh.Reset()
-	iter := mapIter(mh.iter, v)
-	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
-
-	// Temporarily switch to the map hasher's bufio.Writer.
-	oldw := h.setBufioWriter(mh.bw)
-	defer h.setBufioWriter(oldw)
+	iter := mapIter(&mh.iter, v)
+	defer mapIter(&mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
 
+	var sum Sum
 	k := mh.val.get(v.Type().Key())
 	e := mh.val.get(v.Type().Elem())
+	mh.h.visitStack = h.visitStack // always use the parent's visit stack to avoid cycles
 	for iter.Next() {
 		key := iterKey(iter, k)
 		val := iterVal(iter, e)
-		mh.startEntry()
-		if !h.print(key) {
-			return false
-		}
-		if !h.print(val) {
-			return false
-		}
-		mh.endEntry()
+		mh.h.reset()
+		mh.h.hashValue(key)
+		mh.h.hashValue(val)
+		sum.xor(mh.h.sum())
 	}
-	oldw.Write(mh.xbuf[:])
-	return true
+	h.bw.Write(append(h.scratch[:0], sum.sum[:]...)) // append into scratch to avoid heap allocation
 }
 
-func (h *hasher) hashMapFallback(v reflect.Value) (acyclic bool) {
-	acyclic = true
-	sm := newSortedMap(v)
-	w := h.bw
-	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
-	for i, k := range sm.Key {
-		if !h.print(k) {
-			acyclic = false
-		}
-		w.WriteString(": ")
-		if !h.print(sm.Value[i]) {
-			acyclic = false
-		}
-		w.WriteString("\n")
-	}
-	w.WriteString("}\n")
-	return acyclic
+// visitStack is a stack of pointers visited.
+// Pointers are pushed onto the stack when visited, and popped when leaving.
+// The integer value is the depth at which the pointer was visited.
+// The length of this stack should be zero after every hashing operation.
+type visitStack map[pointer]int
+
+func (v visitStack) seen(p pointer) (int, bool) {
+	idx, ok := v[p]
+	return idx, ok
+}
+
+func (v *visitStack) push(p pointer) {
+	if *v == nil {
+		*v = make(map[pointer]int)
+	}
+	(*v)[p] = len(*v)
+}
+
+func (v visitStack) pop(p pointer) {
+	delete(v, p)
+}
+
+// pointer is a thin wrapper over unsafe.Pointer.
+// We only rely on comparability of pointers; we cannot rely on uintptr since
+// that would break if Go ever switched to a moving GC.
+type pointer struct{ p unsafe.Pointer }
+
+func pointerOf(v reflect.Value) pointer {
+	return pointer{unsafe.Pointer(v.Pointer())}
+}
+
+// hashType hashes a reflect.Type.
+// The hash is only consistent within the lifetime of a program.
+func (h *hasher) hashType(t reflect.Type) {
+	// This approach relies on reflect.Type always being backed by a unique
+	// *reflect.rtype pointer. A safer approach is to use a global sync.Map
+	// that maps reflect.Type to some arbitrary and unique index.
+	// While safer, it requires global state with memory that can never be GC'd.
+	rtypeAddr := reflect.ValueOf(t).Pointer() // address of *reflect.rtype
+	h.hashUint64(uint64(rtypeAddr))
 }

util/deephash/deephash_test.go

@@ -5,11 +5,11 @@
 package deephash
 
 import (
+	"archive/tar"
 	"bufio"
 	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
+	"math"
 	"reflect"
 	"testing"
 
@@ -23,6 +23,95 @@ import (
 	"tailscale.com/wgengine/wgcfg"
 )
 
+type appendBytes []byte
+
+func (p appendBytes) AppendTo(b []byte) []byte {
+	return append(b, p...)
+}
+
+func TestHash(t *testing.T) {
+	type tuple [2]interface{}
+	type iface struct{ X interface{} }
+	type scalars struct {
+		I8   int8
+		I16  int16
+		I32  int32
+		I64  int64
+		I    int
+		U8   uint8
+		U16  uint16
+		U32  uint32
+		U64  uint64
+		U    uint
+		UP   uintptr
+		F32  float32
+		F64  float64
+		C64  complex64
+		C128 complex128
+	}
+	type MyBool bool
+	type MyHeader tar.Header
+	tests := []struct {
+		in     tuple
+		wantEq bool
+	}{
+		{in: tuple{false, true}, wantEq: false},
+		{in: tuple{true, true}, wantEq: true},
+		{in: tuple{false, false}, wantEq: true},
+		{
+			in: tuple{
+				scalars{-8, -16, -32, -64, -1234, 8, 16, 32, 64, 1234, 5678, 32.32, 64.64, 32 + 32i, 64 + 64i},
+				scalars{-8, -16, -32, -64, -1234, 8, 16, 32, 64, 1234, 5678, 32.32, 64.64, 32 + 32i, 64 + 64i},
+			},
+			wantEq: true,
+		},
+		{in: tuple{scalars{I8: math.MinInt8}, scalars{I8: math.MinInt8 / 2}}, wantEq: false},
+		{in: tuple{scalars{I16: math.MinInt16}, scalars{I16: math.MinInt16 / 2}}, wantEq: false},
+		{in: tuple{scalars{I32: math.MinInt32}, scalars{I32: math.MinInt32 / 2}}, wantEq: false},
+		{in: tuple{scalars{I64: math.MinInt64}, scalars{I64: math.MinInt64 / 2}}, wantEq: false},
+		{in: tuple{scalars{I: -1234}, scalars{I: -1234 / 2}}, wantEq: false},
+		{in: tuple{scalars{U8: math.MaxUint8}, scalars{U8: math.MaxUint8 / 2}}, wantEq: false},
+		{in: tuple{scalars{U16: math.MaxUint16}, scalars{U16: math.MaxUint16 / 2}}, wantEq: false},
+		{in: tuple{scalars{U32: math.MaxUint32}, scalars{U32: math.MaxUint32 / 2}}, wantEq: false},
+		{in: tuple{scalars{U64: math.MaxUint64}, scalars{U64: math.MaxUint64 / 2}}, wantEq: false},
+		{in: tuple{scalars{U: 1234}, scalars{U: 1234 / 2}}, wantEq: false},
+		{in: tuple{scalars{UP: 5678}, scalars{UP: 5678 / 2}}, wantEq: false},
+		{in: tuple{scalars{F32: 32.32}, scalars{F32: math.Nextafter32(32.32, 0)}}, wantEq: false},
+		{in: tuple{scalars{F64: 64.64}, scalars{F64: math.Nextafter(64.64, 0)}}, wantEq: false},
+		{in: tuple{scalars{F32: float32(math.NaN())}, scalars{F32: float32(math.NaN())}}, wantEq: true},
+		{in: tuple{scalars{F64: float64(math.NaN())}, scalars{F64: float64(math.NaN())}}, wantEq: true},
+		{in: tuple{scalars{C64: 32 + 32i}, scalars{C64: complex(math.Nextafter32(32, 0), 32)}}, wantEq: false},
+		{in: tuple{scalars{C128: 64 + 64i}, scalars{C128: complex(math.Nextafter(64, 0), 64)}}, wantEq: false},
+		{in: tuple{[]appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}, []appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}}, wantEq: true},
+		{in: tuple{[]appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}, []appendBytes{{0, 0, 0, 0, 0, 0, 0, 1}, {}}}, wantEq: false},
+		{in: tuple{iface{MyBool(true)}, iface{MyBool(true)}}, wantEq: true},
+		{in: tuple{iface{true}, iface{MyBool(true)}}, wantEq: false},
+		{in: tuple{iface{MyHeader{}}, iface{MyHeader{}}}, wantEq: true},
+		{in: tuple{iface{MyHeader{}}, iface{tar.Header{}}}, wantEq: false},
+		{in: tuple{iface{&MyHeader{}}, iface{&MyHeader{}}}, wantEq: true},
+		{in: tuple{iface{&MyHeader{}}, iface{&tar.Header{}}}, wantEq: false},
+		{in: tuple{iface{[]map[string]MyBool{}}, iface{[]map[string]MyBool{}}}, wantEq: true},
+		{in: tuple{iface{[]map[string]bool{}}, iface{[]map[string]MyBool{}}}, wantEq: false},
+		{
+			in: func() tuple {
+				i1 := 1
+				i2 := 2
+				v1 := [3]*int{&i1, &i2, &i1}
+				v2 := [3]*int{&i1, &i2, &i2}
+				return tuple{v1, v2}
+			}(),
+			wantEq: false,
+		},
+	}
+
+	for _, tt := range tests {
+		gotEq := Hash(tt.in[0]) == Hash(tt.in[1])
+		if gotEq != tt.wantEq {
+			t.Errorf("(Hash(%v) == Hash(%v)) = %v, want %v", tt.in[0], tt.in[1], gotEq, tt.wantEq)
+		}
+	}
+}
+
 func TestDeepHash(t *testing.T) {
 	// v contains the types of values we care about for our current callers.
 	// Mostly we're just testing that we don't panic on handled types.
@@ -158,13 +247,8 @@ func TestHashMapAcyclic(t *testing.T) {
 		v := reflect.ValueOf(m)
 		buf.Reset()
 		bw.Reset(&buf)
-		h := &hasher{
-			bw:      bw,
-			visited: map[uintptr]bool{},
-		}
-		if !h.hashMapAcyclic(v) {
-			t.Fatal("returned false")
-		}
+		h := &hasher{bw: bw}
+		h.hashMap(v)
 		if got[string(buf.Bytes())] {
 			continue
 		}
@@ -179,17 +263,14 @@ func TestPrintArray(t *testing.T) {
 	type T struct {
 		X [32]byte
 	}
-	x := &T{X: [32]byte{1: 1, 31: 31}}
+	x := T{X: [32]byte{1: 1, 31: 31}}
 	var got bytes.Buffer
 	bw := bufio.NewWriter(&got)
-	h := &hasher{
-		bw:      bw,
-		visited: map[uintptr]bool{},
-	}
-	h.print(reflect.ValueOf(x))
+	h := &hasher{bw: bw}
+	h.hashValue(reflect.ValueOf(x))
 	bw.Flush()
 	const want = "struct" +
-		"\x00\x00\x00\x00\x00\x00\x00\x01" + // 1 field
+		"\x01\x00\x00\x00\x00\x00\x00\x00" + // 1 field
 		"\x00\x00\x00\x00\x00\x00\x00\x00" + // 0th field
 		// the 32 bytes:
 		"\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f"
@@ -209,17 +290,12 @@ func BenchmarkHashMapAcyclic(b *testing.B) {
 	bw := bufio.NewWriter(&buf)
 	v := reflect.ValueOf(m)
 
-	h := &hasher{
-		bw:      bw,
-		visited: map[uintptr]bool{},
-	}
+	h := &hasher{bw: bw}
 
 	for i := 0; i < b.N; i++ {
 		buf.Reset()
 		bw.Reset(&buf)
-		if !h.hashMapAcyclic(v) {
-			b.Fatal("returned false")
-		}
+		h.hashMap(v)
 	}
 }
 
@@ -233,7 +309,7 @@ func BenchmarkTailcfgNode(b *testing.B) {
 }
 
 func TestExhaustive(t *testing.T) {
-	seen := make(map[[sha256.Size]byte]bool)
+	seen := make(map[Sum]bool)
 	for i := 0; i < 100000; i++ {
 		s := Hash(i)
 		if seen[s] {
@@ -243,19 +319,6 @@ func TestExhaustive(t *testing.T) {
 	}
 }
 
-func TestSHA256EqualHex(t *testing.T) {
-	for i := 0; i < 1000; i++ {
-		sum := Hash(i)
-		hx := hex.EncodeToString(sum[:])
-		if !sha256EqualHex(sum, hx) {
-			t.Fatal("didn't match, should've")
-		}
-		if sha256EqualHex(sum, hx[:len(hx)-1]) {
-			t.Fatal("matched on wrong length")
-		}
-	}
-}
-
 // verify this doesn't loop forever, as it used to (Issue 2340)
 func TestMapCyclicFallback(t *testing.T) {
 	type T struct {

util/deephash/fmtsort.go

@@ -1,224 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// and
-
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
-
-package deephash
-
-import (
-	"reflect"
-	"sort"
-)
-
-// Note: Throughout this package we avoid calling reflect.Value.Interface as
-// it is not always legal to do so and it's easier to avoid the issue than to face it.
-
-// sortedMap represents a map's keys and values. The keys and values are
-// aligned in index order: Value[i] is the value in the map corresponding to Key[i].
-type sortedMap struct {
-	Key   []reflect.Value
-	Value []reflect.Value
-}
-
-func (o *sortedMap) Len() int           { return len(o.Key) }
-func (o *sortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 }
-func (o *sortedMap) Swap(i, j int) {
-	o.Key[i], o.Key[j] = o.Key[j], o.Key[i]
-	o.Value[i], o.Value[j] = o.Value[j], o.Value[i]
-}
-
-// Sort accepts a map and returns a sortedMap that has the same keys and
-// values but in a stable sorted order according to the keys, modulo issues
-// raised by unorderable key values such as NaNs.
-//
-// The ordering rules are more general than with Go's < operator:
-//
-//  - when applicable, nil compares low
-//  - ints, floats, and strings order by <
-//  - NaN compares less than non-NaN floats
-//  - bool compares false before true
-//  - complex compares real, then imag
-//  - pointers compare by machine address
-//  - channel values compare by machine address
-//  - structs compare each field in turn
-//  - arrays compare each element in turn.
-//    Otherwise identical arrays compare by length.
-//  - interface values compare first by reflect.Type describing the concrete type
-//    and then by concrete value as described in the previous rules.
-//
-func newSortedMap(mapValue reflect.Value) *sortedMap {
-	if mapValue.Type().Kind() != reflect.Map {
-		return nil
-	}
-	// Note: this code is arranged to not panic even in the presence
-	// of a concurrent map update. The runtime is responsible for
-	// yelling loudly if that happens. See issue 33275.
-	n := mapValue.Len()
-	key := make([]reflect.Value, 0, n)
-	value := make([]reflect.Value, 0, n)
-	iter := mapValue.MapRange()
-	for iter.Next() {
-		key = append(key, iter.Key())
-		value = append(value, iter.Value())
-	}
-	sorted := &sortedMap{
-		Key:   key,
-		Value: value,
-	}
-	sort.Stable(sorted)
-	return sorted
-}
-
-// compare compares two values of the same type. It returns -1, 0, 1
-// according to whether a > b (1), a == b (0), or a < b (-1).
-// If the types differ, it returns -1.
-// See the comment on Sort for the comparison rules.
-func compare(aVal, bVal reflect.Value) int {
-	aType, bType := aVal.Type(), bVal.Type()
-	if aType != bType {
-		return -1 // No good answer possible, but don't return 0: they're not equal.
-	}
-	switch aVal.Kind() {
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		a, b := aVal.Int(), bVal.Int()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		a, b := aVal.Uint(), bVal.Uint()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.String:
-		a, b := aVal.String(), bVal.String()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Float32, reflect.Float64:
-		return floatCompare(aVal.Float(), bVal.Float())
-	case reflect.Complex64, reflect.Complex128:
-		a, b := aVal.Complex(), bVal.Complex()
-		if c := floatCompare(real(a), real(b)); c != 0 {
-			return c
-		}
-		return floatCompare(imag(a), imag(b))
-	case reflect.Bool:
-		a, b := aVal.Bool(), bVal.Bool()
-		switch {
-		case a == b:
-			return 0
-		case a:
-			return 1
-		default:
-			return -1
-		}
-	case reflect.Ptr:
-		a, b := aVal.Pointer(), bVal.Pointer()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Chan:
-		if c, ok := nilCompare(aVal, bVal); ok {
-			return c
-		}
-		ap, bp := aVal.Pointer(), bVal.Pointer()
-		switch {
-		case ap < bp:
-			return -1
-		case ap > bp:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Struct:
-		for i := 0; i < aVal.NumField(); i++ {
-			if c := compare(aVal.Field(i), bVal.Field(i)); c != 0 {
-				return c
-			}
-		}
-		return 0
-	case reflect.Array:
-		for i := 0; i < aVal.Len(); i++ {
-			if c := compare(aVal.Index(i), bVal.Index(i)); c != 0 {
-				return c
-			}
-		}
-		return 0
-	case reflect.Interface:
-		if c, ok := nilCompare(aVal, bVal); ok {
-			return c
-		}
-		c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type()))
-		if c != 0 {
-			return c
-		}
-		return compare(aVal.Elem(), bVal.Elem())
-	default:
-		// Certain types cannot appear as keys (maps, funcs, slices), but be explicit.
-		panic("bad type in compare: " + aType.String())
-	}
-}
-
-// nilCompare checks whether either value is nil. If not, the boolean is false.
-// If either value is nil, the boolean is true and the integer is the comparison
-// value. The comparison is defined to be 0 if both are nil, otherwise the one
-// nil value compares low. Both arguments must represent a chan, func,
-// interface, map, pointer, or slice.
-func nilCompare(aVal, bVal reflect.Value) (int, bool) {
-	if aVal.IsNil() {
-		if bVal.IsNil() {
-			return 0, true
-		}
-		return -1, true
-	}
-	if bVal.IsNil() {
-		return 1, true
-	}
-	return 0, false
-}
-
-// floatCompare compares two floating-point values. NaNs compare low.
-func floatCompare(a, b float64) int {
-	switch {
-	case isNaN(a):
-		return -1 // No good answer if b is a NaN so don't bother checking.
-	case isNaN(b):
-		return 1
-	case a < b:
-		return -1
-	case a > b:
-		return 1
-	}
-	return 0
-}
-
-func isNaN(a float64) bool {
-	return a != a
-}

version/version.go

@@ -8,7 +8,7 @@ package version
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-var Long = "date.20210603"
+var Long = "date.20210727"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.

wgengine/filter/filter.go

@@ -11,10 +11,10 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/rate"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 )

wgengine/filter/filter_test.go

@@ -13,11 +13,11 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/rate"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 )

wgengine/magicsock/legacy.go

@@ -23,6 +23,7 @@ import (
 	"golang.zx2c4.com/wireguard/tai64n"
 	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
@@ -348,12 +349,12 @@ type addrSet struct {
 	ipPorts []netaddr.IPPort
 
 	// clock, if non-nil, is used in tests instead of time.Now.
-	clock func() time.Time
+	clock func() mono.Time
 	Logf  logger.Logf // must not be nil
 
 	mu sync.Mutex // guards following fields
 
-	lastSend time.Time
+	lastSend mono.Time
 
 	// roamAddr is non-nil if/when we receive a correctly signed
 	// WireGuard packet from an unexpected address. If so, we
@@ -369,10 +370,10 @@ type addrSet struct {
 	curAddr int
 
 	// stopSpray is the time after which we stop spraying packets.
-	stopSpray time.Time
+	stopSpray mono.Time
 
 	// lastSpray is the last time we sprayed a packet.
-	lastSpray time.Time
+	lastSpray mono.Time
 
 	// loggedLogPriMask is a bit field of that tracks whether
 	// we've already logged about receiving a packet from a low
@@ -395,11 +396,11 @@ func (as *addrSet) derpID() int {
 	return 0
 }
 
-func (as *addrSet) timeNow() time.Time {
+func (as *addrSet) timeNow() mono.Time {
 	if as.clock != nil {
 		return as.clock()
 	}
-	return time.Now()
+	return mono.Now()
 }
 
 var noAddr, _ = netaddr.FromStdAddr(net.ParseIP("127.127.127.127"), 127, "")

wgengine/magicsock/magicsock.go

@@ -47,6 +47,7 @@ import (
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -83,6 +84,8 @@ var (
 	// on mobile devices, lowers the shutdown interval, and logs more
 	// verbosely about idle measurements.
 	debugReSTUNStopOnIdle, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_RESTUN_STOP_ON_IDLE"))
+	// debugAlwaysDERP disables the use of UDP, forcing all peer communication over DERP.
+	debugAlwaysDERP, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_ALWAYS_USE_DERP"))
 )
 
 // useDerpRoute reports whether magicsock should enable the DERP
@@ -812,20 +815,20 @@ func (c *Conn) SetNetInfoCallback(fn func(*tailcfg.NetInfo)) {
 	}
 }
 
-// LastRecvActivityOfDisco returns the time we last got traffic from
+// LastRecvActivityOfDisco describes the time we last got traffic from
 // this endpoint (updated every ~10 seconds).
-func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) time.Time {
+func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) string {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	de, ok := c.endpointOfDisco[dk]
 	if !ok {
-		return time.Time{}
+		return "never"
 	}
-	unix := atomic.LoadInt64(&de.lastRecvUnixAtomic)
-	if unix == 0 {
-		return time.Time{}
+	saw := de.lastRecv.LoadAtomic()
+	if saw == 0 {
+		return "never"
 	}
-	return time.Unix(unix, 0)
+	return mono.Since(saw).Round(time.Second).String()
 }
 
 // Ping handles a "tailscale ping" CLI query.
@@ -2665,6 +2668,12 @@ func (c *Conn) bindSocket(rucPtr **RebindingUDPConn, network string, curPortFate
 	ruc.mu.Lock()
 	defer ruc.mu.Unlock()
 
+	if debugAlwaysDERP {
+		c.logf("disabled %v per TS_DEBUG_ALWAYS_USE_DERP", network)
+		ruc.pconn = newBlockForeverConn()
+		return nil
+	}
+
 	// Build a list of preferred ports.
 	// Best is the port that the user requested.
 	// Second best is the port that is currently in use.
@@ -3118,7 +3127,7 @@ func ippDebugString(ua netaddr.IPPort) string {
 // advertise a DiscoKey and participate in active discovery.
 type discoEndpoint struct {
 	// atomically accessed; declared first for alignment reasons
-	lastRecvUnixAtomic    int64
+	lastRecv              mono.Time
 	numStopAndResetAtomic int64
 
 	// These fields are initialized once and never modified.
@@ -3137,13 +3146,13 @@ type discoEndpoint struct {
 	mu sync.Mutex // Lock ordering: Conn.mu, then discoEndpoint.mu
 
 	heartBeatTimer *time.Timer    // nil when idle
-	lastSend       time.Time      // last time there was outgoing packets sent to this peer (from wireguard-go)
-	lastFullPing   time.Time      // last time we pinged all endpoints
+	lastSend       mono.Time      // last time there was outgoing packets sent to this peer (from wireguard-go)
+	lastFullPing   mono.Time      // last time we pinged all endpoints
 	derpAddr       netaddr.IPPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients)
 
 	bestAddr           addrLatency // best non-DERP path; zero if none
-	bestAddrAt         time.Time   // time best address re-confirmed
-	trustBestAddrUntil time.Time   // time when bestAddr expires
+	bestAddrAt         mono.Time   // time best address re-confirmed
+	trustBestAddrUntil mono.Time   // time when bestAddr expires
 	sentPing           map[stun.TxID]sentPing
 	endpointState      map[netaddr.IPPort]*endpointState
 	isCallMeMaybeEP    map[netaddr.IPPort]bool
@@ -3210,7 +3219,7 @@ type endpointState struct {
 	// all fields guarded by discoEndpoint.mu
 
 	// lastPing is the last (outgoing) ping time.
-	lastPing time.Time
+	lastPing mono.Time
 
 	// lastGotPing, if non-zero, means that this was an endpoint
 	// that we learned about at runtime (from an incoming ping)
@@ -3258,14 +3267,14 @@ const pongHistoryCount = 64
 
 type pongReply struct {
 	latency time.Duration
-	pongAt  time.Time      // when we received the pong
+	pongAt  mono.Time      // when we received the pong
 	from    netaddr.IPPort // the pong's src (usually same as endpoint map key)
 	pongSrc netaddr.IPPort // what they reported they heard
 }
 
 type sentPing struct {
 	to      netaddr.IPPort
-	at      time.Time
+	at      mono.Time
 	timer   *time.Timer // timeout timer
 	purpose discoPingPurpose
 }
@@ -3285,10 +3294,10 @@ func (de *discoEndpoint) initFakeUDPAddr() {
 // endpoint and reports whether it's been at least 10 seconds since the last
 // receive activity (including having never received from this peer before).
 func (de *discoEndpoint) isFirstRecvActivityInAwhile() bool {
-	now := time.Now().Unix()
-	old := atomic.LoadInt64(&de.lastRecvUnixAtomic)
-	if old <= now-10 {
-		atomic.StoreInt64(&de.lastRecvUnixAtomic, now)
+	now := mono.Now()
+	elapsed := now.Sub(de.lastRecv.LoadAtomic())
+	if elapsed > 10*time.Second {
+		de.lastRecv.StoreAtomic(now)
 		return true
 	}
 	return false
@@ -3313,7 +3322,7 @@ func (de *discoEndpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr)
 // addr may be non-zero.
 //
 // de.mu must be held.
-func (de *discoEndpoint) addrForSendLocked(now time.Time) (udpAddr, derpAddr netaddr.IPPort) {
+func (de *discoEndpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netaddr.IPPort) {
 	udpAddr = de.bestAddr.IPPort
 	if udpAddr.IsZero() || now.After(de.trustBestAddrUntil) {
 		// We had a bestAddr but it expired so send both to it
@@ -3336,13 +3345,13 @@ func (de *discoEndpoint) heartbeat() {
 		return
 	}
 
-	if time.Since(de.lastSend) > sessionActiveTimeout {
+	if mono.Since(de.lastSend) > sessionActiveTimeout {
 		// Session's idle. Stop heartbeating.
 		de.c.logf("[v1] magicsock: disco: ending heartbeats for idle session to %v (%v)", de.publicKey.ShortString(), de.discoShort)
 		return
 	}
 
-	now := time.Now()
+	now := mono.Now()
 	udpAddr, _ := de.addrForSendLocked(now)
 	if !udpAddr.IsZero() {
 		// We have a preferred path. Ping that every 2 seconds.
@@ -3360,7 +3369,7 @@ func (de *discoEndpoint) heartbeat() {
 // a better path.
 //
 // de.mu must be held.
-func (de *discoEndpoint) wantFullPingLocked(now time.Time) bool {
+func (de *discoEndpoint) wantFullPingLocked(now mono.Time) bool {
 	if de.bestAddr.IsZero() || de.lastFullPing.IsZero() {
 		return true
 	}
@@ -3377,7 +3386,7 @@ func (de *discoEndpoint) wantFullPingLocked(now time.Time) bool {
 }
 
 func (de *discoEndpoint) noteActiveLocked() {
-	de.lastSend = time.Now()
+	de.lastSend = mono.Now()
 	if de.heartBeatTimer == nil {
 		de.heartBeatTimer = time.AfterFunc(heartbeatInterval, de.heartbeat)
 	}
@@ -3391,7 +3400,7 @@ func (de *discoEndpoint) cliPing(res *ipnstate.PingResult, cb func(*ipnstate.Pin
 
 	de.pendingCLIPings = append(de.pendingCLIPings, pendingCLIPing{res, cb})
 
-	now := time.Now()
+	now := mono.Now()
 	udpAddr, derpAddr := de.addrForSendLocked(now)
 	if !derpAddr.IsZero() {
 		de.startPingLocked(derpAddr, now, pingCLI)
@@ -3411,7 +3420,7 @@ func (de *discoEndpoint) cliPing(res *ipnstate.PingResult, cb func(*ipnstate.Pin
 }
 
 func (de *discoEndpoint) send(b []byte) error {
-	now := time.Now()
+	now := mono.Now()
 
 	de.mu.Lock()
 	udpAddr, derpAddr := de.addrForSendLocked(now)
@@ -3444,7 +3453,7 @@ func (de *discoEndpoint) pingTimeout(txid stun.TxID) {
 	if !ok {
 		return
 	}
-	if debugDisco || de.bestAddr.IsZero() || time.Now().After(de.trustBestAddrUntil) {
+	if debugDisco || de.bestAddr.IsZero() || mono.Now().After(de.trustBestAddrUntil) {
 		de.c.logf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort)
 	}
 	de.removeSentPingLocked(txid, sp)
@@ -3481,7 +3490,7 @@ func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLev
 // discoPingPurpose is the reason why a discovery ping message was sent.
 type discoPingPurpose int
 
-//go:generate stringer -type=discoPingPurpose -trimprefix=ping
+//go:generate go run tailscale.com/cmd/addlicense -year 2020 -file discopingpurpose_string.go go run golang.org/x/tools/cmd/stringer -type=discoPingPurpose -trimprefix=ping
 const (
 	// pingDiscovery means that purpose of a ping was to see if a
 	// path was valid.
@@ -3496,7 +3505,7 @@ const (
 	pingCLI
 )
 
-func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time, purpose discoPingPurpose) {
+func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now mono.Time, purpose discoPingPurpose) {
 	if purpose != pingCLI {
 		st, ok := de.endpointState[ep]
 		if !ok {
@@ -3522,7 +3531,7 @@ func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time, purpo
 	go de.sendDiscoPing(ep, txid, logLevel)
 }
 
-func (de *discoEndpoint) sendPingsLocked(now time.Time, sendCallMeMaybe bool) {
+func (de *discoEndpoint) sendPingsLocked(now mono.Time, sendCallMeMaybe bool) {
 	de.lastFullPing = now
 	var sentAny bool
 	for ep, st := range de.endpointState {
@@ -3644,7 +3653,7 @@ func (de *discoEndpoint) noteConnectivityChange() {
 	de.mu.Lock()
 	defer de.mu.Unlock()
 
-	de.trustBestAddrUntil = time.Time{}
+	de.trustBestAddrUntil = 0
 }
 
 // handlePongConnLocked handles a Pong message (a reply to an earlier ping).
@@ -3662,7 +3671,7 @@ func (de *discoEndpoint) handlePongConnLocked(m *disco.Pong, src netaddr.IPPort)
 	}
 	de.removeSentPingLocked(m.TxID, sp)
 
-	now := time.Now()
+	now := mono.Now()
 	latency := now.Sub(sp.at)
 
 	if !isDerp {
@@ -3814,9 +3823,9 @@ func (de *discoEndpoint) handleCallMeMaybe(m *disco.CallMeMaybe) {
 	// Zero out all the lastPing times to force sendPingsLocked to send new ones,
 	// even if it's been less than 5 seconds ago.
 	for _, st := range de.endpointState {
-		st.lastPing = time.Time{}
+		st.lastPing = 0
 	}
-	de.sendPingsLocked(time.Now(), false)
+	de.sendPingsLocked(mono.Now(), false)
 }
 
 func (de *discoEndpoint) populatePeerStatus(ps *ipnstate.PeerStatus) {
@@ -3829,7 +3838,7 @@ func (de *discoEndpoint) populatePeerStatus(ps *ipnstate.PeerStatus) {
 
 	ps.LastWrite = de.lastSend
 
-	now := time.Now()
+	now := mono.Now()
 	if udpAddr, derpAddr := de.addrForSendLocked(now); !udpAddr.IsZero() && derpAddr.IsZero() {
 		ps.CurAddr = udpAddr.String()
 	}
@@ -3847,13 +3856,13 @@ func (de *discoEndpoint) stopAndReset() {
 
 	// Zero these fields so if the user re-starts the network, the discovery
 	// state isn't a mix of before & after two sessions.
-	de.lastSend = time.Time{}
-	de.lastFullPing = time.Time{}
+	de.lastSend = 0
+	de.lastFullPing = 0
 	de.bestAddr = addrLatency{}
-	de.bestAddrAt = time.Time{}
-	de.trustBestAddrUntil = time.Time{}
+	de.bestAddrAt = 0
+	de.trustBestAddrUntil = 0
 	for _, es := range de.endpointState {
-		es.lastPing = time.Time{}
+		es.lastPing = 0
 	}
 
 	for txid, sp := range de.sentPing {

wgengine/magicsock/magicsock_test.go

@@ -38,6 +38,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/natlab"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -1146,13 +1147,13 @@ func TestAddrSet(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			faket := time.Unix(0, 0)
+			faket := mono.Time(0)
 			var logBuf bytes.Buffer
 			tt.as.Logf = func(format string, args ...interface{}) {
 				fmt.Fprintf(&logBuf, format, args...)
 				t.Logf(format, args...)
 			}
-			tt.as.clock = func() time.Time { return faket }
+			tt.as.clock = func() mono.Time { return faket }
 			for i, st := range tt.steps {
 				faket = faket.Add(st.advance)
 
@@ -1229,8 +1230,8 @@ func TestDiscoStringLogRace(t *testing.T) {
 func Test32bitAlignment(t *testing.T) {
 	var de discoEndpoint
 
-	if off := unsafe.Offsetof(de.lastRecvUnixAtomic); off%8 != 0 {
-		t.Fatalf("discoEndpoint.lastRecvUnixAtomic is not 8-byte aligned")
+	if off := unsafe.Offsetof(de.lastRecv); off%8 != 0 {
+		t.Fatalf("discoEndpoint.lastRecv is not 8-byte aligned")
 	}
 
 	if !de.isFirstRecvActivityInAwhile() { // verify this doesn't panic on 32-bit

wgengine/monitor/monitor.go

@@ -62,6 +62,7 @@ type Mon struct {
 
 	mu         sync.Mutex // guards all following fields
 	cbs        map[*callbackHandle]ChangeFunc
+	ruleDelCB  map[*callbackHandle]RuleDeleteCallback
 	ifState    *interfaces.State
 	gwValid    bool       // whether gw and gwSelfIP are valid
 	gw         netaddr.IP // our gateway's IP
@@ -148,6 +149,30 @@ func (m *Mon) RegisterChangeCallback(callback ChangeFunc) (unregister func()) {
 	}
 }
 
+// RuleDeleteCallback is a callback when a Linux IP policy routing
+// rule is deleted. The table is the table number (52, 253, 354) and
+// priority is the priority order number (for Tailscale rules
+// currently: 5210, 5230, 5250, 5270)
+type RuleDeleteCallback func(table uint8, priority uint32)
+
+// RegisterRuleDeleteCallback adds callback to the set of parties to be
+// notified (in their own goroutine) when a Linux ip rule is deleted.
+// To remove this callback, call unregister (or close the monitor).
+func (m *Mon) RegisterRuleDeleteCallback(callback RuleDeleteCallback) (unregister func()) {
+	handle := new(callbackHandle)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.ruleDelCB == nil {
+		m.ruleDelCB = map[*callbackHandle]RuleDeleteCallback{}
+	}
+	m.ruleDelCB[handle] = callback
+	return func() {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		delete(m.ruleDelCB, handle)
+	}
+}
+
 // Start starts the monitor.
 // A monitor can only be started & closed once.
 func (m *Mon) Start() {
@@ -242,6 +267,10 @@ func (m *Mon) pump() {
 			time.Sleep(time.Second)
 			continue
 		}
+		if rdm, ok := msg.(ipRuleDeletedMessage); ok {
+			m.notifyRuleDeleted(rdm)
+			continue
+		}
 		if msg.ignore() {
 			continue
 		}
@@ -249,6 +278,14 @@ func (m *Mon) pump() {
 	}
 }
 
+func (m *Mon) notifyRuleDeleted(rdm ipRuleDeletedMessage) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for _, cb := range m.ruleDelCB {
+		go cb(rdm.table, rdm.priority)
+	}
+}
+
 // debounce calls the callback function with a delay between events
 // and exits when a stop is issued.
 func (m *Mon) debounce() {
@@ -338,3 +375,10 @@ func (m *Mon) checkWallTimeAdvanceLocked() {
 	}
 	m.lastWall = now
 }
+
+type ipRuleDeletedMessage struct {
+	table    uint8
+	priority uint32
+}
+
+func (ipRuleDeletedMessage) ignore() bool { return true }

wgengine/monitor/monitor_linux.go

@@ -42,7 +42,9 @@ func newOSMon(logf logger.Logf, m *Mon) (osMon, error) {
 		// address as well to cover things like DHCP deciding to give
 		// us a new address upon renewal - routing wouldn't change,
 		// but all reachability would.
-		Groups: unix.RTMGRP_IPV4_IFADDR | unix.RTMGRP_IPV6_IFADDR | unix.RTMGRP_IPV4_ROUTE | unix.RTMGRP_IPV6_ROUTE,
+		Groups: unix.RTMGRP_IPV4_IFADDR | unix.RTMGRP_IPV6_IFADDR |
+			unix.RTMGRP_IPV4_ROUTE | unix.RTMGRP_IPV6_ROUTE |
+			unix.RTMGRP_IPV4_RULE, // no IPV6_RULE in x/sys/unix
 	})
 	if err != nil {
 		// Google Cloud Run does not implement NETLINK_ROUTE RTMGRP support
@@ -117,6 +119,25 @@ func (c *nlConn) Receive() (message, error) {
 			Dst:     dst,
 			Gateway: gw,
 		}, nil
+	case unix.RTM_NEWRULE:
+		// Probably ourselves adding it.
+		return ignoreMessage{}, nil
+	case unix.RTM_DELRULE:
+		// For https://github.com/tailscale/tailscale/issues/1591 where
+		// systemd-networkd deletes our rules.
+		var rmsg rtnetlink.RouteMessage
+		err := rmsg.UnmarshalBinary(msg.Data)
+		if err != nil {
+			c.logf("ip rule deleted; failed to parse netlink message: %v", err)
+		} else {
+			c.logf("ip rule deleted: %+v", rmsg)
+			// On `ip -4 rule del pref 5210 table main`, logs:
+			// monitor: ip rule deleted: {Family:2 DstLength:0 SrcLength:0 Tos:0 Table:254 Protocol:0 Scope:0 Type:1 Flags:0 Attributes:{Dst:<nil> Src:<nil> Gateway:<nil> OutIface:0 Priority:5210 Table:254 Mark:4294967295 Expires:<nil> Metrics:<nil> Multipath:[]}}
+		}
+		return ipRuleDeletedMessage{
+			table:    rmsg.Table,
+			priority: rmsg.Attributes.Priority,
+		}, nil
 	default:
 		c.logf("unhandled netlink msg type %+v, %q", msg.Header, msg.Data)
 		return unspecifiedMessage{}, nil

wgengine/netstack/netstack.go

@@ -136,8 +136,25 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 	return ns, nil
 }
 
-// Start sets up all the handlers so netstack can start working. Implements
+// wrapProtoHandler returns protocol handler h wrapped in a version
+// that dynamically reconfigures ns's subnet addresses as needed for
+// outbound traffic.
+func (ns *Impl) wrapProtoHandler(h func(stack.TransportEndpointID, *stack.PacketBuffer) bool) func(stack.TransportEndpointID, *stack.PacketBuffer) bool {
+	return func(tei stack.TransportEndpointID, pb *stack.PacketBuffer) bool {
+		addr := tei.LocalAddress
+		ip, ok := netaddr.FromStdIP(net.IP(addr))
+		if !ok {
+			ns.logf("netstack: could not parse local address for incoming connection")
+			return false
+		}
+		if !ns.isLocalIP(ip) {
+			ns.addSubnetAddress(ip)
+		}
+		return h(tei, pb)
+	}
+}
 
+// Start sets up all the handlers so netstack can start working. Implements
 // wgengine.FakeImpl.
 func (ns *Impl) Start() error {
 	ns.e.AddNetworkMapCallback(ns.updateIPs)
@@ -146,25 +163,8 @@ func (ns *Impl) Start() error {
 	const maxInFlightConnectionAttempts = 16
 	tcpFwd := tcp.NewForwarder(ns.ipstack, tcpReceiveBufferSize, maxInFlightConnectionAttempts, ns.acceptTCP)
 	udpFwd := udp.NewForwarder(ns.ipstack, ns.acceptUDP)
-	ns.ipstack.SetTransportProtocolHandler(tcp.ProtocolNumber, func(tei stack.TransportEndpointID, pb *stack.PacketBuffer) bool {
-		addr := tei.LocalAddress
-		var pn tcpip.NetworkProtocolNumber
-		if addr.To4() != "" {
-			pn = ipv4.ProtocolNumber
-		} else {
-			pn = ipv6.ProtocolNumber
-		}
-		ip, ok := netaddr.FromStdIP(net.IP(addr))
-		if !ok {
-			ns.logf("netstack: could not parse local address %s for incoming TCP connection", ip)
-			return false
-		}
-		if !ns.isLocalIP(ip) {
-			ns.addSubnetAddress(pn, ip)
-		}
-		return tcpFwd.HandlePacket(tei, pb)
-	})
-	ns.ipstack.SetTransportProtocolHandler(udp.ProtocolNumber, udpFwd.HandlePacket)
+	ns.ipstack.SetTransportProtocolHandler(tcp.ProtocolNumber, ns.wrapProtoHandler(tcpFwd.HandlePacket))
+	ns.ipstack.SetTransportProtocolHandler(udp.ProtocolNumber, ns.wrapProtoHandler(udpFwd.HandlePacket))
 	go ns.injectOutbound()
 	ns.tundev.PostFilterIn = ns.injectInbound
 	return nil
@@ -215,13 +215,19 @@ func (ns *Impl) updateDNS(nm *netmap.NetworkMap) {
 	ns.dns = DNSMapFromNetworkMap(nm)
 }
 
-func (ns *Impl) addSubnetAddress(pn tcpip.NetworkProtocolNumber, ip netaddr.IP) {
+func (ns *Impl) addSubnetAddress(ip netaddr.IP) {
 	ns.mu.Lock()
 	ns.connsOpenBySubnetIP[ip]++
 	needAdd := ns.connsOpenBySubnetIP[ip] == 1
 	ns.mu.Unlock()
 	// Only register address into netstack for first concurrent connection.
 	if needAdd {
+		var pn tcpip.NetworkProtocolNumber
+		if ip.Is4() {
+			pn = ipv4.ProtocolNumber
+		} else if ip.Is6() {
+			pn = ipv6.ProtocolNumber
+		}
 		ns.ipstack.AddAddress(nicID, pn, tcpip.Address(ip.IPAddr().IP))
 	}
 }
@@ -544,9 +550,9 @@ func (ns *Impl) forwardTCP(client *gonet.TCPConn, wq *waiter.Queue, dialAddr tcp
 }
 
 func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {
-	reqDetails := r.ID()
+	sess := r.ID()
 	if debugNetstack {
-		ns.logf("[v2] UDP ForwarderRequest: %v", stringifyTEI(reqDetails))
+		ns.logf("[v2] UDP ForwarderRequest: %v", stringifyTEI(sess))
 	}
 	var wq waiter.Queue
 	ep, err := r.CreateEndpoint(&wq)
@@ -554,30 +560,50 @@ func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {
 		ns.logf("acceptUDP: could not create endpoint: %v", err)
 		return
 	}
-	localAddr, err := ep.GetLocalAddress()
-	if err != nil {
+	dstAddr, ok := ipPortOfNetstackAddr(sess.LocalAddress, sess.LocalPort)
+	if !ok {
 		return
 	}
-	remoteAddr, err := ep.GetRemoteAddress()
-	if err != nil {
+	srcAddr, ok := ipPortOfNetstackAddr(sess.RemoteAddress, sess.RemotePort)
+	if !ok {
 		return
 	}
+
 	c := gonet.NewUDPConn(ns.ipstack, &wq, ep)
-	go ns.forwardUDP(c, &wq, localAddr, remoteAddr)
+	go ns.forwardUDP(c, &wq, srcAddr, dstAddr)
 }
 
-func (ns *Impl) forwardUDP(client *gonet.UDPConn, wq *waiter.Queue, clientLocalAddr, clientRemoteAddr tcpip.FullAddress) {
-	port := clientLocalAddr.Port
+// forwardUDP proxies between client (with addr clientAddr) and dstAddr.
+//
+// dstAddr may be either a local Tailscale IP, in which we case we proxy to
+// 127.0.0.1, or any other IP (from an advertised subnet), in which case we
+// proxy to it directly.
+func (ns *Impl) forwardUDP(client *gonet.UDPConn, wq *waiter.Queue, clientAddr, dstAddr netaddr.IPPort) {
+	port, srcPort := dstAddr.Port(), clientAddr.Port()
 	ns.logf("[v2] netstack: forwarding incoming UDP connection on port %v", port)
-	backendListenAddr := &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(clientRemoteAddr.Port)}
-	backendRemoteAddr := &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(port)}
-	backendConn, err := net.ListenUDP("udp4", backendListenAddr)
+
+	var backendListenAddr *net.UDPAddr
+	var backendRemoteAddr *net.UDPAddr
+	isLocal := ns.isLocalIP(dstAddr.IP())
+	if isLocal {
+		backendRemoteAddr = &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(port)}
+		backendListenAddr = &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(srcPort)}
+	} else {
+		backendRemoteAddr = dstAddr.UDPAddr()
+		if dstAddr.IP().Is4() {
+			backendListenAddr = &net.UDPAddr{IP: net.ParseIP("0.0.0.0"), Port: int(srcPort)}
+		} else {
+			backendListenAddr = &net.UDPAddr{IP: net.ParseIP("::"), Port: int(srcPort)}
+		}
+	}
+
+	backendConn, err := net.ListenUDP("udp", backendListenAddr)
 	if err != nil {
-		ns.logf("netstack: could not bind local port %v: %v, trying again with random port", clientRemoteAddr.Port, err)
+		ns.logf("netstack: could not bind local port %v: %v, trying again with random port", backendListenAddr.Port, err)
 		backendListenAddr.Port = 0
-		backendConn, err = net.ListenUDP("udp4", backendListenAddr)
+		backendConn, err = net.ListenUDP("udp", backendListenAddr)
 		if err != nil {
-			ns.logf("netstack: could not connect to local UDP server on port %v: %v", port, err)
+			ns.logf("netstack: could not create UDP socket, preventing forwarding to %v: %v", dstAddr, err)
 			return
 		}
 	}
@@ -586,28 +612,47 @@ func (ns *Impl) forwardUDP(client *gonet.UDPConn, wq *waiter.Queue, clientLocalA
 	if !ok {
 		ns.logf("could not get backend local IP:port from %v:%v", backendLocalAddr.IP, backendLocalAddr.Port)
 	}
-	clientRemoteIP, _ := netaddr.FromStdIP(net.ParseIP(clientRemoteAddr.Addr.String()))
-	ns.e.RegisterIPPortIdentity(backendLocalIPPort, clientRemoteIP)
+	if isLocal {
+		ns.e.RegisterIPPortIdentity(backendLocalIPPort, dstAddr.IP())
+	}
 	ctx, cancel := context.WithCancel(context.Background())
-	timer := time.AfterFunc(2*time.Minute, func() {
-		ns.e.UnregisterIPPortIdentity(backendLocalIPPort)
-		ns.logf("netstack: UDP session between %s and %s timed out", clientRemoteAddr, backendRemoteAddr)
+
+	idleTimeout := 2 * time.Minute
+	if port == 53 {
+		// Make DNS packet copies time out much sooner.
+		//
+		// TODO(bradfitz): make DNS queries over UDP forwarding even
+		// cheaper by adding an additional idleTimeout post-DNS-reply.
+		// For instance, after the DNS response goes back out, then only
+		// wait a few seconds (or zero, really)
+		idleTimeout = 30 * time.Second
+	}
+	timer := time.AfterFunc(idleTimeout, func() {
+		if isLocal {
+			ns.e.UnregisterIPPortIdentity(backendLocalIPPort)
+		}
+		ns.logf("netstack: UDP session between %s and %s timed out", backendListenAddr, backendRemoteAddr)
 		cancel()
 		client.Close()
 		backendConn.Close()
 	})
 	extend := func() {
-		timer.Reset(2 * time.Minute)
+		timer.Reset(idleTimeout)
 	}
-	startPacketCopy(ctx, cancel, client, &net.UDPAddr{
-		IP:   net.ParseIP(clientRemoteAddr.Addr.String()),
-		Port: int(clientRemoteAddr.Port),
-	}, backendConn, ns.logf, extend)
+	startPacketCopy(ctx, cancel, client, clientAddr.UDPAddr(), backendConn, ns.logf, extend)
 	startPacketCopy(ctx, cancel, backendConn, backendRemoteAddr, client, ns.logf, extend)
-
+	if isLocal {
+		// Wait for the copies to be done before decrementing the
+		// subnet address count to potentially remove the route.
+		<-ctx.Done()
+		ns.removeSubnetAddress(dstAddr.IP())
+	}
 }
 
 func startPacketCopy(ctx context.Context, cancel context.CancelFunc, dst net.PacketConn, dstAddr net.Addr, src net.PacketConn, logf logger.Logf, extend func()) {
+	if debugNetstack {
+		logf("[v2] netstack: startPacketCopy to %v (%T) from %T", dstAddr, dst, src)
+	}
 	go func() {
 		defer cancel() // tear down the other direction's copy
 		pkt := make([]byte, mtu)
@@ -644,3 +689,7 @@ func stringifyTEI(tei stack.TransportEndpointID) string {
 	remoteHostPort := net.JoinHostPort(tei.RemoteAddress.String(), strconv.Itoa(int(tei.RemotePort)))
 	return fmt.Sprintf("%s -> %s", remoteHostPort, localHostPort)
 }
+
+func ipPortOfNetstackAddr(a tcpip.Address, port uint16) (ipp netaddr.IPPort, ok bool) {
+	return netaddr.FromStdAddr(net.IP(a), int(port), "") // TODO(bradfitz): can do without allocs
+}

wgengine/pendopen.go

@@ -231,7 +231,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	e.logf("open-conn-track: timeout opening %v to node %v; online=%v, lastRecv=%v",
 		flow, n.Key.ShortString(),
 		online,
-		durFmt(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
+		e.magicConn.LastRecvActivityOfDisco(n.DiscoKey))
 }
 
 func durFmt(t time.Time) string {

wgengine/router/router.go

@@ -11,6 +11,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
+	"tailscale.com/wgengine/monitor"
 )
 
 // Router is responsible for managing the system network stack.
@@ -31,9 +32,12 @@ type Router interface {
 
 // New returns a new Router for the current platform, using the
 // provided tun device.
-func New(logf logger.Logf, tundev tun.Device) (Router, error) {
+//
+// If linkMon is nil, it's not used. It's currently (2021-07-20) only
+// used on Linux in some situations.
+func New(logf logger.Logf, tundev tun.Device, linkMon *monitor.Mon) (Router, error) {
 	logf = logger.WithPrefix(logf, "router: ")
-	return newUserspaceRouter(logf, tundev)
+	return newUserspaceRouter(logf, tundev, linkMon)
 }
 
 // Cleanup restores the system network configuration to its original state