net/portmapper: fix UPnP probing, work against all ports

Prior to Tailscale 1.12 it detected UPnP on any port.
Starting with Tailscale 1.11.x, it stopped detecting UPnP on all ports.

Then start plumbing its discovered Location header port number to the
code that was assuming port 5000.

Fixes #2109

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/go_generate.yml

@@ -0,0 +1,34 @@
+name: go generate
+
+on:
+  push:
+    branches:
+      - main
+      - "release-branch/*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16
+
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: check 'go generate' is clean
+        run: |
+          mkdir gentools
+          go build -o gentools/stringer golang.org/x/tools/cmd/stringer
+          PATH="$PATH:$(pwd)/gentools" go generate ./...
+          echo
+          echo
+          git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1)

.github/workflows/xe-experimental-vm-test.yml

@@ -4,8 +4,6 @@ on:
   pull_request:
     paths:
       - "tstest/integration/vms/**"
-  push:
-    branches: [ main ]
   release:
     types: [ created ]
 

Dockerfile

@@ -42,8 +42,7 @@ FROM golang:1.16-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
-COPY go.mod .
-COPY go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .

VERSION.txt

@@ -1 +1 @@
-1.11.0
+1.13.0

cmd/addlicense/main.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program addlicense adds a license header to a file.
+// It is intended for use with 'go generate',
+// so it has a slightly weird usage.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+var (
+	year = flag.Int("year", 0, "copyright year")
+	file = flag.String("file", "", "file to modify")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `
+usage: addlicense -year YEAR -file FILE <subcommand args...>
+`[1:])
+
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, `
+addlicense adds a Tailscale license to the beginning of file,
+using year as the copyright year.
+
+It is intended for use with 'go generate', so it also runs a subcommand,
+which presumably creates the file.
+
+Sample usage:
+
+addlicense -year 2021 -file pull_strings.go stringer -type=pull
+`[1:])
+	os.Exit(2)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	cmd := exec.Command(flag.Arg(0), flag.Args()[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	check(err)
+	b, err := os.ReadFile(*file)
+	check(err)
+	f, err := os.OpenFile(*file, os.O_TRUNC|os.O_WRONLY, 0644)
+	check(err)
+	_, err = fmt.Fprintf(f, license, *year)
+	check(err)
+	_, err = f.Write(b)
+	check(err)
+	err = f.Close()
+	check(err)
+}
+
+func check(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+var license = `
+// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+`[1:]

cmd/derper/derper.go

@@ -58,7 +58,12 @@ func loadConfig() config {
 		return config{PrivateKey: mustNewKey()}
 	}
 	if *configPath == "" {
-		log.Fatalf("derper: -c <config path> not specified")
+		if os.Getuid() == 0 {
+			*configPath = "/var/lib/derper/derper.key"
+		} else {
+			log.Fatalf("derper: -c <config path> not specified")
+		}
+		log.Printf("no config path specified; using %s", *configPath)
 	}
 	b, err := ioutil.ReadFile(*configPath)
 	switch {

cmd/derper/mesh.go

@@ -9,7 +9,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net"
 	"strings"
+	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -39,6 +41,34 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return err
 	}
 	c.MeshKey = s.MeshKey()
+
+	// For meshed peers within a region, connect via VPC addresses.
+	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var d net.Dialer
+		var r net.Resolver
+		if port == "443" && strings.HasSuffix(host, ".tailscale.com") {
+			base := strings.TrimSuffix(host, ".tailscale.com")
+			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+			defer cancel()
+			vpcHost := base + "-vpc.tailscale.com"
+			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
+			if len(ips) > 0 {
+				vpcAddr := net.JoinHostPort(ips[0].String(), port)
+				c, err := d.DialContext(subCtx, network, vpcAddr)
+				if err == nil {
+					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
+					return c, nil
+				}
+				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+			}
+		}
+		return d.DialContext(ctx, network, addr)
+	})
+
 	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
 	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)

cmd/derpprobe/derpprobe.go

@@ -190,6 +190,16 @@ func probe() error {
 }
 
 func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (latency time.Duration, err error) {
+	// The passed in context is a minute for the whole region. The
+	// idea is that each node pair in the region will be done
+	// serially and regularly in the future, reusing connections
+	// (at least in the happy path). For now they don't reuse
+	// connections and probe at most once every 15 seconds. We
+	// bound the duration of a single node pair within a region
+	// so one bad one can't starve others.
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
 	fromc, err := newConn(ctx, dm, from)
 	if err != nil {
 		return 0, err
@@ -201,6 +211,13 @@ func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.D
 	}
 	defer toc.Close()
 
+	// Wait a bit for from's node to hear about to existing on the
+	// other node in the region, in the case where the two nodes
+	// are different.
+	if from.Name != to.Name {
+		time.Sleep(100 * time.Millisecond) // pretty arbitrary
+	}
+
 	// Make a random packet
 	pkt := make([]byte, 8)
 	crand.Read(pkt)

cmd/speedtest/speedtest.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program speedtest provides the speedtest command. The reason to keep it separate from
+// the normal tailscale cli is because it is not yet ready to go in the tailscale binary.
+// It will be included in the tailscale cli after it has been added to tailscaled.
+
+// Example usage for client command: go run cmd/speedtest -host 127.0.0.1:20333 -t 5s
+// This will connect to the server on 127.0.0.1:20333 and start a 5 second download speedtest.
+// Example usage for server command: go run cmd/speedtest -s -host :20333
+// This will start a speedtest server on port 20333.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"text/tabwriter"
+	"time"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/net/speedtest"
+)
+
+// Runs the speedtest command as a commandline program
+func main() {
+	args := os.Args[1:]
+	if err := speedtestCmd.Parse(args); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	err := speedtestCmd.Run(context.Background())
+	if errors.Is(err, flag.ErrHelp) {
+		fmt.Fprintln(os.Stderr, speedtestCmd.ShortUsage)
+		os.Exit(2)
+	}
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}
+
+// speedtestCmd is the root command. It runs either the server or client depending on the
+// flags passed to it.
+var speedtestCmd = &ffcli.Command{
+	Name:       "speedtest",
+	ShortUsage: "speedtest [-host <host:port>] [-s] [-r] [-t <test duration>]",
+	ShortHelp:  "Run a speed test",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("speedtest", flag.ExitOnError)
+		fs.StringVar(&speedtestArgs.host, "host", ":20333", "host:port pair to connect to or listen on")
+		fs.DurationVar(&speedtestArgs.testDuration, "t", speedtest.DefaultDuration, "duration of the speed test")
+		fs.BoolVar(&speedtestArgs.runServer, "s", false, "run a speedtest server")
+		fs.BoolVar(&speedtestArgs.reverse, "r", false, "run in reverse mode (server sends, client receives)")
+		return fs
+	})(),
+	Exec: runSpeedtest,
+}
+
+var speedtestArgs struct {
+	host         string
+	testDuration time.Duration
+	runServer    bool
+	reverse      bool
+}
+
+func runSpeedtest(ctx context.Context, args []string) error {
+
+	if _, _, err := net.SplitHostPort(speedtestArgs.host); err != nil {
+		var addrErr *net.AddrError
+		if errors.As(err, &addrErr) && addrErr.Err == "missing port in address" {
+			// if no port is provided, append the default port
+			speedtestArgs.host = net.JoinHostPort(speedtestArgs.host, strconv.Itoa(speedtest.DefaultPort))
+		}
+	}
+
+	if speedtestArgs.runServer {
+		listener, err := net.Listen("tcp", speedtestArgs.host)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("listening on %v\n", listener.Addr())
+
+		return speedtest.Serve(listener)
+	}
+
+	// Ensure the duration is within the allowed range
+	if speedtestArgs.testDuration < speedtest.MinDuration || speedtestArgs.testDuration > speedtest.MaxDuration {
+		return fmt.Errorf("test duration must be within %v and %v", speedtest.MinDuration, speedtest.MaxDuration)
+	}
+
+	dir := speedtest.Download
+	if speedtestArgs.reverse {
+		dir = speedtest.Upload
+	}
+
+	fmt.Printf("Starting a %s test with %s\n", dir, speedtestArgs.host)
+	results, err := speedtest.RunClient(dir, speedtestArgs.testDuration, speedtestArgs.host)
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 12, 0, 0, ' ', tabwriter.TabIndent)
+	fmt.Println("Results:")
+	fmt.Fprintln(w, "Interval\t\tTransfer\t\tBandwidth\t\t")
+	for _, r := range results {
+		if r.Total {
+			fmt.Fprintln(w, "-------------------------------------------------------------------------")
+		}
+		fmt.Fprintf(w, "%.2f-%.2f\tsec\t%.4f\tMBits\t%.4f\tMbits/sec\t\n", r.IntervalStart.Seconds(), r.IntervalEnd.Seconds(), r.MegaBits(), r.MBitsPerSecond())
+	}
+	w.Flush()
+	return nil
+}

cmd/tailscale/cli/status.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/util/dnsname"
 )
 
@@ -193,7 +194,7 @@ func runStatus(ctx context.Context, args []string) error {
 //
 // TODO: have the server report this bool instead.
 func peerActive(ps *ipnstate.PeerStatus) bool {
-	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+	return !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 }
 
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {

cmd/tailscale/cli/up.go

@@ -51,7 +51,14 @@ flag is also used.
 	Exec:    runUp,
 }
 
-var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+func effectiveGOOS() string {
+	if v := os.Getenv("TS_DEBUG_UP_FLAG_GOOS"); v != "" {
+		return v
+	}
+	return runtime.GOOS
+}
+
+var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
@@ -63,13 +70,13 @@ func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
@@ -327,7 +334,7 @@ func runUp(ctx context.Context, args []string) error {
 		}
 	}
 
-	prefs, err := prefsFromUpArgs(upArgs, warnf, st, runtime.GOOS)
+	prefs, err := prefsFromUpArgs(upArgs, warnf, st, effectiveGOOS())
 	if err != nil {
 		fatalf("%s", err)
 	}
@@ -344,7 +351,7 @@ func runUp(ctx context.Context, args []string) error {
 	}
 
 	env := upCheckEnv{
-		goos:          runtime.GOOS,
+		goos:          effectiveGOOS(),
 		user:          os.Getenv("USER"),
 		flagSet:       upFlagSet,
 		upArgs:        upArgs,
@@ -384,7 +391,7 @@ func runUp(ctx context.Context, args []string) error {
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
-				switch runtime.GOOS {
+				switch effectiveGOOS() {
 				case "windows":
 					msg += " (Tailscale service in use by other user?)"
 				default:
@@ -458,7 +465,7 @@ func runUp(ctx context.Context, args []string) error {
 		// Windows service (~tailscaled) is the one that computes the
 		// StateKey based on the connection identity. So for now, just
 		// do as the Windows GUI's always done:
-		if runtime.GOOS == "windows" {
+		if effectiveGOOS() == "windows" {
 			// The Windows service will set this as needed based
 			// on our connection's identity.
 			opts.StateKey = ""

cmd/tailscale/depaware.txt

@@ -7,6 +7,12 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
         github.com/peterbourgon/ff/v2/ffcli                          from tailscale.com/cmd/tailscale/cli
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
         github.com/toqueteos/webbrowser                              from tailscale.com/cmd/tailscale/cli
      💣 go4.org/intern                                               from inet.af/netaddr
@@ -19,6 +25,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
+        tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
         tailscale.com/disco                                          from tailscale.com/derp
@@ -42,6 +49,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/derp+
@@ -77,7 +86,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/sync/errgroup                                   from tailscale.com/derp
+        golang.org/x/sync/errgroup                                   from tailscale.com/derp+
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from tailscale.com/net/netns+
@@ -127,7 +136,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
-        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli
+        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from github.com/peterbourgon/ff/v2+

cmd/tailscaled/debug.go

@@ -14,18 +14,23 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/url"
 	"os"
+	"strings"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/portmapper"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -33,6 +38,7 @@ var debugArgs struct {
 	monitor   bool
 	getURL    string
 	derpCheck string
+	portmap   bool
 }
 
 var debugModeFunc = debugMode // so it can be addressable
@@ -40,6 +46,7 @@ var debugModeFunc = debugMode // so it can be addressable
 func debugMode(args []string) error {
 	fs := flag.NewFlagSet("debug", flag.ExitOnError)
 	fs.BoolVar(&debugArgs.monitor, "monitor", false, "If true, run link monitor forever. Precludes all other options.")
+	fs.BoolVar(&debugArgs.portmap, "portmap", false, "If true, run portmap debugging. Precludes all other options.")
 	fs.StringVar(&debugArgs.getURL, "get-url", "", "If non-empty, fetch provided URL.")
 	fs.StringVar(&debugArgs.derpCheck, "derp", "", "if non-empty, test a DERP ping via named region code")
 	if err := fs.Parse(args); err != nil {
@@ -55,6 +62,9 @@ func debugMode(args []string) error {
 	if debugArgs.monitor {
 		return runMonitor(ctx)
 	}
+	if debugArgs.portmap {
+		return debugPortmap(ctx)
+	}
 	if debugArgs.getURL != "" {
 		return getURL(ctx, debugArgs.getURL)
 	}
@@ -191,3 +201,83 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 	log.Printf("ok")
 	return err
 }
+
+func debugPortmap(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	portmapper.VerboseLogs = true
+
+	done := make(chan bool, 1)
+
+	var c *portmapper.Client
+	logf := log.Printf
+	c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), func() {
+		logf("portmapping changed.")
+		logf("have mapping: %v", c.HaveMapping())
+
+		if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+			logf("cb: mapping: %v", ext)
+			select {
+			case done <- true:
+			default:
+			}
+			return
+		}
+		logf("cb: no mapping")
+	})
+	linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
+	if err != nil {
+		return err
+	}
+
+	gatewayAndSelfIP := func() (gw, self netaddr.IP, ok bool) {
+		if v := os.Getenv("TS_DEBUG_GW_SELF"); strings.Contains(v, "/") {
+			i := strings.Index(v, "/")
+			gw = netaddr.MustParseIP(v[:i])
+			self = netaddr.MustParseIP(v[i+1:])
+			return gw, self, true
+		}
+		return linkMon.GatewayAndSelfIP()
+	}
+
+	c.SetGatewayLookupFunc(gatewayAndSelfIP)
+
+	gw, selfIP, ok := gatewayAndSelfIP()
+	if !ok {
+		logf("no gateway or self IP; %v", linkMon.InterfaceState())
+		return nil
+	}
+	logf("gw=%v; self=%v", gw, selfIP)
+
+	res, err := c.Probe(ctx)
+	if err != nil {
+		return fmt.Errorf("Probe: %v", err)
+	}
+	logf("Probe: %+v", res)
+
+	if !res.PCP && !res.PMP && !res.UPnP {
+		logf("no portmapping services available")
+		return nil
+	}
+
+	uc, err := net.ListenPacket("udp", "0.0.0.0:0")
+	if err != nil {
+		return err
+	}
+	defer uc.Close()
+	c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
+
+	if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+		logf("mapping: %v", ext)
+	} else {
+		logf("no mapping")
+	}
+
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscaled/depaware.txt

@@ -23,6 +23,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
    W    github.com/pkg/errors                                        from github.com/tailscale/certstore
    W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/derp+
@@ -79,6 +85,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
+        tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck+
         tailscale.com/disco                                          from tailscale.com/derp+
@@ -123,6 +130,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
@@ -136,7 +145,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
-        tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
@@ -182,7 +191,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/sync/errgroup                                   from tailscale.com/derp
+        golang.org/x/sync/errgroup                                   from tailscale.com/derp+
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/mdlayher/netlink+
@@ -236,6 +245,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
+        encoding/xml                                                 from github.com/tailscale/goupnp+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from tailscale.com/cmd/tailscaled+

cmd/tailscaled/tailscaled.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/go-multierror/multierror"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
@@ -45,15 +46,6 @@ import (
 	"tailscale.com/wgengine/router"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
 // defaultTunName returns the default tun device name for the platform.
 func defaultTunName() string {
 	switch runtime.GOOS {
@@ -167,6 +159,28 @@ func main() {
 	}
 }
 
+func ipnServerOpts() (o ipnserver.Options) {
+	// Allow changing the OS-specific IPN behavior for tests
+	// so we can e.g. test Windows-specific behaviors on Linux.
+	goos := os.Getenv("TS_DEBUG_TAILSCALED_IPN_GOOS")
+	if goos == "" {
+		goos = runtime.GOOS
+	}
+
+	o.Port = 41112
+	o.StatePath = args.statepath
+	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+
+	switch goos {
+	default:
+		o.SurviveDisconnects = true
+		o.AutostartStateKey = ipn.GlobalDaemonStateKey
+	case "windows":
+		// Not those.
+	}
+	return o
+}
+
 func run() error {
 	var err error
 
@@ -196,6 +210,9 @@ func run() error {
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
+		if os.Getenv("TS_PLEASE_PANIC") != "" {
+			panic("TS_PLEASE_PANIC asked us to panic")
+		}
 		dns.Cleanup(logf, args.tunname)
 		router.Cleanup(logf, args.tunname)
 		return nil
@@ -271,14 +288,8 @@ func run() error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		SocketPath:         args.socketpath,
-		Port:               41112,
-		StatePath:          args.statepath,
-		AutostartStateKey:  globalStateKey,
-		SurviveDisconnects: runtime.GOOS != "windows",
-		DebugMux:           debugMux,
-	}
+	opts := ipnServerOpts()
+	opts.DebugMux = debugMux
 	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
@@ -341,7 +352,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			return nil, false, err
 		}
 		conf.Tun = dev
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
 			return nil, false, err

cmd/tailscaled/tailscaled_windows.go

@@ -168,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			return nil, fmt.Errorf("TUN: %w", err)
 		}
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, nil)
 		if err != nil {
 			dev.Close()
 			return nil, fmt.Errorf("router: %w", err)
@@ -233,12 +233,6 @@ func startIPNServer(ctx context.Context, logid string) error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		Port:               41112,
-		SurviveDisconnects: false,
-		StatePath:          args.statepath,
-	}
-
 	// getEngine is called by ipnserver to get the engine. It's
 	// not called concurrently and is not called again once it
 	// successfully returns an engine.
@@ -263,7 +257,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, opts)
+	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -29,8 +29,8 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/creack/pty"
 	"github.com/gliderlabs/ssh"
-	"github.com/kr/pty"
 	gossh "golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"

control/controlclient/debug.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"regexp"
 	"runtime"
 	"strconv"
 	"time"
@@ -42,28 +41,82 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 	}
 }
 
-var reHexArgs = regexp.MustCompile(`\b0x[0-9a-f]+\b`)
-
 // scrubbedGoroutineDump returns the list of all current goroutines, but with the actual
 // values of arguments scrubbed out, lest it contain some private key material.
 func scrubbedGoroutineDump() []byte {
-	buf := make([]byte, 1<<20)
-	buf = buf[:runtime.Stack(buf, true)]
+	var buf []byte
+	// Grab stacks multiple times into increasingly larger buffer sizes
+	// to minimize the risk that we blow past our iOS memory limit.
+	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			// It fit.
+			break
+		}
+	}
+	return scrubHex(buf)
+}
 
+func scrubHex(buf []byte) []byte {
 	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
-	return reHexArgs.ReplaceAllFunc(buf, func(in []byte) []byte {
+
+	foreachHexAddress(buf, func(in []byte) {
 		if string(in) == "0x0" {
-			return in
+			return
 		}
 		if v, ok := saw[string(in)]; ok {
-			return v
+			for i := range in {
+				in[i] = '_'
+			}
+			copy(in, v)
+			return
 		}
+		inStr := string(in)
 		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
+		for i := range in {
+			in[i] = '_'
+		}
 		if err != nil {
-			return []byte("??")
+			in[0] = '?'
+			return
 		}
 		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
-		saw[string(in)] = v
-		return v
+		saw[inStr] = v
+		copy(in, v)
 	})
+	return buf
+}
+
+var ohx = []byte("0x")
+
+// foreachHexAddress calls f with each subslice of b that matches
+// regexp `0x[0-9a-f]*`.
+func foreachHexAddress(b []byte, f func([]byte)) {
+	for len(b) > 0 {
+		i := bytes.Index(b, ohx)
+		if i == -1 {
+			return
+		}
+		b = b[i:]
+		hx := hexPrefix(b)
+		f(hx)
+		b = b[len(hx):]
+	}
+}
+
+func hexPrefix(b []byte) []byte {
+	for i, c := range b {
+		if i < 2 {
+			continue
+		}
+		if !isHexByte(c) {
+			return b[:i]
+		}
+	}
+	return b
+}
+
+func isHexByte(b byte) bool {
+	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
 }

control/controlclient/debug_test.go

@@ -9,3 +9,22 @@ import "testing"
 func TestScrubbedGoroutineDump(t *testing.T) {
 	t.Logf("Got:\n%s\n", scrubbedGoroutineDump())
 }
+
+func TestScrubHex(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"foo", "foo"},
+		{"", ""},
+		{"0x", "?_"},
+		{"0x001 and same 0x001", "v1%1_ and same v1%1_"},
+		{"0x008 and same 0x008", "v1%0_ and same v1%0_"},
+		{"0x001 and diff 0x002", "v1%1_ and diff v2%2_"},
+	}
+	for _, tt := range tests {
+		got := scrubHex([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Errorf("for input:\n%s\n\ngot:\n%s\n\nwant:\n%s\n", tt.in, got, tt.want)
+		}
+	}
+}

control/controlclient/direct.go

@@ -31,6 +31,7 @@ import (
 
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
+	"tailscale.com/control/controlknobs"
 	"tailscale.com/health"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
@@ -219,6 +220,13 @@ func packageType() string {
 		// Using tailscaled or IPNExtension?
 		exe, _ := os.Executable()
 		return filepath.Base(exe)
+	case "linux":
+		// Report whether this is in a snap.
+		// See https://snapcraft.io/docs/environment-variables
+		// We just look at two somewhat arbitrarily.
+		if os.Getenv("SNAP_NAME") != "" && os.Getenv("SNAP") != "" {
+			return "snap"
+		}
 	}
 	return ""
 }
@@ -797,7 +805,10 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			continue
 		}
 
-		if resp.Debug != nil {
+		hasDebug := resp.Debug != nil
+		// being conservative here, if Debug not present set to False
+		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
+		if hasDebug {
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}

control/controlknobs/controlknobs.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlknobs contains client options configurable from control which can be turned on
+// or off. The ability to turn options on and off is for incrementally adding features in.
+package controlknobs
+
+import (
+	"os"
+	"strconv"
+
+	"tailscale.com/syncs"
+)
+
+// disableUPnP indicates whether to attempt UPnP mapping.
+var disableUPnP syncs.AtomicBool
+
+func init() {
+	v, _ := strconv.ParseBool(os.Getenv("TS_DISABLE_UPNP"))
+	SetDisableUPnP(v)
+}
+
+// DisableUPnP reports the last reported value from control
+// whether UPnP portmapping should be disabled.
+func DisableUPnP() bool {
+	return disableUPnP.Get()
+}
+
+// SetDisableUPnP sets whether control says that UPnP should be
+// disabled.
+func SetDisableUPnP(v bool) {
+	disableUPnP.Set(v)
+}

derp/derp_server.go

@@ -36,6 +36,7 @@ import (
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
@@ -118,6 +119,8 @@ type Server struct {
 	curClients                   expvar.Int
 	curHomeClients               expvar.Int // ones with preferred
 	clientsReplaced              expvar.Int
+	clientsReplaceLimited        expvar.Int
+	clientsReplaceSleeping       expvar.Int
 	unknownFrames                expvar.Int
 	homeMovesIn                  expvar.Int // established clients announce home server moves in
 	homeMovesOut                 expvar.Int // established clients announce home server moves out
@@ -164,7 +167,7 @@ type PacketForwarder interface {
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
 // It is a defined type so that non-net connections can be used.
 type Conn interface {
-	io.Closer
+	io.WriteCloser
 
 	// The *Deadline methods follow the semantics of net.Conn.
 
@@ -346,14 +349,28 @@ func (s *Server) initMetacert() {
 func (s *Server) MetaCert() []byte { return s.metaCert }
 
 // registerClient notes that client c is now authenticated and ready for packets.
-// If c's public key was already connected with a different connection, the prior one is closed.
-func (s *Server) registerClient(c *sclient) {
+//
+// If c's public key was already connected with a different
+// connection, the prior one is closed, unless it's fighting rapidly
+// with another client with the same key, in which case the returned
+// ok is false, and the caller should wait the provided duration
+// before trying again.
+func (s *Server) registerClient(c *sclient) (ok bool, d time.Duration) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	old := s.clients[c.key]
 	if old == nil {
 		c.logf("adding connection")
 	} else {
+		// Take over the old rate limiter, discarding the one
+		// our caller just made.
+		c.replaceLimiter = old.replaceLimiter
+		if rr := c.replaceLimiter.ReserveN(timeNow(), 1); rr.OK() {
+			if d := rr.DelayFrom(timeNow()); d > 0 {
+				s.clientsReplaceLimited.Add(1)
+				return false, d
+			}
+		}
 		s.clientsReplaced.Add(1)
 		c.logf("adding connection, replacing %s", old.remoteAddr)
 		go old.nc.Close()
@@ -365,6 +382,7 @@ func (s *Server) registerClient(c *sclient) {
 	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
+	return true, 0
 }
 
 // broadcastPeerStateChangeLocked enqueues a message to all watchers
@@ -452,8 +470,9 @@ func (s *Server) addWatcher(c *sclient) {
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
-	br, bw := brw.Reader, brw.Writer
+	br := brw.Reader
 	nc.SetDeadline(time.Now().Add(10 * time.Second))
+	bw := &lazyBufioWriter{w: nc, lbw: brw.Writer}
 	if err := s.sendServerKey(bw); err != nil {
 		return fmt.Errorf("send server key: %v", err)
 	}
@@ -490,7 +509,14 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
 		peerGone:       make(chan key.Public),
 		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+
+		// Allow kicking out previous connections once a
+		// minute, with a very high burst of 100. Once a
+		// minute is less than the client's 2 minute
+		// inactivity timeout.
+		replaceLimiter: rate.NewLimiter(rate.Every(time.Minute), 100),
 	}
+
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
 	}
@@ -498,10 +524,18 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		c.info = *clientInfo
 	}
 
-	s.registerClient(c)
+	for {
+		ok, d := s.registerClient(c)
+		if ok {
+			break
+		}
+		s.clientsReplaceSleeping.Add(1)
+		timeSleep(d)
+		s.clientsReplaceSleeping.Add(-1)
+	}
 	defer s.unregisterClient(c)
 
-	err = s.sendServerInfo(bw, clientKey)
+	err = s.sendServerInfo(c.bw, clientKey)
 	if err != nil {
 		return fmt.Errorf("send server info: %v", err)
 	}
@@ -509,6 +543,12 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	return c.run(ctx)
 }
 
+// for testing
+var (
+	timeSleep = time.Sleep
+	timeNow   = time.Now
+)
+
 // run serves the client until there's an error.
 // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error.
 func (c *sclient) run(ctx context.Context) error {
@@ -698,7 +738,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 // dropReason is why we dropped a DERP frame.
 type dropReason int
 
-//go:generate stringer -type=dropReason -trimprefix=dropReason
+//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason
 
 const (
 	dropReasonUnknownDest      dropReason = iota // unknown destination pubkey
@@ -807,18 +847,20 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	return nil
 }
 
-func (s *Server) sendServerKey(bw *bufio.Writer) error {
+func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
 	buf := make([]byte, 0, len(magic)+len(s.publicKey))
 	buf = append(buf, magic...)
 	buf = append(buf, s.publicKey[:]...)
-	return writeFrame(bw, frameServerKey, buf)
+	err := writeFrame(lw.bw(), frameServerKey, buf)
+	lw.Flush() // redundant (no-op) flush to release bufio.Writer
+	return err
 }
 
 type serverInfo struct {
 	Version int `json:"version,omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
+func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
 	var nonce [24]byte
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
@@ -829,7 +871,7 @@ func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 	}
 
 	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw, frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
+	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(nonce[:]); err != nil {
@@ -952,13 +994,18 @@ type sclient struct {
 	meshUpdate     chan struct{}   // write request to write peerStateChange
 	canMesh        bool            // clientInfo had correct mesh token for inter-region routing
 
+	// replaceLimiter controls how quickly two connections with
+	// the same client key can kick each other off the server by
+	// taking over ownership of a key.
+	replaceLimiter *rate.Limiter
+
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
 	connectedAt time.Time
 	preferred   bool
 
 	// Owned by sender, not thread-safe.
-	bw *bufio.Writer
+	bw *lazyBufioWriter
 
 	// Guarded by s.mu
 	//
@@ -1120,14 +1167,14 @@ func (c *sclient) setWriteDeadline() {
 // sendKeepAlive sends a keep-alive frame, without flushing.
 func (c *sclient) sendKeepAlive() error {
 	c.setWriteDeadline()
-	return writeFrameHeader(c.bw, frameKeepAlive, 0)
+	return writeFrameHeader(c.bw.bw(), frameKeepAlive, 0)
 }
 
 // sendPeerGone sends a peerGone frame, without flushing.
 func (c *sclient) sendPeerGone(peer key.Public) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerGone, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1137,7 +1184,7 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 // sendPeerPresent sends a peerPresent frame, without flushing.
 func (c *sclient) sendPeerPresent(peer key.Public) error {
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1210,11 +1257,11 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	if withKey {
 		pktLen += len(srcKey)
 	}
-	if err = writeFrameHeader(c.bw, frameRecvPacket, uint32(pktLen)); err != nil {
+	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw, &srcKey)
+		err := writePublicKey(c.bw.bw(), &srcKey)
 		if err != nil {
 			return err
 		}
@@ -1351,6 +1398,8 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
+	m.Set("clients_replace_limited", &s.clientsReplaceLimited)
+	m.Set("gauge_clients_replace_sleeping", &s.clientsReplaceSleeping)
 	m.Set("bytes_received", &s.bytesRecv)
 	m.Set("bytes_sent", &s.bytesSent)
 	m.Set("packets_dropped", &s.packetsDropped)
@@ -1527,3 +1576,45 @@ func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(minTimeBetweenLogs)
 	}
 }
+
+var bufioWriterPool = &sync.Pool{
+	New: func() interface{} {
+		return bufio.NewWriterSize(ioutil.Discard, 2<<10)
+	},
+}
+
+// lazyBufioWriter is a bufio.Writer-like wrapping writer that lazily
+// allocates its actual bufio.Writer from a sync.Pool, releasing it to
+// the pool upon flush.
+//
+// We do this to reduce memory overhead; most DERP connections are
+// idle and the idle bufio.Writers were 30% of overall memory usage.
+type lazyBufioWriter struct {
+	w   io.Writer     // underlying
+	lbw *bufio.Writer // lazy; nil means it needs an associated buffer
+}
+
+func (w *lazyBufioWriter) bw() *bufio.Writer {
+	if w.lbw == nil {
+		w.lbw = bufioWriterPool.Get().(*bufio.Writer)
+		w.lbw.Reset(w.w)
+	}
+	return w.lbw
+}
+
+func (w *lazyBufioWriter) Available() int { return w.bw().Available() }
+
+func (w *lazyBufioWriter) Write(p []byte) (int, error) { return w.bw().Write(p) }
+
+func (w *lazyBufioWriter) Flush() error {
+	if w.lbw == nil {
+		return nil
+	}
+	err := w.lbw.Flush()
+
+	w.lbw.Reset(ioutil.Discard)
+	bufioWriterPool.Put(w.lbw)
+	w.lbw = nil
+
+	return err
+}

derp/derp_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -849,6 +850,136 @@ func TestClientSendPong(t *testing.T) {
 
 }
 
+func TestServerReplaceClients(t *testing.T) {
+	defer func() {
+		timeSleep = time.Sleep
+		timeNow = time.Now
+	}()
+
+	var (
+		mu     sync.Mutex
+		now    = time.Unix(123, 0)
+		sleeps int
+		slept  time.Duration
+	)
+	timeSleep = func(d time.Duration) {
+		mu.Lock()
+		defer mu.Unlock()
+		sleeps++
+		slept += d
+		now = now.Add(d)
+	}
+	timeNow = func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}
+
+	serverPrivateKey := newPrivateKey(t)
+	var logger logger.Logf = logger.Discard
+	const debug = false
+	if debug {
+		logger = t.Logf
+	}
+
+	s := NewServer(serverPrivateKey, logger)
+	defer s.Close()
+
+	priv := newPrivateKey(t)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	connNum := 0
+	connect := func() *Client {
+		connNum++
+		cout, err := net.Dial("tcp", ln.Addr().String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cin, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		brwServer := bufio.NewReadWriter(bufio.NewReader(cin), bufio.NewWriter(cin))
+		go s.Accept(cin, brwServer, fmt.Sprintf("test-client-%d", connNum))
+
+		brw := bufio.NewReadWriter(bufio.NewReader(cout), bufio.NewWriter(cout))
+		c, err := NewClient(priv, cout, brw, logger)
+		if err != nil {
+			t.Fatalf("client %d: %v", connNum, err)
+		}
+		return c
+	}
+
+	wantVar := func(v *expvar.Int, want int64) {
+		t.Helper()
+		if got := v.Value(); got != want {
+			t.Errorf("got %d; want %d", got, want)
+		}
+	}
+
+	wantClosed := func(c *Client) {
+		t.Helper()
+		for {
+			m, err := c.Recv()
+			if err != nil {
+				t.Logf("got expected error: %v", err)
+				return
+			}
+			switch m.(type) {
+			case ServerInfoMessage:
+				continue
+			default:
+				t.Fatalf("client got %T; wanted an error", m)
+			}
+		}
+	}
+
+	c1 := connect()
+	waitConnect(t, c1)
+	c2 := connect()
+	waitConnect(t, c2)
+	wantVar(&s.clientsReplaced, 1)
+	wantClosed(c1)
+
+	for i := 0; i < 100+5; i++ {
+		c := connect()
+		defer c.nc.Close()
+		if s.clientsReplaceLimited.Value() == 0 && i < 90 {
+			continue
+		}
+		t.Logf("for %d: replaced=%d, limited=%d, sleeping=%d", i,
+			s.clientsReplaced.Value(),
+			s.clientsReplaceLimited.Value(),
+			s.clientsReplaceSleeping.Value(),
+		)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if sleeps == 0 {
+		t.Errorf("no sleeps")
+	}
+	if slept == 0 {
+		t.Errorf("total sleep duration was 0")
+	}
+}
+
+func TestLimiter(t *testing.T) {
+	rl := rate.NewLimiter(rate.Every(time.Minute), 100)
+	for i := 0; i < 200; i++ {
+		r := rl.Reserve()
+		d := r.Delay()
+		t.Logf("i=%d, allow=%v, d=%v", i, r.OK(), d)
+	}
+}
+
 func BenchmarkSendRecv(b *testing.B) {
 	for _, size := range []int{10, 100, 1000, 10000} {
 		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })

derp/derphttp/derphttp_client.go

@@ -54,6 +54,7 @@ type Client struct {
 
 	privateKey key.Private
 	logf       logger.Logf
+	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
 	// Either url or getRegion is non-nil:
 	url       *url.URL
@@ -363,14 +364,26 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	return c.client, c.connGen, nil
 }
 
+// SetURLDialer sets the dialer to use for dialing URLs.
+// This dialer is only use for clients created with NewClient, not NewRegionClient.
+// If unset or nil, the default dialer is used.
+//
+// The primary use for this is the derper mesh mode to connect to each
+// other over a VPC network.
+func (c *Client) SetURLDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
+	c.dialer = dialer
+}
+
 func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 	host := c.url.Hostname()
+	if c.dialer != nil {
+		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
+	}
 	hostOrIP := host
-
 	dialer := netns.NewDialer()
 
 	if c.DNSCache != nil {
-		ip, _, err := c.DNSCache.LookupIP(ctx, host)
+		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
 		if err == nil {
 			hostOrIP = ip.String()
 		}

derp/dropreason_string.go

@@ -1,3 +1,7 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 // Code generated by "stringer -type=dropReason -trimprefix=dropReason"; DO NOT EDIT.
 
 package derp

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
+	github.com/creack/pty v1.1.9
+	github.com/dave/jennifer v1.4.1
 	github.com/frankban/quicktest v1.13.0
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
@@ -16,10 +18,10 @@ require (
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
+	github.com/iancoleman/strcase v0.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
-	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
@@ -29,6 +31,8 @@ require (
 	github.com/pkg/sftp v1.13.0
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
+	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
+	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
@@ -42,7 +46,7 @@ require (
 	golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0
 	golang.zx2c4.com/wireguard/windows v0.3.16
 	honnef.co/go/tools v0.1.4
-	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
+	inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1
 	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756

go.sum

@@ -82,12 +82,13 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/daixiang0/gci v0.2.7 h1:bosLNficubzJZICsVzxuyNc6oAbdz0zcqLG2G/RxtY4=
 github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
+github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
+github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -296,6 +297,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -355,8 +358,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -580,6 +581,10 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -950,8 +955,8 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ const (
 	AWSLambda       = EnvType("lm")
 	Heroku          = EnvType("hr")
 	AzureAppService = EnvType("az")
+	AWSFargate      = EnvType("fg")
 )
 
 var envType atomic.Value // of EnvType
@@ -53,6 +54,9 @@ func getEnvType() EnvType {
 	if inAzureAppService() {
 		return AzureAppService
 	}
+	if inAWSFargate() {
+		return AWSFargate
+	}
 	return ""
 }
 
@@ -115,3 +119,10 @@ func inAzureAppService() bool {
 	}
 	return false
 }
+
+func inAWSFargate() bool {
+	if os.Getenv("AWS_EXECUTION_ENV") == "AWS_ECS_FARGATE" {
+		return true
+	}
+	return false
+}

ipn/ipnlocal/local.go

@@ -95,7 +95,7 @@ type LocalBackend struct {
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
 
-	filterHash string
+	filterHash deephash.Sum
 
 	// The mutex protects the following elements.
 	mu             sync.Mutex
@@ -179,6 +179,7 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		gotPortPollRes: make(chan struct{}),
 	}
 	b.statusChanged = sync.NewCond(&b.statusLock)
+	b.e.SetStatusCallback(b.setWgengineStatus)
 
 	linkMon := e.GetLinkMonitor()
 	b.prevIfState = linkMon.InterfaceState()
@@ -214,6 +215,15 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
+// b.mu must be held.
+func (b *LocalBackend) maybePauseControlClientLocked() {
+	if b.cc == nil {
+		return
+	}
+	networkUp := b.prevIfState.AnyInterfaceUp()
+	b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
+}
+
 // linkChange is our link monitor callback, called whenever the network changes.
 // major is whether ifst is different than earlier.
 func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
@@ -222,11 +232,7 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	hadPAC := b.prevIfState.HasPAC()
 	b.prevIfState = ifst
-
-	networkUp := ifst.AnyInterfaceUp()
-	if b.cc != nil {
-		go b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
-	}
+	b.maybePauseControlClientLocked()
 
 	// If the PAC-ness of the network changed, reconfig wireguard+route to
 	// add/remove subnets.
@@ -605,8 +611,8 @@ func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 
 	if cc != nil {
 		cc.UpdateEndpoints(0, s.LocalAddrs)
+		b.stateMachine()
 	}
-	b.stateMachine()
 
 	b.statusLock.Lock()
 	b.statusChanged.Broadcast()
@@ -863,7 +869,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	cc.SetStatusFunc(b.setClientStatus)
-	b.e.SetStatusCallback(b.setWgengineStatus)
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
 	b.mu.Lock()
@@ -939,7 +944,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets, _ := localNetsB.IPSet()
 	logNets, _ := logNetsB.IPSet()
 
-	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.Update(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -979,6 +984,44 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 	tsaddr.TailscaleULARange(),
 }
 
+// internalAndExternalInterfaces splits interface routes into "internal"
+// and "external" sets. Internal routes are those of virtual ethernet
+// network interfaces used by guest VMs and containers, such as WSL and
+// Docker.
+//
+// Given that "internal" routes don't leave the device, we choose to
+// trust them more, allowing access to them when an Exit Node is enabled.
+func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
+	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
+			return
+		}
+		if pfx.IsSingleIP() {
+			return
+		}
+		if runtime.GOOS == "windows" {
+			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
+			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
+			//
+			// This includes WSL2 vEthernet.
+			// Importantly: by default WSL2 /etc/resolv.conf points to
+			// a stub resolver running on the host vEthernet IP.
+			// So enabling exit nodes with the default tailnet
+			// configuration breaks WSL2 DNS without this.
+			mac := iface.Interface.HardwareAddr
+			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
+				internal = append(internal, pfx)
+				return
+			}
+		}
+		external = append(external, pfx)
+	}); err != nil {
+		return nil, nil, err
+	}
+
+	return internal, external, nil
+}
+
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
@@ -2114,18 +2157,21 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
+		internalIPs, externalIPs, err := internalAndExternalInterfaces()
+		if err != nil {
+			b.logf("failed to discover interface ips: %v", err)
+		}
 		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
-			// Only allow local lan access on linux machines for now.
-			ips, _, err := interfaceRoutes()
-			if err != nil {
-				b.logf("failed to discover interface ips: %v", err)
-			}
+			rs.LocalRoutes = internalIPs // unconditionally allow access to guest VM networks
 			if prefs.ExitNodeAllowLANAccess {
-				rs.LocalRoutes = ips.Prefixes()
+				rs.LocalRoutes = append(rs.LocalRoutes, externalIPs...)
+				if len(externalIPs) != 0 {
+					b.logf("allowing exit node access to internal IPs: %v", internalIPs)
+				}
 			} else {
 				// Explicitly add routes to the local network so that we do not
 				// leak any traffic.
-				rs.Routes = append(rs.Routes, ips.Prefixes()...)
+				rs.Routes = append(rs.Routes, externalIPs...)
 			}
 		}
 	}
@@ -2154,6 +2200,18 @@ func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *ipn.Prefs) {
 	}
 	if v := prefs.OSVersion; v != "" {
 		hi.OSVersion = v
+
+		// The Android app annotates when Google Play Services
+		// aren't available by tacking on a string to the
+		// OSVersion. Promote that to the Hostinfo.Package
+		// field instead, rather than adding a new pref, as
+		// this applyPrefsToHostinfo mechanism is mostly
+		// abused currently. TODO(bradfitz): instead let
+		// frontends update Hostinfo, without using Prefs.
+		if runtime.GOOS == "android" && strings.HasSuffix(v, " [nogoogle]") {
+			hi.Package = "nogoogle"
+			hi.OSVersion = strings.TrimSuffix(v, " [nogoogle]")
+		}
 	}
 	if m := prefs.DeviceModel; m != "" {
 		hi.DeviceModel = m
@@ -2173,9 +2231,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	oldState := b.state
 	b.state = newState
 	prefs := b.prefs
-	cc := b.cc
 	netMap := b.netMap
-	networkUp := b.prevIfState.AnyInterfaceUp()
 	activeLogin := b.activeLogin
 	authURL := b.authURL
 	if newState == ipn.Running {
@@ -2185,6 +2241,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		// Transitioning away from running.
 		b.closePeerAPIListenersLocked()
 	}
+	b.maybePauseControlClientLocked()
 	b.mu.Unlock()
 
 	if oldState == newState {
@@ -2195,10 +2252,6 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
-	if cc != nil {
-		cc.SetPaused((newState == ipn.Stopped && netMap != nil) || !networkUp)
-	}
-
 	switch newState {
 	case ipn.NeedsLogin:
 		systemd.Status("Needs login: %s", authURL)
@@ -2459,6 +2512,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		b.logf("active login: %v", login)
 		b.activeLogin = login
 	}
+	b.maybePauseControlClientLocked()
 
 	// Determine if file sharing is enabled
 	fs := hasCapability(nm, tailcfg.CapabilityFileSharing)
@@ -2730,17 +2784,18 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 
+	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
 		}
 		if !on {
-			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
+			return fmt.Errorf("%s is disabled.%s", key, suffix)
 		}
 	}
 	return nil

ipn/ipnlocal/state_test.go

@@ -354,7 +354,7 @@ func TestStateMachine(t *testing.T) {
 	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -389,7 +389,7 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause"})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
@@ -414,7 +414,7 @@ func TestStateMachine(t *testing.T) {
 		// We're still not logged in so there's nothing we can do
 		// with it. (And empirically, it's providing an empty list
 		// of endpoints.)
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
 		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
 	}
@@ -440,7 +440,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, url2, false, nil)
 	{
 		// BUG: UpdateEndpoints again, this is getting silly.
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		// This time, backend should emit it to the UI right away,
 		// because the UI is anxiously awaiting a new URL to visit.
@@ -470,7 +470,7 @@ func TestStateMachine(t *testing.T) {
 		// wait until it gets into Starting.
 		// TODO: (Currently this test doesn't detect that bug, but
 		// it's visible in the logs)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -483,7 +483,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(1)
 	// BUG: the real controlclient sends LoginFinished with every
 	// notification while it's in StateAuthenticated, but not StateSynced.
-	// We should send it exactly once, or every time we're authenticated,
+	// It should send it exactly once, or every time we're authenticated,
 	// but the current code is brittle.
 	// (ie. I suspect it would be better to change false->true in send()
 	// below, and do the same in the real controlclient.)
@@ -492,7 +492,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
@@ -534,7 +534,7 @@ func TestStateMachine(t *testing.T) {
 		nn := notifies.drain(2)
 		// BUG: UpdateEndpoints isn't needed here.
 		// BUG: Login isn't needed here. We never logged out.
-		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -543,7 +543,8 @@ func TestStateMachine(t *testing.T) {
 	}
 
 	// Test the fast-path frontend reconnection.
-	// This one is very finicky, so we have to force State==Running.
+	// This one is very finicky, so we have to force State==Running
+	// or it won't use the fast path.
 	// TODO: actually get to State==Running, rather than cheating.
 	//  That'll require spinning up a fake DERP server and putting it in
 	//  the netmap.
@@ -570,7 +571,7 @@ func TestStateMachine(t *testing.T) {
 	b.Logout()
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"pause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause", "StartLogout", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
@@ -587,7 +588,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
@@ -603,7 +604,7 @@ func TestStateMachine(t *testing.T) {
 		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
-		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"StartLogout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -617,8 +618,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -632,7 +632,7 @@ func TestStateMachine(t *testing.T) {
 	// I guess, since that's supposed to be synchronous.
 	{
 		notifies.drain(0)
-		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Logout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -646,8 +646,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -678,7 +677,7 @@ func TestStateMachine(t *testing.T) {
 		// BUG: We already called Shutdown(), no need to do it again.
 		// BUG: Way too soon for UpdateEndpoints.
 		// BUG: don't unpause because we're not logged in.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -703,7 +702,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -743,9 +742,8 @@ func TestStateMachine(t *testing.T) {
 		//  on startup, otherwise UIs can't show the node list, login
 		//  name, etc when in state ipn.Stopped.
 		//  Arguably they shouldn't try. But they currently do.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
-
 		nn := notifies.drain(2)
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
@@ -754,6 +752,25 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
 	}
 
+	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
+	// the first netmap. Simulate that netmap being received, after which
+	// it should pause us, to avoid wasting CPU retrieving unnecessarily
+	// additional netmap updates.
+	//
+	// TODO: really the various GUIs and prefs should be refactored to
+	//  not require the netmap structure at all when starting while
+	//  !WantRunning. That would remove the need for this (or contacting
+	//  the control server at all when stopped).
+	t.Logf("\n\nStart4 -> netmap")
+	notifies.expect(0)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		notifies.drain(0)
+		c.Assert([]string{"pause", "pause"}, qt.DeepEquals, cc.getCalls())
+	}
+
 	// Request connection.
 	// The state machine didn't call Login() earlier, so now it needs to.
 	t.Logf("\n\nWantRunning4 -> true")
@@ -780,7 +797,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -790,25 +807,27 @@ func TestStateMachine(t *testing.T) {
 	// We want to try logging in as a different user, while Stopped.
 	// First, start the login process (without logging out first).
 	t.Logf("\n\nLoginDifferent")
-	notifies.expect(2)
+	notifies.expect(1)
 	b.StartLoginInteractive()
 	url3 := "http://localhost:1/3"
 	cc.send(nil, url3, false, nil)
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(1)
 		// It might seem like WantRunning should switch to true here,
 		// but that would be risky since we already have a valid
 		// user account. It might try to reconnect to the old account
 		// before the new one is ready. So no change yet.
-		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		//
+		// Because the login hasn't yet completed, the old login
+		// is still valid, so it's correct that we stay paused.
+		c.Assert([]string{"Login", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
-		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 	}
 
-	// Now, let's say the interactive login completed, using a different
-	// user account than before.
+	// Now, let's complete the interactive login, using a different
+	// user account than before. WantRunning changes to true after an
+	// interactive login, so we end up unpaused.
 	t.Logf("\n\nLoginDifferent URL visited")
 	notifies.expect(3)
 	cc.persist.LoginName = "user3"
@@ -817,7 +836,13 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: pause() being called here is a bad sign.
+		//  It means that either the state machine ran at least once
+		//  with the old netmap, or it ran with the new login+netmap
+		//  and !WantRunning. But since it's a fresh and successful
+		//  new login, WantRunning is true, so there was never a
+		//  reason to pause().
+		c.Assert([]string{"pause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -836,7 +861,7 @@ func TestStateMachine(t *testing.T) {
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() ourselves.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(1)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -855,7 +880,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))

ipn/ipnstate/ipnstate.go

@@ -20,6 +20,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/util/dnsname"
 )
@@ -90,7 +91,7 @@ type PeerStatus struct {
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
+	LastWrite     mono.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -320,7 +321,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	f("<tr><th>Peer</th><th>OS</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Connection</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
-	now := time.Now()
+	now := mono.Now()
 
 	var peers []*PeerStatus
 	for _, peer := range st.Peers() {
@@ -378,7 +379,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 		f("<td>")
 
 		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		active := !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 		if active {
 			if ps.Relay != "" && ps.CurAddr == "" {
 				f("relay <b>%s</b>", html.EscapeString(ps.Relay))

ipn/localapi/localapi.go

@@ -266,7 +266,7 @@ func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "file access denied", http.StatusForbidden)
 		return
 	}
-	suffix := strings.TrimPrefix(r.URL.Path, "/localapi/v0/files/")
+	suffix := strings.TrimPrefix(r.URL.EscapedPath(), "/localapi/v0/files/")
 	if suffix == "" {
 		if r.Method != "GET" {
 			http.Error(w, "want GET to list files", 400)

logpolicy/logpolicy.go

@@ -128,6 +128,13 @@ func (l logWriter) Write(buf []byte) (int, error) {
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
 func logsDir(logf logger.Logf) string {
+	if d := os.Getenv("TS_LOGS_DIR"); d != "" {
+		fi, err := os.Stat(d)
+		if err == nil && fi.IsDir() {
+			return d
+		}
+	}
+
 	// STATE_DIRECTORY is set by systemd 240+ but we support older
 	// systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237.
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")
@@ -180,6 +187,10 @@ func runningUnderSystemd() bool {
 	return false
 }
 
+func redirectStderrToLogPanics() bool {
+	return runningUnderSystemd() || os.Getenv("TS_PLEASE_PANIC") != ""
+}
+
 // tryFixLogStateLocation is a temporary fixup for
 // https://github.com/tailscale/tailscale/issues/247 . We accidentally
 // wrote logging state files to /, and then later to $CACHE_DIRECTORY
@@ -428,9 +439,14 @@ func New(collection string) *Policy {
 		c.HTTPC = &http.Client{Transport: newLogtailTransport(u.Host)}
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{
+		ReplaceStderr: redirectStderrToLogPanics(),
+	})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
+		if filchBuf.OrigStderr != nil {
+			c.Stderr = filchBuf.OrigStderr
+		}
 	}
 	lw := logtail.NewLogger(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here

logtail/logtail.go

@@ -118,9 +118,10 @@ type Logger struct {
 	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
+	explainedRaw   bool
 
 	shutdownStart chan struct{} // closed when shutdown begins
-	shutdownDone  chan struct{} // closd when shutdown complete
+	shutdownDone  chan struct{} // closed when shutdown complete
 }
 
 // SetVerbosityLevel controls the verbosity level that should be
@@ -230,6 +231,14 @@ func (l *Logger) drainPending() (res []byte) {
 			// outside of the logtail logger. Encode it.
 			// Do not add a client time, as it could have been
 			// been written a long time ago.
+			if !l.explainedRaw {
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: *** Lines prefixed with RAW-STDERR below bypassed logtail and probably come from a previous run of the program\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR:\n")
+				l.explainedRaw = true
+			}
+			fmt.Fprintf(l.stderr, "RAW-STDERR: %s", b)
 			b = l.encodeText(b, true)
 		}
 

net/dns/config.go

@@ -5,9 +5,12 @@
 package dns
 
 import (
+	"bufio"
+	"fmt"
 	"sort"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/util/dnsname"
 )
 
@@ -38,6 +41,20 @@ type Config struct {
 	Hosts map[dnsname.FQDN][]netaddr.IP
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{DefaultResolvers:")
+	resolver.WriteIPPorts(w, c.DefaultResolvers)
+
+	w.WriteString(" Routes:")
+	resolver.WriteRoutes(w, c.Routes)
+
+	fmt.Fprintf(w, " SearchDomains:%v", c.SearchDomains)
+	fmt.Fprintf(w, " Hosts:%v", len(c.Hosts))
+	w.WriteString("}")
+}
+
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {

net/dns/ini.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build windows
+
 package dns
 
 import (

net/dns/ini_test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build windows
+
 package dns
 
 import (

net/dns/manager.go

@@ -5,6 +5,7 @@
 package dns
 
 import (
+	"bufio"
 	"runtime"
 	"time"
 
@@ -50,14 +51,18 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, li
 }
 
 func (m *Manager) Set(cfg Config) error {
-	m.logf("Set: %+v", cfg)
+	m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		cfg.WriteToBufioWriter(w)
+	}))
 
 	rcfg, ocfg, err := m.compileConfig(cfg)
 	if err != nil {
 		return err
 	}
 
-	m.logf("Resolvercfg: %+v", rcfg)
+	m.logf("Resolvercfg: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		rcfg.WriteToBufioWriter(w)
+	}))
 	m.logf("OScfg: %+v", ocfg)
 
 	if err := m.resolver.SetConfig(rcfg); err != nil {

net/dns/manager_windows.go

@@ -293,7 +293,7 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 
 		t0 = time.Now()
-		m.logf("running ipconfig /registerdns ...")
+		m.logf("running ipconfig /flushdns ...")
 		cmd = exec.Command("ipconfig", "/flushdns")
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
 		err = cmd.Run()

net/dns/resolver/doh_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"testing"
+
+	"golang.org/x/net/dns/dnsmessage"
+)
+
+var testDoH = flag.Bool("test-doh", false, "do real DoH tests against the network")
+
+const someDNSID = 123 // something non-zero as a test; in violation of spec's SHOULD of 0
+
+func someDNSQuestion(t testing.TB) []byte {
+	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{
+		OpCode:           0, // query
+		RecursionDesired: true,
+		ID:               someDNSID,
+	})
+	b.StartQuestions() // err
+	b.Question(dnsmessage.Question{
+		Name:  dnsmessage.MustNewName("tailscale.com."),
+		Type:  dnsmessage.TypeA,
+		Class: dnsmessage.ClassINET,
+	})
+	msg, err := b.Finish()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return msg
+}
+
+func TestDoH(t *testing.T) {
+	if !*testDoH {
+		t.Skip("skipping manual test without --test-doh flag")
+	}
+	if len(knownDoH) == 0 {
+		t.Fatal("no known DoH")
+	}
+
+	f := &forwarder{
+		dohSem: make(chan struct{}, 10),
+	}
+
+	for ip := range knownDoH {
+		t.Run(ip.String(), func(t *testing.T) {
+			urlBase, c, ok := f.getDoHClient(ip)
+			if !ok {
+				t.Fatal("expected DoH")
+			}
+			res, err := f.sendDoH(context.Background(), urlBase, c, someDNSQuestion(t))
+			if err != nil {
+				t.Fatal(err)
+			}
+			c.Transport.(*http.Transport).CloseIdleConnections()
+
+			var p dnsmessage.Parser
+			h, err := p.Start(res)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if h.ID != someDNSID {
+				t.Errorf("response DNS ID = %v; want %v", h.ID, someDNSID)
+			}
+
+			p.SkipAllQuestions()
+			aa, err := p.AllAnswers()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(aa) == 0 {
+				t.Fatal("no answers")
+			}
+			for _, r := range aa {
+				t.Logf("got: %v", r.GoString())
+			}
+		})
+	}
+}
+
+func TestDoHV6Fallback(t *testing.T) {
+	for ip, base := range knownDoH {
+		if ip.Is4() {
+			ip6, ok := dohV6(base)
+			if !ok {
+				t.Errorf("no v6 DoH known for %v", ip)
+			} else if !ip6.Is6() {
+				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+			}
+		}
+	}
+}

net/dns/resolver/forwarder.go

@@ -9,15 +9,22 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net"
+	"net/http"
+	"runtime"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
+	"tailscale.com/net/netns"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/monitor"
@@ -29,6 +36,16 @@ const headerBytes = 12
 const (
 	// responseTimeout is the maximal amount of time to wait for a DNS response.
 	responseTimeout = 5 * time.Second
+
+	// dohTransportTimeout is how long to keep idle HTTP
+	// connections open to DNS-over-HTTPs servers. This is pretty
+	// arbitrary.
+	dohTransportTimeout = 30 * time.Second
+
+	// wellKnownHostBackupDelay is how long to artificially delay upstream
+	// DNS queries to the "fallback" DNS server IP for a known provider
+	// (e.g. how long to wait to query Google's 8.8.4.4 after 8.8.8.8).
+	wellKnownHostBackupDelay = 200 * time.Millisecond
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
@@ -108,6 +125,7 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		return
 	}
 
+	// https://datatracker.ietf.org/doc/html/rfc6891#section-6.1.2
 	opt := packet[len(packet)-optFixedBytes:]
 
 	if opt[0] != 0 {
@@ -124,8 +142,8 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		// Be conservative and don't touch unknown versions.
 		return
 	}
-	// Ignore flags in opt[7:9]
-	if binary.BigEndian.Uint16(opt[10:12]) != 0 {
+	// Ignore flags in opt[6:9]
+	if binary.BigEndian.Uint16(opt[9:11]) != 0 {
 		// RDLEN must be 0 (no variable length data). We're at the end of the
 		// packet so this should be 0 anyway)..
 		return
@@ -141,7 +159,20 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 
 type route struct {
 	Suffix    dnsname.FQDN
-	Resolvers []netaddr.IPPort
+	Resolvers []resolverAndDelay
+}
+
+// resolverAndDelay is an upstream DNS resolver and a delay for how
+// long to wait before querying it.
+type resolverAndDelay struct {
+	// ipp is the upstream resolver.
+	ipp netaddr.IPPort
+
+	// startDelay is an amount to delay this resolver at
+	// start. It's used when, say, there are four Google or
+	// Cloudflare DNS IPs (two IPv4 + two IPv6) and we don't want
+	// to race all four at once.
+	startDelay time.Duration
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
@@ -149,6 +180,7 @@ type forwarder struct {
 	logf    logger.Logf
 	linkMon *monitor.Mon
 	linkSel ForwardLinkSelector
+	dohSem  chan struct{}
 
 	ctx       context.Context    // good until Close
 	ctxCancel context.CancelFunc // closes ctx
@@ -158,6 +190,8 @@ type forwarder struct {
 
 	mu sync.Mutex // guards following
 
+	dohClient map[netaddr.IP]*http.Client
+
 	// routes are per-suffix resolvers to use, with
 	// the most specific routes first.
 	routes []route
@@ -168,11 +202,18 @@ func init() {
 }
 
 func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+	maxDoHInFlight := 1000 // effectively unlimited
+	if runtime.GOOS == "ios" {
+		// No HTTP/2 on iOS yet (for size reasons), so DoH is
+		// pricier.
+		maxDoHInFlight = 10
+	}
 	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		linkMon:   linkMon,
 		linkSel:   linkSel,
 		responses: responses,
+		dohSem:    make(chan struct{}, maxDoHInFlight),
 	}
 	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
 	return f
@@ -183,7 +224,84 @@ func (f *forwarder) Close() error {
 	return nil
 }
 
-func (f *forwarder) setRoutes(routes []route) {
+// resolversWithDelays maps from a set of DNS server ip:ports (currently
+// the port is always 53) to a slice of a type that included a
+// startDelay. So if ipps contains e.g. four Google DNS IPs (two IPv4
+// + twoIPv6), this function partition adds delays to some.
+func resolversWithDelays(ipps []netaddr.IPPort) []resolverAndDelay {
+	type hostAndFam struct {
+		host string // some arbitrary string representing DNS host (currently the DoH base)
+		bits uint8  // either 32 or 128 for IPv4 vs IPv6s address family
+	}
+
+	// Track how many of each known resolver host are in the list,
+	// per address family.
+	total := map[hostAndFam]int{}
+
+	rr := make([]resolverAndDelay, len(ipps))
+	for _, ipp := range ipps {
+		ip := ipp.IP()
+		if host, ok := knownDoH[ip]; ok {
+			total[hostAndFam{host, ip.BitLen()}]++
+		}
+	}
+
+	done := map[hostAndFam]int{}
+	for i, ipp := range ipps {
+		ip := ipp.IP()
+		var startDelay time.Duration
+		if host, ok := knownDoH[ip]; ok {
+			key4 := hostAndFam{host, 32}
+			key6 := hostAndFam{host, 128}
+			switch {
+			case ip.Is4():
+				if done[key4] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			case ip.Is6():
+				total4 := total[key4]
+				if total4 >= 2 {
+					// If we have two IPv4 IPs of the same provider
+					// already in the set, delay the IPv6 queries
+					// until halfway through the timeout (so wait
+					// 2.5 seconds). Even the network is IPv6-only,
+					// the DoH dialer will fallback to IPv6
+					// immediately anyway.
+					startDelay = responseTimeout / 2
+				} else if total4 == 1 {
+					startDelay += wellKnownHostBackupDelay
+				}
+				if done[key6] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			}
+			done[hostAndFam{host, ip.BitLen()}]++
+		}
+		rr[i] = resolverAndDelay{
+			ipp:        ipp,
+			startDelay: startDelay,
+		}
+	}
+	return rr
+}
+
+// setRoutes sets the routes to use for DNS forwarding. It's called by
+// Resolver.SetConfig on reconfig.
+//
+// The memory referenced by routesBySuffix should not be modified.
+func (f *forwarder) setRoutes(routesBySuffix map[dnsname.FQDN][]netaddr.IPPort) {
+	routes := make([]route, 0, len(routesBySuffix))
+	for suffix, ipps := range routesBySuffix {
+		routes = append(routes, route{
+			Suffix:    suffix,
+			Resolvers: resolversWithDelays(ipps),
+		})
+	}
+	// Sort from longest prefix to shortest.
+	sort.Slice(routes, func(i, j int) bool {
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
+	})
+
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.routes = routes
@@ -210,21 +328,104 @@ func (f *forwarder) packetListener(ip netaddr.IP) (packetListener, error) {
 	return lc, nil
 }
 
+func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client, ok bool) {
+	urlBase, ok = knownDoH[ip]
+	if !ok {
+		return
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if c, ok := f.dohClient[ip]; ok {
+		return urlBase, c, true
+	}
+	if f.dohClient == nil {
+		f.dohClient = map[netaddr.IP]*http.Client{}
+	}
+	nsDialer := netns.NewDialer()
+	c = &http.Client{
+		Transport: &http.Transport{
+			IdleConnTimeout: dohTransportTimeout,
+			DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) {
+				if !strings.HasPrefix(netw, "tcp") {
+					return nil, fmt.Errorf("unexpected network %q", netw)
+				}
+				c, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
+				// If v4 failed, try an equivalent v6 also in the time remaining.
+				if err != nil && ctx.Err() == nil {
+					if ip6, ok := dohV6(urlBase); ok && ip.Is4() {
+						if c6, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip6.String(), "443")); err == nil {
+							return c6, nil
+						}
+					}
+				}
+				return c, err
+			},
+		},
+	}
+	f.dohClient[ip] = c
+	return urlBase, c, true
+}
+
+const dohType = "application/dns-message"
+
+func (f *forwarder) releaseDoHSem() { <-f.dohSem }
+
+func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client, packet []byte) ([]byte, error) {
+	// Bound the number of HTTP requests in flight. This primarily
+	// matters for iOS where we're very memory constrained and
+	// HTTP requests are heavier on iOS where we don't include
+	// HTTP/2 for binary size reasons (as binaries on iOS linked
+	// with Go code cost memory proportional to the binary size,
+	// for reasons not fully understood).
+	select {
+	case f.dohSem <- struct{}{}:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	defer f.releaseDoHSem()
+
+	req, err := http.NewRequestWithContext(ctx, "POST", urlBase, bytes.NewReader(packet))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", dohType)
+	// Note: we don't currently set the Accept header (which is
+	// only a SHOULD in the spec) as iOS doesn't use HTTP/2 and
+	// we'd rather save a few bytes on outgoing requests when
+	// empirically no provider cares about the Accept header's
+	// absence.
+
+	hres, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer hres.Body.Close()
+	if hres.StatusCode != 200 {
+		return nil, errors.New(hres.Status)
+	}
+	if ct := hres.Header.Get("Content-Type"); ct != dohType {
+		return nil, fmt.Errorf("unexpected response Content-Type %q", ct)
+	}
+	return ioutil.ReadAll(hres.Body)
+}
+
 // send sends packet to dst. It is best effort.
 //
 // send expects the reply to have the same txid as txidOut.
 //
-// The provided closeOnCtxDone lets send register values to Close if
-// the caller's ctx expires. This avoids send from allocating its own
-// waiting goroutine to interrupt the ReadFrom, as memory is tight on
-// iOS and we want the number of pending DNS lookups to be bursty
-// without too much associated goroutine/memory cost.
-func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
-	// TODO(bradfitz): if dst.IP is 8.8.8.8 or 8.8.4.4 or 1.1.1.1, etc, or
-	// something dynamically probed earlier to support DoH or DoT,
-	// do that here instead.
+func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPort) ([]byte, error) {
+	ip := dst.IP()
 
-	ln, err := f.packetListener(dst.IP())
+	// Upgrade known DNS IPs to DoH (DNS-over-HTTPs).
+	if urlBase, dc, ok := f.getDoHClient(ip); ok {
+		res, err := f.sendDoH(ctx, urlBase, dc, fq.packet)
+		if err == nil || ctx.Err() != nil {
+			return res, err
+		}
+		f.logf("DoH error from %v: %v", ip, err)
+	}
+
+	ln, err := f.packetListener(ip)
 	if err != nil {
 		return nil, err
 	}
@@ -235,10 +436,10 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	defer conn.Close()
 
-	closeOnCtxDone.Add(conn)
-	defer closeOnCtxDone.Remove(conn)
+	fq.closeOnCtxDone.Add(conn)
+	defer fq.closeOnCtxDone.Remove(conn)
 
-	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
+	if _, err := conn.WriteTo(fq.packet, dst.UDPAddr()); err != nil {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
@@ -267,7 +468,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	out = out[:n]
 	txid := getTxID(out)
-	if txid != txidOut {
+	if txid != fq.txid {
 		return nil, errors.New("txid doesn't match")
 	}
 
@@ -290,7 +491,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 }
 
 // resolvers returns the resolvers to use for domain.
-func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
+func (f *forwarder) resolvers(domain dnsname.FQDN) []resolverAndDelay {
 	f.mu.Lock()
 	routes := f.routes
 	f.mu.Unlock()
@@ -302,6 +503,30 @@ func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
 	return nil
 }
 
+// forwardQuery is information and state about a forwarded DNS query that's
+// being sent to 1 or more upstreams.
+//
+// In the case of racing against multiple equivalent upstreams
+// (e.g. Google or CloudFlare's 4 DNS IPs: 2 IPv4 + 2 IPv6), this type
+// handles racing them more intelligently than just blasting away 4
+// queries at once.
+type forwardQuery struct {
+	txid   txid
+	packet []byte
+
+	// closeOnCtxDone lets send register values to Close if the
+	// caller's ctx expires. This avoids send from allocating its
+	// own waiting goroutine to interrupt the ReadFrom, as memory
+	// is tight on iOS and we want the number of pending DNS
+	// lookups to be bursty without too much associated
+	// goroutine/memory cost.
+	closeOnCtxDone *closePool
+
+	// TODO(bradfitz): add race delay state:
+	// mu sync.Mutex
+	// ...
+}
+
 // forward forwards the query to all upstream nameservers and returns the first response.
 func (f *forwarder) forward(query packet) error {
 	domain, err := nameFromQuery(query.bs)
@@ -309,7 +534,6 @@ func (f *forwarder) forward(query packet) error {
 		return err
 	}
 
-	txid := getTxID(query.bs)
 	clampEDNSSize(query.bs, maxResponseBytes)
 
 	resolvers := f.resolvers(domain)
@@ -317,8 +541,12 @@ func (f *forwarder) forward(query packet) error {
 		return errNoUpstreams
 	}
 
-	closeOnCtxDone := new(closePool)
-	defer closeOnCtxDone.Close()
+	fq := &forwardQuery{
+		txid:           getTxID(query.bs),
+		packet:         query.bs,
+		closeOnCtxDone: new(closePool),
+	}
+	defer fq.closeOnCtxDone.Close()
 
 	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
 	defer cancel()
@@ -329,9 +557,18 @@ func (f *forwarder) forward(query packet) error {
 		firstErr error
 	)
 
-	for _, ipp := range resolvers {
-		go func(ipp netaddr.IPPort) {
-			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
+	for _, rr := range resolvers {
+		go func(rr resolverAndDelay) {
+			if rr.startDelay > 0 {
+				timer := time.NewTimer(rr.startDelay)
+				select {
+				case <-timer.C:
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			resb, err := f.send(ctx, fq, rr.ipp)
 			if err != nil {
 				mu.Lock()
 				defer mu.Unlock()
@@ -344,7 +581,7 @@ func (f *forwarder) forward(query packet) error {
 			case resc <- resb:
 			default:
 			}
-		}(ipp)
+		}(rr)
 	}
 
 	select {
@@ -432,3 +669,63 @@ func (p *closePool) Close() error {
 	}
 	return nil
 }
+
+var knownDoH = map[netaddr.IP]string{}
+
+var dohIPsOfBase = map[string][]netaddr.IP{}
+
+func addDoH(ipStr, base string) {
+	ip := netaddr.MustParseIP(ipStr)
+	knownDoH[ip] = base
+	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
+}
+
+func dohV6(base string) (ip netaddr.IP, ok bool) {
+	for _, ip := range dohIPsOfBase[base] {
+		if ip.Is6() {
+			return ip, true
+		}
+	}
+	return ip, false
+}
+
+func init() {
+	// Cloudflare
+	addDoH("1.1.1.1", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.1", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1111", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1001", "https://cloudflare-dns.com/dns-query")
+
+	// Cloudflare -Malware
+	addDoH("1.1.1.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1112", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1002", "https://security.cloudflare-dns.com/dns-query")
+
+	// Cloudflare -Malware -Adult
+	addDoH("1.1.1.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1113", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1003", "https://family.cloudflare-dns.com/dns-query")
+
+	// Google
+	addDoH("8.8.8.8", "https://dns.google/dns-query")
+	addDoH("8.8.4.4", "https://dns.google/dns-query")
+	addDoH("2001:4860:4860::8888", "https://dns.google/dns-query")
+	addDoH("2001:4860:4860::8844", "https://dns.google/dns-query")
+
+	// OpenDNS
+	// TODO(bradfitz): OpenDNS is unique amongst this current set in that
+	// its DoH DNS names resolve to different IPs than its normal DNS
+	// IPs. Support that later. For now we assume that they're the same.
+	// addDoH("208.67.222.222", "https://doh.opendns.com/dns-query")
+	// addDoH("208.67.220.220", "https://doh.opendns.com/dns-query")
+	// addDoH("208.67.222.123", "https://doh.familyshield.opendns.com/dns-query")
+	// addDoH("208.67.220.123", "https://doh.familyshield.opendns.com/dns-query")
+
+	// Quad9
+	addDoH("9.9.9.9", "https://dns.quad9.net/dns-query")
+	addDoH("149.112.112.112", "https://dns.quad9.net/dns-query")
+	addDoH("2620:fe::fe", "https://dns.quad9.net/dns-query")
+	addDoH("2620:fe::fe:9", "https://dns.quad9.net/dns-query")
+}

net/dns/resolver/forwarder_test.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+func (rr resolverAndDelay) String() string {
+	return fmt.Sprintf("%v+%v", rr.ipp, rr.startDelay)
+}
+
+func TestResolversWithDelays(t *testing.T) {
+	// query
+	q := func(ss ...string) (ipps []netaddr.IPPort) {
+		for _, s := range ss {
+			ipps = append(ipps, netaddr.MustParseIPPort(s))
+		}
+		return
+	}
+	// output
+	o := func(ss ...string) (rr []resolverAndDelay) {
+		for _, s := range ss {
+			var d time.Duration
+			if i := strings.Index(s, "+"); i != -1 {
+				var err error
+				d, err = time.ParseDuration(s[i+1:])
+				if err != nil {
+					panic(fmt.Sprintf("parsing duration in %q: %v", s, err))
+				}
+				s = s[:i]
+			}
+			rr = append(rr, resolverAndDelay{
+				ipp:        netaddr.MustParseIPPort(s),
+				startDelay: d,
+			})
+		}
+		return
+	}
+
+	tests := []struct {
+		name string
+		in   []netaddr.IPPort
+		want []resolverAndDelay
+	}{
+		{
+			name: "unknown-no-delays",
+			in:   q("1.2.3.4:53", "2.3.4.5:53"),
+			want: o("1.2.3.4:53", "2.3.4.5:53"),
+		},
+		{
+			name: "google-all-ipv4",
+			in:   q("8.8.8.8:53", "8.8.4.4:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms"),
+		},
+		{
+			name: "google-only-ipv6",
+			in:   q("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53+200ms"),
+		},
+		{
+			name: "google-all-four",
+			in:   q("8.8.8.8:53", "8.8.4.4:53", "[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms", "[2001:4860:4860::8888]:53+2.5s", "[2001:4860:4860::8844]:53+2.7s"),
+		},
+		{
+			name: "quad9-one-v4-one-v6",
+			in:   q("9.9.9.9:53", "[2620:fe::fe]:53"),
+			want: o("9.9.9.9:53", "[2620:fe::fe]:53+200ms"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := resolversWithDelays(tt.in)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+
+}

net/dns/resolver/tsdns.go

@@ -7,8 +7,10 @@
 package resolver
 
 import (
+	"bufio"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"runtime"
 	"sort"
 	"strings"
@@ -79,6 +81,74 @@ type Config struct {
 	LocalDomains []dnsname.FQDN
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{Routes:")
+	WriteRoutes(w, c.Routes)
+	fmt.Fprintf(w, " Hosts:%v LocalDomains:[", len(c.Hosts))
+	space := false
+	arpa := 0
+	for _, d := range c.LocalDomains {
+		if strings.HasSuffix(string(d), ".arpa.") {
+			arpa++
+			continue
+		}
+		if space {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(d))
+		space = true
+	}
+	w.WriteString("]")
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+	w.WriteString("}")
+}
+
+// WriteIPPorts writes vv to w.
+func WriteIPPorts(w *bufio.Writer, vv []netaddr.IPPort) {
+	w.WriteByte('[')
+	var b []byte
+	for i, v := range vv {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		b = v.AppendTo(b[:0])
+		w.Write(b)
+	}
+	w.WriteByte(']')
+}
+
+// WriteRoutes writes routes to w, omitting *.arpa routes and instead
+// summarizing how many of them there were.
+func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]netaddr.IPPort) {
+	var kk []dnsname.FQDN
+	arpa := 0
+	for k := range routes {
+		if strings.HasSuffix(string(k), ".arpa.") {
+			arpa++
+			continue
+		}
+		kk = append(kk, k)
+	}
+	sort.Slice(kk, func(i, j int) bool { return kk[i] < kk[j] })
+	w.WriteByte('{')
+	for i, k := range kk {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(k))
+		w.WriteByte(':')
+		WriteIPPorts(w, routes[k])
+	}
+	w.WriteByte('}')
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+}
+
 // Resolver is a DNS resolver for nodes on the Tailscale network,
 // associating them with domain names of the form <mynode>.<mydomain>.<root>.
 // If it is asked to resolve a domain that is not of that form,
@@ -138,7 +208,6 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		r.saveConfigForTests(cfg)
 	}
 
-	routes := make([]route, 0, len(cfg.Routes))
 	reverse := make(map[netaddr.IP]dnsname.FQDN, len(cfg.Hosts))
 
 	for host, ips := range cfg.Hosts {
@@ -147,18 +216,7 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		}
 	}
 
-	for suffix, ips := range cfg.Routes {
-		routes = append(routes, route{
-			Suffix:    suffix,
-			Resolvers: ips,
-		})
-	}
-	// Sort from longest prefix to shortest.
-	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
-	})
-
-	r.forwarder.setRoutes(routes)
+	r.forwarder.setRoutes(cfg.Routes)
 
 	r.mu.Lock()
 	defer r.mu.Unlock()

net/dns/resolver/tsdns_test.go

@@ -931,8 +931,11 @@ func TestAllocs(t *testing.T) {
 		query []byte
 		want  int
 	}{
-		// Name lowercasing and response slice created by dns.NewBuilder.
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
+		// Name lowercasing, response slice created by dns.NewBuilder,
+		// and closure allocation from go call.
+		// (Closure allocation only happens when using new register ABI,
+		// which is amd64 with Go 1.17, and probably more platforms later.)
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 3},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
 		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}

net/dnscache/dnscache.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// TODO(bradfitz): update this code to use netaddr more
+
 // Package dnscache contains a minimal DNS cache that makes a bunch of
 // assumptions that are only valid for us. Not recommended for general use.
 package dnscache
@@ -78,8 +80,9 @@ type Resolver struct {
 }
 
 type ipCacheEntry struct {
-	ip      net.IP // either v4 or v6
-	ip6     net.IP // nil if no v4 or no v6
+	ip      net.IP       // either v4 or v6
+	ip6     net.IP       // nil if no v4 or no v6
+	allIPs  []net.IPAddr // 1+ v4 and/or v6
 	expires time.Time
 }
 
@@ -105,81 +108,82 @@ var debug, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DNS_CACHE"))
 //
 // If err is nil, ip will be non-nil. The v6 address may be nil even
 // with a nil error.
-func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, err error) {
+func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, allIPs []net.IPAddr, err error) {
 	if ip := net.ParseIP(host); ip != nil {
 		if ip4 := ip.To4(); ip4 != nil {
-			return ip4, nil, nil
+			return ip4, nil, []net.IPAddr{{IP: ip4}}, nil
 		}
 		if debug {
 			log.Printf("dnscache: %q is an IP", host)
 		}
-		return ip, nil, nil
+		return ip, nil, []net.IPAddr{{IP: ip}}, nil
 	}
 
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q = %v (cached)", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
-	type ipPair struct {
+	type ipRes struct {
 		ip, ip6 net.IP
+		allIPs  []net.IPAddr
 	}
 	ch := r.sf.DoChan(host, func() (interface{}, error) {
-		ip, ip6, err := r.lookupIP(host)
+		ip, ip6, allIPs, err := r.lookupIP(host)
 		if err != nil {
 			return nil, err
 		}
-		return ipPair{ip, ip6}, nil
+		return ipRes{ip, ip6, allIPs}, nil
 	})
 	select {
 	case res := <-ch:
 		if res.Err != nil {
 			if r.UseLastGood {
-				if ip, ip6, ok := r.lookupIPCacheExpired(host); ok {
+				if ip, ip6, allIPs, ok := r.lookupIPCacheExpired(host); ok {
 					if debug {
 						log.Printf("dnscache: %q using %v after error", host, ip)
 					}
-					return ip, ip6, nil
+					return ip, ip6, allIPs, nil
 				}
 			}
 			if debug {
 				log.Printf("dnscache: error resolving %q: %v", host, res.Err)
 			}
-			return nil, nil, res.Err
+			return nil, nil, nil, res.Err
 		}
-		pair := res.Val.(ipPair)
-		return pair.ip, pair.ip6, nil
+		r := res.Val.(ipRes)
+		return r.ip, r.ip6, r.allIPs, nil
 	case <-ctx.Done():
 		if debug {
 			log.Printf("dnscache: context done while resolving %q: %v", host, ctx.Err())
 		}
-		return nil, nil, ctx.Err()
+		return nil, nil, nil, ctx.Err()
 	}
 }
 
-func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok && ent.expires.After(time.Now()) {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
-func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
 func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	if r.UseLastGood {
-		if _, _, ok := r.lookupIPCacheExpired(host); ok {
+		if _, _, _, ok := r.lookupIPCacheExpired(host); ok {
 			// If we have some previous good value for this host,
 			// don't give this DNS lookup much time. If we're in a
 			// situation where the user's DNS server is unreachable
@@ -194,12 +198,12 @@ func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	return 10 * time.Second
 }
 
-func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, err error) {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q found in cache as %v", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), r.lookupTimeoutForHost(host))
@@ -218,10 +222,10 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 		}
 	}
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	if len(ips) == 0 {
-		return nil, nil, fmt.Errorf("no IPs for %q found", host)
+		return nil, nil, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
 	have4 := false
@@ -240,12 +244,12 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 			}
 		}
 	}
-	r.addIPCache(host, ip, ip6, r.ttl())
-	return ip, ip6, nil
+	r.addIPCache(host, ip, ip6, ips, r.ttl())
+	return ip, ip6, ips, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
-	if isPrivateIP(ip) {
+func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr, d time.Duration) {
+	if naIP, _ := netaddr.FromStdIP(ip); naIP.IsPrivate() {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
 		if debug {
@@ -263,27 +267,14 @@ func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
 	if r.ipCache == nil {
 		r.ipCache = make(map[string]ipCacheEntry)
 	}
-	r.ipCache[host] = ipCacheEntry{ip: ip, ip6: ip6, expires: time.Now().Add(d)}
-}
-
-func mustCIDR(s string) *net.IPNet {
-	_, ipNet, err := net.ParseCIDR(s)
-	if err != nil {
-		panic(err)
+	r.ipCache[host] = ipCacheEntry{
+		ip:      ip,
+		ip6:     ip6,
+		allIPs:  allIPs,
+		expires: time.Now().Add(d),
 	}
-	return ipNet
 }
 
-func isPrivateIP(ip net.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
-var (
-	private1 = mustCIDR("10.0.0.0/8")
-	private2 = mustCIDR("172.16.0.0/12")
-	private3 = mustCIDR("192.168.0.0/16")
-)
-
 type DialContextFunc func(ctx context.Context, network, address string) (net.Conn, error)
 
 // Dialer returns a wrapped DialContext func that uses the provided dnsCache.
@@ -305,41 +296,131 @@ func Dialer(fwd DialContextFunc, dnsCache *Resolver) DialContextFunc {
 				// Return with original error
 				return
 			}
-			for _, ip := range ips {
-				dst := net.JoinHostPort(ip.String(), port)
-				if c, err := fwd(ctx, network, dst); err == nil {
-					retConn = c
-					ret = nil
-					return
-				}
+			if c, err := raceDial(ctx, fwd, network, ips, port); err == nil {
+				retConn = c
+				ret = nil
+				return
 			}
 		}()
 
-		ip, ip6, err := dnsCache.LookupIP(ctx, host)
+		ip, ip6, allIPs, err := dnsCache.LookupIP(ctx, host)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve %q: %w", host, err)
 		}
-		dst := net.JoinHostPort(ip.String(), port)
-		if debug {
-			log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+		i4s := v4addrs(allIPs)
+		if len(i4s) < 2 {
+			dst := net.JoinHostPort(ip.String(), port)
+			if debug {
+				log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+			}
+			c, err := fwd(ctx, network, dst)
+			if err == nil || ctx.Err() != nil || ip6 == nil {
+				return c, err
+			}
+			// Fall back to trying IPv6.
+			dst = net.JoinHostPort(ip6.String(), port)
+			return fwd(ctx, network, dst)
 		}
-		c, err := fwd(ctx, network, dst)
-		if err == nil || ctx.Err() != nil || ip6 == nil {
-			return c, err
-		}
-		// Fall back to trying IPv6.
-		// TODO(bradfitz): this is a primarily for IPv6-only
-		// hosts; it's not supposed to be a real Happy
-		// Eyeballs implementation. We should use the net
-		// package's implementation of that by plumbing this
-		// dnscache impl into net.Dialer.Resolver.Dial and
-		// unmarshal/marshal DNS queries/responses to the net
-		// package. This works for v6-only hosts for now.
-		dst = net.JoinHostPort(ip6.String(), port)
-		return fwd(ctx, network, dst)
+
+		// Multiple IPv4 candidates, and 0+ IPv6.
+		ipsToTry := append(i4s, v6addrs(allIPs)...)
+		return raceDial(ctx, fwd, network, ipsToTry, port)
 	}
 }
 
+// fallbackDelay is how long to wait between trying subsequent
+// addresses when multiple options are available.
+// 300ms is the same as Go's Happy Eyeballs fallbackDelay value.
+const fallbackDelay = 300 * time.Millisecond
+
+// raceDial tries to dial port on each ip in ips, starting a new race
+// dial every fallbackDelay apart, returning whichever completes first.
+func raceDial(ctx context.Context, fwd DialContextFunc, network string, ips []netaddr.IP, port string) (net.Conn, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	type res struct {
+		c   net.Conn
+		err error
+	}
+	resc := make(chan res)           // must be unbuffered
+	failBoost := make(chan struct{}) // best effort send on dial failure
+
+	go func() {
+		for i, ip := range ips {
+			if i != 0 {
+				timer := time.NewTimer(fallbackDelay)
+				select {
+				case <-timer.C:
+				case <-failBoost:
+					timer.Stop()
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			go func(ip netaddr.IP) {
+				c, err := fwd(ctx, network, net.JoinHostPort(ip.String(), port))
+				if err != nil {
+					// Best effort wake-up a pending dial.
+					// e.g. IPv4 dials failing quickly on an IPv6-only system.
+					// In that case we don't want to wait 300ms per IPv4 before
+					// we get to the IPv6 addresses.
+					select {
+					case failBoost <- struct{}{}:
+					default:
+					}
+				}
+				select {
+				case resc <- res{c, err}:
+				case <-ctx.Done():
+					if c != nil {
+						c.Close()
+					}
+				}
+			}(ip)
+		}
+	}()
+
+	var firstErr error
+	var fails int
+	for {
+		select {
+		case r := <-resc:
+			if r.c != nil {
+				return r.c, nil
+			}
+			fails++
+			if firstErr == nil {
+				firstErr = r.err
+			}
+			if fails == len(ips) {
+				return nil, firstErr
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+func v4addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is4() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
+func v6addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is6() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
 var errTLSHandshakeTimeout = errors.New("timeout doing TLS handshake")
 
 // TLSDialer is like Dialer but returns a func suitable for using with net/http.Transport.DialTLSContext.

net/dnscache/dnscache_test.go

@@ -5,24 +5,29 @@
 package dnscache
 
 import (
+	"context"
+	"flag"
 	"net"
 	"testing"
+	"time"
 )
 
-func TestIsPrivateIP(t *testing.T) {
-	tests := []struct {
-		ip   string
-		want bool
-	}{
-		{"10.1.2.3", true},
-		{"172.16.1.100", true},
-		{"192.168.1.1", true},
-		{"1.2.3.4", false},
-	}
+var dialTest = flag.String("dial-test", "", "if non-empty, addr:port to test dial")
 
-	for _, test := range tests {
-		if got := isPrivateIP(net.ParseIP(test.ip)); got != test.want {
-			t.Errorf("isPrivateIP(%q)=%v, want %v", test.ip, got, test.want)
-		}
+func TestDialer(t *testing.T) {
+	if *dialTest == "" {
+		t.Skip("skipping; --dial-test is blank")
 	}
+	r := new(Resolver)
+	var std net.Dialer
+	dialer := Dialer(std.DialContext, r)
+	t0 := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	c, err := dialer(ctx, "tcp", *dialTest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("dialed in %v", time.Since(t0))
+	c.Close()
 }

net/dnsfallback/dns-fallback-servers.json

@@ -90,18 +90,25 @@
 			"RegionName": "r4",
 			"Nodes": [
 				{
-					"Name": "4a",
+					"Name": "4c",
 					"RegionID": 4,
-					"HostName": "derp4.tailscale.com",
-					"IPv4": "167.172.182.26",
-					"IPv6": "2a03:b0c0:3:e0::36e:9001"
+					"HostName": "derp4c.tailscale.com",
+					"IPv4": "134.122.77.138",
+					"IPv6": "2a03:b0c0:3:d0::1501:6001"
 				},
 				{
-					"Name": "4b",
+					"Name": "4d",
 					"RegionID": 4,
-					"HostName": "derp4b.tailscale.com",
-					"IPv4": "157.230.25.0",
-					"IPv6": "2a03:b0c0:3:e0::58f:3001"
+					"HostName": "derp4d.tailscale.com",
+					"IPv4": "134.122.94.167",
+					"IPv6": "2a03:b0c0:3:d0::1501:b001"
+				},
+				{
+					"Name": "4e",
+					"RegionID": 4,
+					"HostName": "derp4e.tailscale.com",
+					"IPv4": "134.122.74.153",
+					"IPv6": "2a03:b0c0:3:d0::29:9001"
 				}
 			]
 		},
@@ -153,11 +160,25 @@
 			"RegionName": "r8",
 			"Nodes": [
 				{
-					"Name": "8a",
+					"Name": "8b",
 					"RegionID": 8,
-					"HostName": "derp8.tailscale.com",
-					"IPv4": "167.71.139.179",
-					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
+					"HostName": "derp8b.tailscale.com",
+					"IPv4": "46.101.74.201",
+					"IPv6": "2a03:b0c0:1:d0::ec1:e001"
+				},
+				{
+					"Name": "8c",
+					"RegionID": 8,
+					"HostName": "derp8c.tailscale.com",
+					"IPv4": "206.189.16.32",
+					"IPv6": "2a03:b0c0:1:d0::e1f:4001"
+				},
+				{
+					"Name": "8d",
+					"RegionID": 8,
+					"HostName": "derp8d.tailscale.com",
+					"IPv4": "178.62.44.132",
+					"IPv6": "2a03:b0c0:1:d0::e08:e001"
 				}
 			]
 		},

net/interfaces/interfaces.go

@@ -134,7 +134,7 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 					// but their OS supports IPv6 so they have an fe80::
 					// address. We don't want to report all of those
 					// IPv6 LL to Control.
-				} else if ip.Is6() && tsaddr.IsULA(ip) {
+				} else if ip.Is6() && ip.IsPrivate() {
 					// Google Cloud Run uses NAT with IPv6 Unique
 					// Local Addresses to provide IPv6 connectivity.
 					ula6 = append(ula6, ip)
@@ -479,7 +479,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
 		ip := pfx.IP()
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			if privateIP == "" {
 				privateIP = ip.String()
 			}
@@ -519,21 +519,15 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
-		for _, prefix := range privatev4s {
-			if prefix.Contains(gateway) && prefix.Contains(ip) {
-				myIP = ip
-				ok = true
-				return
-			}
+		if gateway.IsPrivate() && ip.IsPrivate() {
+			myIP = ip
+			ok = true
+			return
 		}
 	})
 	return gateway, myIP, !myIP.IsZero()
 }
 
-func isPrivateIP(ip netaddr.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
 // isUsableV4 reports whether ip is a usable IPv4 address which could
 // conceivably be used to get Internet connectivity. Globally routable and
 // private IPv4 addresses are always Usable, and link local 169.254.x.x
@@ -554,23 +548,11 @@ func isUsableV4(ip netaddr.IP) bool {
 // (fc00::/7) are in some environments used with address translation.
 func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
-		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
-}
-
-func mustCIDR(s string) netaddr.IPPrefix {
-	prefix, err := netaddr.ParseIPPrefix(s)
-	if err != nil {
-		panic(err)
-	}
-	return prefix
+		(ip.Is6() && ip.IsPrivate() && !tsaddr.TailscaleULARange().Contains(ip))
 }
 
 var (
-	private1   = mustCIDR("10.0.0.0/8")
-	private2   = mustCIDR("172.16.0.0/12")
-	private3   = mustCIDR("192.168.0.0/16")
-	privatev4s = []netaddr.IPPrefix{private1, private2, private3}
-	v6Global1  = mustCIDR("2000::/3")
+	v6Global1 = netaddr.MustParseIPPrefix("2000::/3")
 )
 
 // anyInterestingIP reports whether pfxs contains any IP that matches

net/interfaces/interfaces_darwin_test.go

@@ -73,7 +73,7 @@ func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
 			return nil
 		}
 		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
-		if err == nil && isPrivateIP(ip) {
+		if err == nil && ip.IsPrivate() {
 			ret = ip
 			// We've found what we're looking for.
 			return errStopReadingNetstatTable

net/interfaces/interfaces_linux.go

@@ -72,7 +72,7 @@ func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
 			return nil // ignore error, skip line and keep going
 		}
 		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			ret = ip
 		}
 		return nil

net/interfaces/interfaces_linux_test.go

@@ -73,6 +73,32 @@ func TestExtremelyLongProcNetRoute(t *testing.T) {
 	}
 }
 
+// test the specific /proc/net/route path as found on AWS App Runner instances
+func TestAwsAppRunnerDefaultRouteInterface(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "CloudRun")
+	buf := []byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" +
+		"eth0\t00000000\tF9AFFEA9\t0003\t0\t0\t0\t00000000\t0\t0\t0\n" +
+		"*\tFEA9FEA9\t00000000\t0005\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n" +
+		"ecs-eth0\t02AAFEA9\t01ACFEA9\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n" +
+		"ecs-eth0\t00ACFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n" +
+		"eth0\t00AFFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n")
+	err := ioutil.WriteFile(procNetRoutePath, buf, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "eth0" {
+		t.Fatalf("got %s, want eth0", got)
+	}
+}
+
 func BenchmarkDefaultRouteInterface(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {

net/interfaces/interfaces_test.go

@@ -58,6 +58,8 @@ func TestIsUsableV6(t *testing.T) {
 		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
 		{"Link Local", "fe80::1", false},
 		{"Global", "2602::1", true},
+		{"IPv4 public", "192.0.2.1", false},
+		{"IPv4 private", "192.168.1.1", false},
 	}
 
 	for _, test := range tests {

net/interfaces/interfaces_windows.go

@@ -93,7 +93,7 @@ func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
 		}
 	}
 
-	if !ret.IsZero() && !isPrivateIP(ret) {
+	if !ret.IsZero() && !ret.IsPrivate() {
 		// Default route has a non-private gateway
 		return netaddr.IP{}, false
 	}

net/netns/netns.go

@@ -17,6 +17,8 @@ package netns
 import (
 	"context"
 	"net"
+
+	"inet.af/netaddr"
 )
 
 // Listener returns a new net.Listener with its Control hook func
@@ -66,3 +68,19 @@ type Dialer interface {
 	Dial(network, address string) (net.Conn, error)
 	DialContext(ctx context.Context, network, address string) (net.Conn, error)
 }
+
+func isLocalhost(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		// error means the string didn't contain a port number, so use the string directly
+		host = addr
+	}
+
+	// localhost6 == RedHat /etc/hosts for ::1, ip6-loopback & ip6-localhost == Debian /etc/hosts for ::1
+	if host == "localhost" || host == "localhost6" || host == "ip6-loopback" || host == "ip6-localhost" {
+		return true
+	}
+
+	ip, _ := netaddr.ParseIP(host)
+	return ip.IsLoopback()
+}

net/netns/netns_linux.go

@@ -7,8 +7,8 @@
 package netns
 
 import (
-	"flag"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
 	"sync"
@@ -26,32 +26,54 @@ import (
 // wgengine/router/router_linux.go.
 const tailscaleBypassMark = 0x80000
 
-// ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
-var ipRuleOnce struct {
+// socketMarkWorksOnce is the sync.Once & cached value for useSocketMark.
+var socketMarkWorksOnce struct {
 	sync.Once
 	v bool
 }
 
-// ipRuleAvailable reports whether the 'ip rule' command works.
-// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
-func ipRuleAvailable() bool {
-	ipRuleOnce.Do(func() {
-		ipRuleOnce.v = exec.Command("ip", "rule").Run() == nil
+// socketMarkWorks returns whether SO_MARK works.
+func socketMarkWorks() bool {
+	addr, err := net.ResolveUDPAddr("udp", "127.0.0.1:1")
+	if err != nil {
+		return true // unsure, returning true does the least harm.
+	}
+
+	sConn, err := net.DialUDP("udp", nil, addr)
+	if err != nil {
+		return true // unsure, return true
+	}
+	defer sConn.Close()
+
+	rConn, err := sConn.SyscallConn()
+	if err != nil {
+		return true // unsure, return true
+	}
+
+	var sockErr error
+	err = rConn.Control(func(fd uintptr) {
+		sockErr = setBypassMark(fd)
 	})
-	return ipRuleOnce.v
+	if err != nil || sockErr != nil {
+		return false
+	}
+
+	return true
+}
+
+// useSocketMark reports whether SO_MARK works.
+// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
+func useSocketMark() bool {
+	socketMarkWorksOnce.Do(func() {
+		ipRuleWorks := exec.Command("ip", "rule").Run() == nil
+		socketMarkWorksOnce.v = ipRuleWorks && socketMarkWorks()
+	})
+	return socketMarkWorksOnce.v
 }
 
 // ignoreErrors returns true if we should ignore setsocketopt errors in
 // this instance.
 func ignoreErrors() bool {
-	// If we're in a test, ignore errors. Assume the test knows
-	// what it's doing and will do its own skips or permission
-	// checks if it's setting up a world that needs netns to work.
-	// But by default, assume that tests don't need netns and it's
-	// harmless to ignore the sockopts failing.
-	if flag.CommandLine.Lookup("test.v") != nil {
-		return true
-	}
 	if os.Getuid() != 0 {
 		// only root can manipulate these socket flags
 		return true
@@ -64,9 +86,14 @@ func ignoreErrors() bool {
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
 func control(network, address string, c syscall.RawConn) error {
+	if isLocalhost(address) {
+		// Don't bind to an interface for localhost connections.
+		return nil
+	}
+
 	var sockErr error
 	err := c.Control(func(fd uintptr) {
-		if ipRuleAvailable() {
+		if useSocketMark() {
 			sockErr = setBypassMark(fd)
 		} else {
 			sockErr = bindToDevice(fd)

net/netns/netns_linux_test.go

@@ -49,3 +49,9 @@ func TestBypassMarkInSync(t *testing.T) {
 	}
 	t.Errorf("tailscaleBypassMark not found in router_linux.go")
 }
+
+func TestSocketMarkWorks(t *testing.T) {
+	_ = socketMarkWorks()
+	// we cannot actually assert whether the test runner has SO_MARK available
+	// or not, as we don't know. We're just checking that it doesn't panic.
+}

net/netns/netns_test.go

@@ -40,3 +40,40 @@ func TestDial(t *testing.T) {
 	defer c.Close()
 	t.Logf("got addr %v", c.RemoteAddr())
 }
+
+func TestIsLocalhost(t *testing.T) {
+	tests := []struct {
+		name string
+		host string
+		want bool
+	}{
+		{"IPv4 loopback", "127.0.0.1", true},
+		{"IPv4 !loopback", "192.168.0.1", false},
+		{"IPv4 loopback with port", "127.0.0.1:1", true},
+		{"IPv4 !loopback with port", "192.168.0.1:1", false},
+		{"IPv4 unspecified", "0.0.0.0", false},
+		{"IPv4 unspecified with port", "0.0.0.0:1", false},
+		{"IPv6 loopback", "::1", true},
+		{"IPv6 !loopback", "2001:4860:4860::8888", false},
+		{"IPv6 loopback with port", "[::1]:1", true},
+		{"IPv6 !loopback with port", "[2001:4860:4860::8888]:1", false},
+		{"IPv6 unspecified", "::", false},
+		{"IPv6 unspecified with port", "[::]:1", false},
+		{"empty", "", false},
+		{"hostname", "example.com", false},
+		{"localhost", "localhost", true},
+		{"localhost6", "localhost6", true},
+		{"localhost with port", "localhost:1", true},
+		{"localhost6 with port", "localhost6:1", true},
+		{"ip6-localhost", "ip6-localhost", true},
+		{"ip6-localhost with port", "ip6-localhost:1", true},
+		{"ip6-loopback", "ip6-loopback", true},
+		{"ip6-loopback with port", "ip6-loopback:1", true},
+	}
+
+	for _, test := range tests {
+		if got := isLocalhost(test.host); got != test.want {
+			t.Errorf("isLocalhost(%q) = %v, want %v", test.name, got, test.want)
+		}
+	}
+}

net/portmapper/disabled_stubs.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"context"
+
+	"inet.af/netaddr"
+)
+
+type upnpClient interface{}
+
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	return netaddr.IPPort{}, false
+}

net/portmapper/portmapper.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package portmapper is a UDP port mapping client. It currently only does
-// NAT-PMP, but will likely do UPnP and perhaps PCP later.
+// Package portmapper is a UDP port mapping client. It currently allows for mapping over
+// NAT-PMP and UPnP, but will perhaps do PCP later.
 package portmapper
 
 import (
@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"sync"
 	"time"
 
@@ -63,18 +64,40 @@ type Client struct {
 	pmpPubIPTime time.Time  // time pmpPubIP last verified
 	pmpLastEpoch uint32
 
-	pcpSawTime  time.Time // time we last saw PCP was available
-	uPnPSawTime time.Time // time we last saw UPnP was available
+	pcpSawTime time.Time // time we last saw PCP was available
 
-	localPort  uint16
-	pmpMapping *pmpMapping // non-nil if we have a PMP mapping
+	uPnPSawTime    time.Time         // time we last saw UPnP was available
+	uPnPMeta       uPnPDiscoResponse // Location header from UPnP UDP discovery response
+	uPnPHTTPClient *http.Client      // nil until needed
+
+	localPort uint16
+
+	mapping mapping // non-nil if we have a mapping
+}
+
+// mapping represents a created port-mapping over some protocol.  It specifies a lease duration,
+// how to release the mapping, and whether the map is still valid.
+//
+// After a mapping is created, it should be immutable, and thus reads should be safe across
+// concurrent goroutines.
+type mapping interface {
+	// Release will attempt to unmap the established port mapping. It will block until completion,
+	// but can be called asynchronously. Release should be idempotent, and thus even if called
+	// multiple times should not cause additional side-effects.
+	Release(context.Context)
+	// goodUntil will return the lease time that the mapping is valid for.
+	GoodUntil() time.Time
+	// renewAfter returns the earliest time that the mapping should be renewed.
+	RenewAfter() time.Time
+	// externalIPPort indicates what port the mapping can be reached from on the outside.
+	External() netaddr.IPPort
 }
 
 // HaveMapping reports whether we have a current valid mapping.
 func (c *Client) HaveMapping() bool {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	return c.pmpMapping != nil && c.pmpMapping.goodUntil.After(time.Now())
+	return c.mapping != nil && c.mapping.GoodUntil().After(time.Now())
 }
 
 // pmpMapping is an already-created PMP mapping.
@@ -94,9 +117,13 @@ func (m *pmpMapping) externalValid() bool {
 	return !m.external.IP().IsZero() && m.external.Port() != 0
 }
 
-// release does a best effort fire-and-forget release of the PMP mapping m.
-func (m *pmpMapping) release() {
-	uc, err := netns.Listener().ListenPacket(context.Background(), "udp4", ":0")
+func (p *pmpMapping) GoodUntil() time.Time     { return p.goodUntil }
+func (p *pmpMapping) RenewAfter() time.Time    { return p.renewAfter }
+func (p *pmpMapping) External() netaddr.IPPort { return p.external }
+
+// Release does a best effort fire-and-forget release of the PMP mapping m.
+func (m *pmpMapping) Release(ctx context.Context) {
+	uc, err := netns.Listener().ListenPacket(ctx, "udp4", ":0")
 	if err != nil {
 		return
 	}
@@ -166,7 +193,6 @@ func (c *Client) gatewayAndSelfIP() (gw, myIP netaddr.IP, ok bool) {
 		gw = netaddr.IP{}
 		myIP = netaddr.IP{}
 	}
-
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -179,11 +205,11 @@ func (c *Client) gatewayAndSelfIP() (gw, myIP netaddr.IP, ok bool) {
 }
 
 func (c *Client) invalidateMappingsLocked(releaseOld bool) {
-	if c.pmpMapping != nil {
+	if c.mapping != nil {
 		if releaseOld {
-			c.pmpMapping.release()
+			c.mapping.Release(context.Background())
 		}
-		c.pmpMapping = nil
+		c.mapping = nil
 	}
 	c.pmpPubIP = netaddr.IP{}
 	c.pmpPubIPTime = time.Time{}
@@ -249,7 +275,7 @@ func IsNoMappingError(err error) bool {
 
 var (
 	ErrNoPortMappingServices = errors.New("no port mapping services were found")
-	ErrGatewayNotFound       = errors.New("failed to look up gateway address")
+	ErrGatewayRange          = errors.New("skipping portmap; gateway range likely lacks support")
 )
 
 // GetCachedMappingOrStartCreatingOne quickly returns with our current cached portmapping, if any.
@@ -262,12 +288,12 @@ func (c *Client) GetCachedMappingOrStartCreatingOne() (external netaddr.IPPort,
 
 	// Do we have an existing mapping that's valid?
 	now := time.Now()
-	if m := c.pmpMapping; m != nil {
-		if now.Before(m.goodUntil) {
-			if now.After(m.renewAfter) {
+	if m := c.mapping; m != nil {
+		if now.Before(m.GoodUntil()) {
+			if now.After(m.RenewAfter()) {
 				c.maybeStartMappingLocked()
 			}
-			return m.external, true
+			return m.External(), true
 		}
 	}
 
@@ -310,14 +336,15 @@ func (c *Client) createMapping() {
 func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return netaddr.IPPort{}, NoMappingError{ErrGatewayNotFound}
+		return netaddr.IPPort{}, NoMappingError{ErrGatewayRange}
 	}
 
 	c.mu.Lock()
 	localPort := c.localPort
+	internalAddr := netaddr.IPPortFrom(myIP, localPort)
 	m := &pmpMapping{
 		gw:       gw,
-		internal: netaddr.IPPortFrom(myIP, localPort),
+		internal: internalAddr,
 	}
 
 	// prevPort is the port we had most previously, if any. We try
@@ -326,13 +353,13 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPor
 
 	// Do we have an existing mapping that's valid?
 	now := time.Now()
-	if m := c.pmpMapping; m != nil {
-		if now.Before(m.renewAfter) {
+	if m := c.mapping; m != nil {
+		if now.Before(m.RenewAfter()) {
 			defer c.mu.Unlock()
-			return m.external, nil
+			return m.External(), nil
 		}
 		// The mapping might still be valid, so just try to renew it.
-		prevPort = m.external.Port()
+		prevPort = m.External().Port()
 	}
 
 	// If we just did a Probe (e.g. via netchecker) but didn't
@@ -344,6 +371,10 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	}
 	if c.lastProbe.After(now.Add(-5*time.Second)) && !haveRecentPMP {
 		c.mu.Unlock()
+		// fallback to UPnP portmapping
+		if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
+			return mapping, nil
+		}
 		return netaddr.IPPort{}, NoMappingError{ErrNoPortMappingServices}
 	}
 
@@ -381,6 +412,10 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			if ctx.Err() == context.Canceled {
 				return netaddr.IPPort{}, err
 			}
+			// fallback to UPnP portmapping
+			if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
+				return mapping, nil
+			}
 			return netaddr.IPPort{}, NoMappingError{ErrNoPortMappingServices}
 		}
 		srcu := srci.(*net.UDPAddr)
@@ -413,7 +448,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPor
 		if m.externalValid() {
 			c.mu.Lock()
 			defer c.mu.Unlock()
-			c.pmpMapping = m
+			c.mapping = m
 			return m.external, nil
 		}
 	}
@@ -510,7 +545,7 @@ type ProbeResult struct {
 func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return res, ErrGatewayNotFound
+		return res, ErrGatewayRange
 	}
 	defer func() {
 		if err == nil {
@@ -571,9 +606,15 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 		switch port {
 		case upnpPort:
 			if mem.Contains(mem.B(buf[:n]), mem.S(":InternetGatewayDevice:")) {
+				meta, err := parseUPnPDiscoResponse(buf[:n])
+				if err != nil {
+					c.logf("unrecognized UPnP discovery response; ignoring")
+				}
+				// log.Printf("UPnP reply %+v, %q", meta, buf[:n])
 				res.UPnP = true
 				c.mu.Lock()
 				c.uPnPSawTime = time.Now()
+				c.uPnPMeta = meta
 				c.mu.Unlock()
 			}
 		case pcpPort: // same as pmpPort
@@ -687,6 +728,8 @@ func parsePCPResponse(b []byte) (res pcpResponse, ok bool) {
 	return res, true
 }
 
+var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
 const (
 	upnpPort = 1900
 )
@@ -696,5 +739,3 @@ var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
 	"ST: ssdp:all\r\n" +
 	"MAN: \"ssdp:discover\"\r\n" +
 	"MX: 2\r\n\r\n")
-
-var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"

net/portmapper/portmapper_test.go

@@ -17,6 +17,7 @@ func TestCreateOrGetMapping(t *testing.T) {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
 	c := NewClient(t.Logf, nil)
+	defer c.Close()
 	c.SetLocalPort(1234)
 	for i := 0; i < 2; i++ {
 		if i > 0 {
@@ -32,12 +33,13 @@ func TestClientProbe(t *testing.T) {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
 	c := NewClient(t.Logf, nil)
-	for i := 0; i < 2; i++ {
+	defer c.Close()
+	for i := 0; i < 3; i++ {
 		if i > 0 {
 			time.Sleep(100 * time.Millisecond)
 		}
 		res, err := c.Probe(context.Background())
-		t.Logf("Got: %+v, %v", res, err)
+		t.Logf("Got(t=%dms): %+v, %v", i*100, res, err)
 	}
 }
 
@@ -46,6 +48,7 @@ func TestClientProbeThenMap(t *testing.T) {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
 	c := NewClient(t.Logf, nil)
+	defer c.Close()
 	c.SetLocalPort(1234)
 	res, err := c.Probe(context.Background())
 	t.Logf("Probe: %+v, %v", res, err)

net/portmapper/upnp.go

@@ -0,0 +1,313 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/tailscale/goupnp"
+	"github.com/tailscale/goupnp/dcps/internetgateway2"
+	"inet.af/netaddr"
+	"tailscale.com/control/controlknobs"
+	"tailscale.com/net/netns"
+)
+
+// VerboseLogs controls verbose debug logging.
+// It exists for use by "tailscaled debug --portmap".
+var VerboseLogs bool
+
+// References:
+//
+// WANIP Connection v2: http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v2-Service.pdf
+
+// upnpMapping is a port mapping over the upnp protocol. After being created it is immutable,
+// but the client field may be shared across mapping instances.
+type upnpMapping struct {
+	gw         netaddr.IP
+	external   netaddr.IPPort
+	internal   netaddr.IPPort
+	goodUntil  time.Time
+	renewAfter time.Time
+
+	// client is a connection to a upnp device, and may be reused across different UPnP mappings.
+	client upnpClient
+}
+
+func (u *upnpMapping) GoodUntil() time.Time     { return u.goodUntil }
+func (u *upnpMapping) RenewAfter() time.Time    { return u.renewAfter }
+func (u *upnpMapping) External() netaddr.IPPort { return u.external }
+func (u *upnpMapping) Release(ctx context.Context) {
+	u.client.DeletePortMapping(ctx, "", u.external.Port(), "udp")
+}
+
+// upnpClient is an interface over the multiple different clients exported by goupnp,
+// exposing the functions we need for portmapping. Those clients are auto-generated from XML-specs,
+// which is why they're not very idiomatic.
+type upnpClient interface {
+	AddPortMapping(
+		ctx context.Context,
+
+		// remoteHost is the remote device sending packets to this device, in the format of x.x.x.x.
+		// The empty string, "", means any host out on the internet can send packets in.
+		remoteHost string,
+
+		// externalPort is the exposed port of this port mapping. Visible during NAT operations.
+		// 0 will let the router select the port, but there is an additional call,
+		// `AddAnyPortMapping`, which is available on 1 of the 3 possible protocols,
+		// which should be used if available. See `addAnyPortMapping` below, which calls this if
+		// `AddAnyPortMapping` is not supported.
+		externalPort uint16,
+
+		// protocol is whether this is over TCP or UDP. Either "tcp" or "udp".
+		protocol string,
+
+		// internalPort is the port that the gateway device forwards the traffic to.
+		internalPort uint16,
+		// internalClient is the IP address that packets will be forwarded to for this mapping.
+		// Internal client is of the form "x.x.x.x".
+		internalClient string,
+
+		// enabled is whether this portmapping should be enabled or disabled.
+		enabled bool,
+		// portMappingDescription is a user-readable description of this portmapping.
+		portMappingDescription string,
+		// leaseDurationSec is the duration of this portmapping. The value of this argument must be
+		// greater than 0. From the spec, it appears if it is set to 0, it will switch to using
+		// 604800 seconds, but not sure why this is desired. The recommended time is 3600 seconds.
+		leaseDurationSec uint32,
+	) error
+
+	DeletePortMapping(ctx context.Context, remoteHost string, externalPort uint16, protocol string) error
+	GetExternalIPAddress(ctx context.Context) (externalIPAddress string, err error)
+}
+
+// tsPortMappingDesc gets sent to UPnP clients as a human-readable label for the portmapping.
+// It is not used for anything other than labelling.
+const tsPortMappingDesc = "tailscale-portmap"
+
+// addAnyPortMapping abstracts over different UPnP client connections, calling the available
+// AddAnyPortMapping call if available for WAN IP connection v2, otherwise defaulting to the old
+// behavior of calling AddPortMapping with port = 0 to specify a wildcard port.
+// It returns the new external port (which may not be identical to the external port specified),
+// or an error.
+//
+// TODO(bradfitz): also returned the actual lease duration obtained. and check it regularly.
+func addAnyPortMapping(
+	ctx context.Context,
+	upnp upnpClient,
+	externalPort uint16,
+	internalPort uint16,
+	internalClient string,
+	leaseDuration time.Duration,
+) (newPort uint16, err error) {
+	if upnp, ok := upnp.(*internetgateway2.WANIPConnection2); ok {
+		return upnp.AddAnyPortMapping(
+			ctx,
+			"",
+			externalPort,
+			"udp",
+			internalPort,
+			internalClient,
+			true,
+			tsPortMappingDesc,
+			uint32(leaseDuration.Seconds()),
+		)
+	}
+	for externalPort == 0 {
+		externalPort = uint16(rand.Intn(65535))
+	}
+	err = upnp.AddPortMapping(
+		ctx,
+		"",
+		externalPort,
+		"udp",
+		internalPort,
+		internalClient,
+		true,
+		tsPortMappingDesc,
+		uint32(leaseDuration.Seconds()),
+	)
+	return externalPort, err
+}
+
+// getUPnPClient gets a client for interfacing with UPnP, ignoring the underlying protocol for
+// now.
+// Adapted from https://github.com/huin/goupnp/blob/master/GUIDE.md.
+//
+// The gw is the detected gateway.
+//
+// The meta is the most recently parsed UDP discovery packet response
+// from the Internet Gateway Device.
+//
+// The provided ctx is not retained in the returned upnpClient, but
+// its associated HTTP client is (if set via goupnp.WithHTTPClient).
+func getUPnPClient(ctx context.Context, gw netaddr.IP, meta uPnPDiscoResponse) (upnpClient, error) {
+	if controlknobs.DisableUPnP() {
+		return nil, nil
+	}
+
+	if meta.Location == "" {
+		return nil, nil
+	}
+
+	if VerboseLogs {
+		log.Printf("fetching %v", meta.Location)
+	}
+	u, err := url.Parse(meta.Location)
+	if err != nil {
+		return nil, err
+	}
+
+	ipp, err := netaddr.ParseIPPort(u.Host)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected host %q in %q", u.Host, meta.Location)
+	}
+	if ipp.IP() != gw {
+		return nil, fmt.Errorf("UPnP discovered root %q does not match gateway IP %v; ignoring UPnP",
+			meta.Location, gw)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+	defer cancel()
+
+	// This part does a network fetch.
+	root, err := goupnp.DeviceByURL(ctx, u)
+	if err != nil {
+		return nil, err
+	}
+
+	// These parts don't do a network fetch.
+	// Pick the best service type available.
+	if cc, _ := internetgateway2.NewWANIPConnection2ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANIPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANPPPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	return nil, nil
+}
+
+func (c *Client) upnpHTTPClientLocked() *http.Client {
+	if c.uPnPHTTPClient == nil {
+		c.uPnPHTTPClient = &http.Client{
+			Transport: &http.Transport{
+				DialContext:     netns.NewDialer().DialContext,
+				IdleConnTimeout: 2 * time.Second, // LAN is cheap
+			},
+		}
+	}
+	return c.uPnPHTTPClient
+}
+
+// getUPnPPortMapping attempts to create a port-mapping over the UPnP protocol. On success,
+// it will return the externally exposed IP and port. Otherwise, it will return a zeroed IP and
+// port and an error.
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	if controlknobs.DisableUPnP() {
+		return netaddr.IPPort{}, false
+	}
+	now := time.Now()
+	upnp := &upnpMapping{
+		gw:       gw,
+		internal: internal,
+	}
+
+	var client upnpClient
+	var err error
+	c.mu.Lock()
+	oldMapping, ok := c.mapping.(*upnpMapping)
+	meta := c.uPnPMeta
+	httpClient := c.upnpHTTPClientLocked()
+	c.mu.Unlock()
+	if ok && oldMapping != nil {
+		client = oldMapping.client
+	} else {
+		ctx := goupnp.WithHTTPClient(ctx, httpClient)
+		client, err = getUPnPClient(ctx, gw, meta)
+		if VerboseLogs {
+			log.Printf("getUPnPClient: %T, %v", client, err)
+		}
+		if err != nil {
+			return netaddr.IPPort{}, false
+		}
+	}
+	if client == nil {
+		return netaddr.IPPort{}, false
+	}
+
+	var newPort uint16
+	newPort, err = addAnyPortMapping(
+		ctx,
+		client,
+		prevPort,
+		internal.Port(),
+		internal.IP().String(),
+		time.Second*pmpMapLifetimeSec,
+	)
+	if VerboseLogs {
+		log.Printf("addAnyPortMapping: %v, %v", newPort, err)
+	}
+	if err != nil {
+		return netaddr.IPPort{}, false
+	}
+	// TODO cache this ip somewhere?
+	extIP, err := client.GetExternalIPAddress(ctx)
+	if VerboseLogs {
+		log.Printf("client.GetExternalIPAddress: %v, %v", extIP, err)
+	}
+	if err != nil {
+		// TODO this doesn't seem right
+		return netaddr.IPPort{}, false
+	}
+	externalIP, err := netaddr.ParseIP(extIP)
+	if err != nil {
+		return netaddr.IPPort{}, false
+	}
+
+	upnp.external = netaddr.IPPortFrom(externalIP, newPort)
+	d := time.Duration(pmpMapLifetimeSec) * time.Second
+	upnp.goodUntil = now.Add(d)
+	upnp.renewAfter = now.Add(d / 2)
+	upnp.client = client
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.mapping = upnp
+	c.localPort = newPort
+	return upnp.external, true
+}
+
+type uPnPDiscoResponse struct {
+	Location string
+}
+
+// parseUPnPDiscoResponse parses a UPnP HTTP-over-UDP discovery response.
+func parseUPnPDiscoResponse(body []byte) (uPnPDiscoResponse, error) {
+	var r uPnPDiscoResponse
+	res, err := http.ReadResponse(bufio.NewReaderSize(bytes.NewReader(body), 128), nil)
+	if err != nil {
+		return r, err
+	}
+	r.Location = res.Header.Get("Location")
+	return r, nil
+}

net/portmapper/upnp_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package portmapper
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"inet.af/netaddr"
+)
+
+// Google Wifi
+const (
+	googleWifiUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nUSN: uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece::urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nEXT:\r\nSERVER: Linux/5.4.0-1034-gcp UPnP/1.1 MiniUPnPd/1.9\r\nLOCATION: http://192.168.86.1:5000/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1\r\nBOOTID.UPNP.ORG: 1\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	googleWifiRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0"><specVersion><major>1</major><minor>0</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:2</deviceType><friendlyName>OnHub</friendlyName><manufacturer>Google</manufacturer><manufacturerURL>http://google.com/</manufacturerURL><modelDescription>Wireless Router</modelDescription><modelName>OnHub</modelName><modelNumber>1</modelNumber><modelURL>https://on.google.com/hub/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:Layer3Forwarding1</serviceId><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL><SCPDURL>/L3F.xml</SCPDURL></service><service><serviceType>urn:schemas-upnp-org:service:DeviceProtection:1</serviceType><serviceId>urn:upnp-org:serviceId:DeviceProtection1</serviceId><controlURL>/ctl/DP</controlURL><eventSubURL>/evt/DP</eventSubURL><SCPDURL>/DP.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:2</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ecf</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL><SCPDURL>/WANCfg.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:2</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ec0</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:2</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL><SCPDURL>/WANIPCn.xml</SCPDURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>http://testwifi.here/</presentationURL></device></root>`
+)
+
+// pfSense 2.5.0-RELEASE / FreeBSD 12.2-STABLE
+const (
+	pfSenseUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nUSN: uuid:bee7052b-49e8-3597-b545-55a1e38ac11::urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nEXT:\r\nSERVER: FreeBSD/12.2-STABLE UPnP/1.1 MiniUPnPd/2.2.1\r\nLOCATION: http://192.168.1.1:2189/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1627958564\r\nBOOTID.UPNP.ORG: 1627958564\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	pfSenseRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0" configId="1337"><specVersion><major>1</major><minor>1</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:1</deviceType><friendlyName>FreeBSD router</friendlyName><manufacturer>FreeBSD</manufacturer><manufacturerURL>http://www.freebsd.org/</manufacturerURL><modelDescription>FreeBSD router</modelDescription><modelName>FreeBSD router</modelName><modelNumber>2.5.0-RELEASE</modelNumber><modelURL>http://www.freebsd.org/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac11</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:L3Forwarding1</serviceId><SCPDURL>/L3F.xml</SCPDURL><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:1</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac12</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><SCPDURL>/WANCfg.xml</SCPDURL><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:1</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac13</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:1</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><SCPDURL>/WANIPCn.xml</SCPDURL><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>https://192.168.1.1/</presentationURL></device></root>`
+)
+
+func TestParseUPnPDiscoResponse(t *testing.T) {
+	tests := []struct {
+		name    string
+		headers string
+		want    uPnPDiscoResponse
+	}{
+		{"google", googleWifiUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.86.1:5000/rootDesc.xml",
+		}},
+		{"pfsense", pfSenseUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.1.1:2189/rootDesc.xml",
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseUPnPDiscoResponse([]byte(tt.headers))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unexpected result:\n got: %+v\nwant: %+v\n", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetUPnPClient(t *testing.T) {
+	tests := []struct {
+		name    string
+		xmlBody string
+		want    string
+	}{
+		{"google", googleWifiRootDescXML, "*internetgateway2.WANIPConnection2"},
+		{"pfsense", pfSenseRootDescXML, "*internetgateway2.WANIPConnection1"},
+		// TODO(bradfitz): find a PPP one in the wild
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.RequestURI == "/rootDesc.xml" {
+					io.WriteString(w, tt.xmlBody)
+					return
+				}
+				http.NotFound(w, r)
+			}))
+			defer ts.Close()
+			gw, _ := netaddr.FromStdIP(ts.Listener.Addr().(*net.TCPAddr).IP)
+			c, err := getUPnPClient(context.Background(), gw, uPnPDiscoResponse{
+				Location: ts.URL + "/rootDesc.xml",
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := fmt.Sprintf("%T", c)
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

net/speedtest/speedtest.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package speedtest contains both server and client code for
+// running speedtests between tailscale nodes.
+package speedtest
+
+import (
+	"time"
+)
+
+const (
+	blockSize       = 32000                 // size of the block of data to send
+	MinDuration     = 5 * time.Second       // minimum duration for a test
+	DefaultDuration = MinDuration           // default duration for a test
+	MaxDuration     = 30 * time.Second      // maximum duration for a test
+	version         = 1                     // value used when comparing client and server versions
+	increment       = time.Second           // increment to display results for, in seconds
+	minInterval     = 10 * time.Millisecond // minimum interval length for a result to be included
+	DefaultPort     = 20333
+)
+
+// config is the initial message sent to the server, that contains information on how to
+// conduct the test.
+type config struct {
+	Version      int           `json:"version"`
+	TestDuration time.Duration `json:"time"`
+	Direction    Direction     `json:"direction"`
+}
+
+// configResponse is the response to the testConfig message. If the server has an
+// error with the config, the Error variable will hold that error value.
+type configResponse struct {
+	Error string `json:"error,omitempty"`
+}
+
+// This represents the Result of a speedtest within a specific interval
+type Result struct {
+	Bytes         int           // number of bytes sent/received during the interval
+	IntervalStart time.Duration // duration between the start of the interval and the start of the test
+	IntervalEnd   time.Duration // duration between the end of the interval and the start of the test
+	Total         bool          // if true, this result struct represents the entire test, rather than a segment of the test
+}
+
+func (r Result) MBitsPerSecond() float64 {
+	return r.MegaBits() / (r.IntervalEnd - r.IntervalStart).Seconds()
+}
+
+func (r Result) MegaBytes() float64 {
+	return float64(r.Bytes) / 1000000.0
+}
+
+func (r Result) MegaBits() float64 {
+	return r.MegaBytes() * 8.0
+}
+
+func (r Result) Interval() time.Duration {
+	return r.IntervalEnd - r.IntervalStart
+}
+
+type Direction int
+
+const (
+	Download Direction = iota
+	Upload
+)
+
+func (d Direction) String() string {
+	switch d {
+	case Upload:
+		return "upload"
+	case Download:
+		return "download"
+	default:
+		return ""
+	}
+}
+
+func (d *Direction) Reverse() {
+	switch *d {
+	case Upload:
+		*d = Download
+	case Download:
+		*d = Upload
+	default:
+	}
+}

net/speedtest/speedtest_client.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"encoding/json"
+	"errors"
+	"net"
+	"time"
+)
+
+// RunClient dials the given address and starts a speedtest.
+// It returns any errors that come up in the tests.
+// If there are no errors in the test, it returns a slice of results.
+func RunClient(direction Direction, duration time.Duration, host string) ([]Result, error) {
+	conn, err := net.Dial("tcp", host)
+	if err != nil {
+		return nil, err
+	}
+
+	conf := config{TestDuration: duration, Version: version, Direction: direction}
+
+	defer conn.Close()
+	encoder := json.NewEncoder(conn)
+
+	if err = encoder.Encode(conf); err != nil {
+		return nil, err
+	}
+
+	var response configResponse
+	decoder := json.NewDecoder(conn)
+	if err = decoder.Decode(&response); err != nil {
+		return nil, err
+	}
+	if response.Error != "" {
+		return nil, errors.New(response.Error)
+	}
+
+	return doTest(conn, conf)
+}

net/speedtest/speedtest_server.go

@@ -0,0 +1,158 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+// Serve starts up the server on a given host and port pair. It starts to listen for
+// connections and handles each one in a goroutine. Because it runs in an infinite loop,
+// this function only returns if any of the speedtests return with errors, or if the
+// listener is closed.
+func Serve(l net.Listener) error {
+	for {
+		conn, err := l.Accept()
+		if errors.Is(err, net.ErrClosed) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		err = handleConnection(conn)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// handleConnection handles the initial exchange between the server and the client.
+// It reads the testconfig message into a config struct. If any errors occur with
+// the testconfig (specifically, if there is a version mismatch), it will return those
+// errors to the client with a configResponse. After the exchange, it will start
+// the speed test.
+func handleConnection(conn net.Conn) error {
+	defer conn.Close()
+	var conf config
+
+	decoder := json.NewDecoder(conn)
+	err := decoder.Decode(&conf)
+	encoder := json.NewEncoder(conn)
+
+	// Both return and encode errors that occurred before the test started.
+	if err != nil {
+		encoder.Encode(configResponse{Error: err.Error()})
+		return err
+	}
+
+	// The server should always be doing the opposite of what the client is doing.
+	conf.Direction.Reverse()
+
+	if conf.Version != version {
+		err = fmt.Errorf("version mismatch! Server is version %d, client is version %d", version, conf.Version)
+		encoder.Encode(configResponse{Error: err.Error()})
+		return err
+	}
+
+	// Start the test
+	encoder.Encode(configResponse{})
+	_, err = doTest(conn, conf)
+	return err
+}
+
+// TODO include code to detect whether the code is direct vs DERP
+
+// doTest contains the code to run both the upload and download speedtest.
+// the direction value in the config parameter determines which test to run.
+func doTest(conn net.Conn, conf config) ([]Result, error) {
+	bufferData := make([]byte, blockSize)
+
+	intervalBytes := 0
+	totalBytes := 0
+
+	var currentTime time.Time
+	var results []Result
+
+	startTime := time.Now()
+	lastCalculated := startTime
+
+	if conf.Direction == Download {
+		conn.SetReadDeadline(time.Now().Add(conf.TestDuration).Add(5 * time.Second))
+	} else {
+		_, err := rand.Read(bufferData)
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+SpeedTestLoop:
+	for {
+		var n int
+		var err error
+
+		if conf.Direction == Download {
+			n, err = io.ReadFull(conn, bufferData)
+			switch err {
+			case io.EOF, io.ErrUnexpectedEOF:
+				break SpeedTestLoop
+			case nil:
+				// successful read
+			default:
+				return nil, fmt.Errorf("unexpected error has occured: %w", err)
+			}
+		} else {
+			// Need to change the data a little bit, to avoid any compression.
+			for i := range bufferData {
+				bufferData[i]++
+			}
+			n, err = conn.Write(bufferData)
+			if err != nil {
+				// If the write failed, there is most likely something wrong with the connection.
+				return nil, fmt.Errorf("upload failed: %w", err)
+			}
+		}
+		currentTime = time.Now()
+		intervalBytes += n
+
+		// checks if the current time is more or equal to the lastCalculated time plus the increment
+		if currentTime.After(lastCalculated.Add(increment)) {
+			intervalStart := lastCalculated.Sub(startTime)
+			intervalEnd := currentTime.Sub(startTime)
+			if (intervalEnd - intervalStart) > minInterval {
+				results = append(results, Result{Bytes: intervalBytes, IntervalStart: intervalStart, IntervalEnd: intervalEnd, Total: false})
+			}
+			lastCalculated = currentTime
+			totalBytes += intervalBytes
+			intervalBytes = 0
+		}
+
+		if conf.Direction == Upload && time.Since(startTime) > conf.TestDuration {
+			break SpeedTestLoop
+		}
+	}
+
+	// get last segment
+	intervalStart := lastCalculated.Sub(startTime)
+	intervalEnd := currentTime.Sub(startTime)
+	if (intervalEnd - intervalStart) > minInterval {
+		results = append(results, Result{Bytes: intervalBytes, IntervalStart: intervalStart, IntervalEnd: intervalEnd, Total: false})
+	}
+
+	// get total
+	totalBytes += intervalBytes
+	intervalEnd = currentTime.Sub(startTime)
+	if intervalEnd > minInterval {
+		results = append(results, Result{Bytes: totalBytes, IntervalStart: 0, IntervalEnd: intervalEnd, Total: true})
+	}
+
+	return results, nil
+}

net/speedtest/speedtest_test.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"net"
+	"testing"
+)
+
+func TestDownload(t *testing.T) {
+	// start a listener and find the port where the server will be listening.
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { l.Close() })
+
+	serverIP := l.Addr().String()
+	t.Log("server IP found:", serverIP)
+
+	type state struct {
+		err error
+	}
+	displayResult := func(t *testing.T, r Result) {
+		t.Helper()
+		t.Logf("{ Megabytes: %.2f, Start: %.1f, End: %.1f, Total: %t }", r.MegaBytes(), r.IntervalStart.Seconds(), r.IntervalEnd.Seconds(), r.Total)
+	}
+	stateChan := make(chan state, 1)
+
+	go func() {
+		err := Serve(l)
+		stateChan <- state{err: err}
+	}()
+
+	// ensure that the test returns an appropriate number of Result structs
+	expectedLen := int(DefaultDuration.Seconds()) + 1
+
+	t.Run("download test", func(t *testing.T) {
+		// conduct a download test
+		results, err := RunClient(Download, DefaultDuration, serverIP)
+
+		if err != nil {
+			t.Fatal("download test failed:", err)
+		}
+
+		if len(results) < expectedLen {
+			t.Fatalf("download results: expected length: %d, actual length: %d", expectedLen, len(results))
+		}
+
+		for _, result := range results {
+			displayResult(t, result)
+		}
+	})
+
+	t.Run("upload test", func(t *testing.T) {
+		// conduct an upload test
+		results, err := RunClient(Upload, DefaultDuration, serverIP)
+
+		if err != nil {
+			t.Fatal("upload test failed:", err)
+		}
+
+		if len(results) < expectedLen {
+			t.Fatalf("upload results: expected length: %d, actual length: %d", expectedLen, len(results))
+		}
+
+		for _, result := range results {
+			displayResult(t, result)
+		}
+	})
+
+	// causes the server goroutine to finish
+	l.Close()
+
+	testState := <-stateChan
+	if testState.err != nil {
+		t.Error("server error:", err)
+	}
+}

net/tsaddr/tsaddr.go

@@ -105,11 +105,6 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	return netaddr.IPFrom16(ret)
 }
 
-func IsULA(ip netaddr.IP) bool {
-	ulaRange.Do(func() { mustPrefix(&ulaRange.v, "fc00::/7") })
-	return ulaRange.v.Contains(ip)
-}
-
 func mustPrefix(v *netaddr.IPPrefix, prefix string) {
 	var err error
 	*v, err = netaddr.ParseIPPrefix(prefix)

net/tsaddr/tsaddr_test.go

@@ -43,28 +43,6 @@ func TestCGNATRange(t *testing.T) {
 	}
 }
 
-func TestIsUla(t *testing.T) {
-	tests := []struct {
-		name string
-		ip   string
-		want bool
-	}{
-		{"first ULA", "fc00::1", true},
-		{"not ULA", "fb00::1", false},
-		{"Tailscale", "fd7a:115c:a1e0::1", true},
-		{"Cloud Run", "fddf:3978:feb1:d745::1", true},
-		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
-		{"Link Local", "fe80::1", false},
-		{"Global", "2602::1", false},
-	}
-
-	for _, test := range tests {
-		if got := IsULA(netaddr.MustParseIP(test.ip)); got != test.want {
-			t.Errorf("IsULA(%s) = %v, want %v", test.name, got, test.want)
-		}
-	}
-}
-
 func TestNewContainsIPFunc(t *testing.T) {
 	f := NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.0.0.0/8")})
 	if f(netaddr.MustParseIP("8.8.8.8")) {

net/tstun/wrap.go

@@ -18,6 +18,7 @@ import (
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -64,7 +65,7 @@ type Wrapper struct {
 
 	closeOnce sync.Once
 
-	lastActivityAtomic int64 // unix seconds of last send or receive
+	lastActivityAtomic mono.Time // time of last send or receive
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 
@@ -153,9 +154,10 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		// a goroutine should not block when setting it, even with no listeners.
 		bufferConsumed: make(chan struct{}, 1),
 		closed:         make(chan struct{}),
-		outbound:       make(chan tunReadResult),
-		eventsUpDown:   make(chan tun.Event),
-		eventsOther:    make(chan tun.Event),
+		// outbound can be unbuffered; the buffer is an optimization.
+		outbound:     make(chan tunReadResult, 1),
+		eventsUpDown: make(chan tun.Event),
+		eventsOther:  make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
@@ -164,6 +166,7 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 	go tun.pumpEvents()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
+	tun.noteActivity()
 
 	return tun
 }
@@ -363,16 +366,15 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 
 // noteActivity records that there was a read or write at the current time.
 func (t *Wrapper) noteActivity() {
-	atomic.StoreInt64(&t.lastActivityAtomic, time.Now().Unix())
+	t.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 // IdleDuration reports how long it's been since the last read or write to this device.
 //
-// Its value is only accurate to roughly second granularity.
-// If there's never been activity, the duration is since 1970.
+// Its value should only be presumed accurate to roughly 10ms granularity.
+// If there's never been activity, the duration is since the wrapper was created.
 func (t *Wrapper) IdleDuration() time.Duration {
-	sec := atomic.LoadInt64(&t.lastActivityAtomic)
-	return time.Since(time.Unix(sec, 0))
+	return mono.Since(t.lastActivityAtomic.LoadAtomic())
 }
 
 func (t *Wrapper) Read(buf []byte, offset int) (int, error) {

net/tstun/wrap_test.go

@@ -10,13 +10,13 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"unsafe"
 
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -335,9 +335,9 @@ func TestFilter(t *testing.T) {
 				// data was actually filtered.
 				// If it stays zero, nothing made it through
 				// to the wrapped TUN.
-				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
+				tun.lastActivityAtomic.StoreAtomic(0)
 				_, err = tun.Write(tt.data, 0)
-				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
+				filtered = tun.lastActivityAtomic.LoadAtomic() == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)
@@ -416,7 +416,7 @@ func TestAtomic64Alignment(t *testing.T) {
 	}
 
 	c := new(Wrapper)
-	atomic.StoreInt64(&c.lastActivityAtomic, 123)
+	c.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 func TestPeerAPIBypass(t *testing.T) {

safesocket/safesocket_darwin.go

@@ -53,15 +53,16 @@ func localTCPPortAndTokenDarwin() (port int, token string, err error) {
 	// The current process is running outside the sandbox, so use
 	// lsof to find the IPNExtension:
 
-	out, err := exec.Command("lsof",
+	cmd := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
 		fmt.Sprintf("-u%d", os.Getuid()), // process of same user only
 		"-c", "IPNExtension",             // starting with IPNExtension
 		"-F", // machine-readable output
-	).Output()
+	)
+	out, err := cmd.Output()
 	if err != nil {
-		return 0, "", fmt.Errorf("failed to run lsof looking for IPNExtension: %w", err)
+		return 0, "", fmt.Errorf("failed to run '%s' looking for IPNExtension: %w", cmd, err)
 	}
 	bs := bufio.NewScanner(bytes.NewReader(out))
 	subStr := []byte(".tailscale.ipn.macos/sameuserproof-")

safesocket/unixsocket.go

@@ -28,16 +28,15 @@ func connect(path string, port uint16) (net.Conn, error) {
 	pipe, err := net.Dial("unix", path)
 	if err != nil {
 		if runtime.GOOS == "darwin" {
-			extConn, err := connectMacOSAppSandbox()
-			if err != nil {
-				log.Printf("safesocket: failed to connect to Tailscale IPNExtension: %v", err)
-			} else {
-				return extConn, nil
+			extConn, extErr := connectMacOSAppSandbox()
+			if extErr != nil {
+				return nil, fmt.Errorf("safesocket: failed to connect to %v: %v; failed to connect to Tailscale IPNExtension: %v", path, err, extErr)
 			}
+			return extConn, nil
 		}
 		return nil, err
 	}
-	return pipe, err
+	return pipe, nil
 }
 
 // TODO(apenwarr): handle magic cookie auth

scripts/check_license_headers.sh

@@ -38,9 +38,6 @@ for file in $(find $1 -name '*.go' -not -path '*/.git/*'); do
         $1/wgengine/router/ifconfig_windows.go)
             # WireGuard copyright.
         ;;
-		*_string.go)
-			# Generated file from go:generate stringer
-		;;
         *)
             header="$(head -3 $file)"
             if ! check_file "$header"; then

syncs/syncs.go

@@ -79,6 +79,16 @@ func (b *AtomicBool) Set(v bool) {
 	atomic.StoreInt32((*int32)(b), n)
 }
 
+// Swap sets b to v and reports whether it changed.
+func (b *AtomicBool) Swap(v bool) (changed bool) {
+	var n int32
+	if v {
+		n = 1
+	}
+	old := atomic.SwapInt32((*int32)(b), n)
+	return old != n
+}
+
 func (b *AtomicBool) Get() bool {
 	return atomic.LoadInt32((*int32)(b)) != 0
 }

tailcfg/tailcfg.go

@@ -164,6 +164,12 @@ type Node struct {
 	Hostinfo   Hostinfo
 	Created    time.Time
 
+	// PrimaryRoutes are the routes from AllowedIPs that this node
+	// is currently the primary subnet router for, as determined
+	// by the control plane. It does not include the self address
+	// values from Addresses that are in AllowedIPs.
+	PrimaryRoutes []netaddr.IPPrefix `json:",omitempty"`
+
 	// LastSeen is when the node was last online. It is not
 	// updated when Online is true. It is nil if the current
 	// node doesn't have permission to know, or the node
@@ -1060,6 +1066,13 @@ type Debug struct {
 	// :0 to get a random local port, ignoring any configured
 	// fixed port.
 	RandomizeClientPort bool `json:",omitempty"`
+
+	/// DisableUPnP is whether the client will attempt to perform a UPnP portmapping.
+	// By default, we want to enable it to see if it works on more clients.
+	//
+	// If UPnP catastrophically fails for people, this should be set to True to kill
+	// new attempts at UPnP connections.
+	DisableUPnP opt.Bool `json:",omitempty"`
 }
 
 func (k MachineKey) String() string                   { return fmt.Sprintf("mkey:%x", k[:]) }
@@ -1135,6 +1148,7 @@ func (n *Node) Equal(n2 *Node) bool {
 		eqBoolPtr(n.Online, n2.Online) &&
 		eqCIDRs(n.Addresses, n2.Addresses) &&
 		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqCIDRs(n.PrimaryRoutes, n2.PrimaryRoutes) &&
 		eqStrings(n.Endpoints, n2.Endpoints) &&
 		n.DERP == n2.DERP &&
 		n.Hostinfo.Equal(&n2.Hostinfo) &&

tailcfg/tailcfg_clone.go

@@ -49,6 +49,7 @@ func (src *Node) Clone() *Node {
 	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
 	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
 	dst.Hostinfo = *src.Hostinfo.Clone()
+	dst.PrimaryRoutes = append(src.PrimaryRoutes[:0:0], src.PrimaryRoutes...)
 	if dst.LastSeen != nil {
 		dst.LastSeen = new(time.Time)
 		*dst.LastSeen = *src.LastSeen
@@ -79,6 +80,7 @@ var _NodeNeedsRegeneration = Node(struct {
 	DERP                    string
 	Hostinfo                Hostinfo
 	Created                 time.Time
+	PrimaryRoutes           []netaddr.IPPrefix
 	LastSeen                *time.Time
 	Online                  *bool
 	KeepAlive               bool

tailcfg/tailcfg_test.go

@@ -194,7 +194,8 @@ func TestNodeEqual(t *testing.T) {
 		"ID", "StableID", "Name", "User", "Sharer",
 		"Key", "KeyExpiry", "Machine", "DiscoKey",
 		"Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo",
-		"Created", "LastSeen", "Online", "KeepAlive", "MachineAuthorized",
+		"Created", "PrimaryRoutes",
+		"LastSeen", "Online", "KeepAlive", "MachineAuthorized",
 		"Capabilities",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
 	}

tempfork/wireguard-windows/firewall/zsyscall_windows.go

@@ -1,5 +1,3 @@
-// +build windows
-
 // Code generated by 'go generate'; DO NOT EDIT.
 
 package firewall

tstest/integration/gen_deps.go

@@ -12,16 +12,25 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"os/exec"
 )
 
 func main() {
+	for _, goos := range []string{"windows", "linux", "darwin", "freebsd", "openbsd"} {
+		generate(goos)
+	}
+}
+
+func generate(goos string) {
 	var x struct {
 		Imports []string
 	}
-	j, err := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled").Output()
+	cmd := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled")
+	cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH=amd64")
+	j, err := cmd.Output()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("GOOS=%s GOARCH=amd64 %s: %v", goos, cmd, err)
 	}
 	if err := json.Unmarshal(j, &x); err != nil {
 		log.Fatal(err)
@@ -46,7 +55,8 @@ import (
 	}
 	fmt.Fprintf(&out, ")\n")
 
-	err = ioutil.WriteFile("tailscaled_deps_test.go", out.Bytes(), 0644)
+	filename := fmt.Sprintf("tailscaled_deps_test_%s.go", goos)
+	err = ioutil.WriteFile(filename, out.Bytes(), 0644)
 	if err != nil {
 		log.Fatal(err)
 	}

tstest/integration/integration.go

@@ -205,6 +205,13 @@ func (lc *LogCatcher) logsString() string {
 	return lc.buf.String()
 }
 
+// Reset clears the buffered logs from memory.
+func (lc *LogCatcher) Reset() {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	lc.buf.Reset()
+}
+
 func (lc *LogCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var body io.Reader = r.Body
 	if r.Header.Get("Content-Encoding") == "zstd" {

tstest/integration/integration_test.go

@@ -42,11 +42,14 @@ import (
 
 var (
 	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
+	verboseTailscale  = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging")
 )
 
 var mainError atomic.Value // of error
 
 func TestMain(m *testing.M) {
+	// Have to disable UPnP which hits the network, otherwise it fails due to HTTP proxy.
+	os.Setenv("TS_DISABLE_UPNP", "true")
 	flag.Parse()
 	v := m.Run()
 	if v != 0 {
@@ -70,29 +73,9 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-
-	n1.AwaitListening(t)
-
-	st := n1.MustStatus(t)
-	t.Logf("Status: %s", st.BackendState)
-
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `Program starting: `
-		if !env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Error(err)
-	}
-
+	n1.AwaitResponding(t)
 	n1.MustUp()
 
-	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
-		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
-		time.Sleep(d)
-	}
-
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
 	n1.AwaitRunning(t)
 
@@ -101,6 +84,40 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+func TestCollectPanic(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n := newTestNode(t, env)
+
+	cmd := exec.Command(n.env.Binaries.Daemon, "--cleanup")
+	cmd.Env = append(os.Environ(),
+		"TS_PLEASE_PANIC=1",
+		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
+	)
+	got, _ := cmd.CombinedOutput() // we expect it to fail, ignore err
+	t.Logf("initial run: %s", got)
+
+	// Now we run it again, and on start, it will upload the logs to logcatcher.
+	cmd = exec.Command(n.env.Binaries.Daemon, "--cleanup")
+	cmd.Env = append(os.Environ(), "TS_LOG_TARGET="+n.env.LogCatcherServer.URL)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("cleanup failed: %v: %q", err, out)
+	}
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `panic`
+		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // test Issue 2321: Start with UpdatePrefs should save prefs to disk
 func TestStateSavedOnStart(t *testing.T) {
 	t.Parallel()
@@ -113,22 +130,7 @@ func TestStateSavedOnStart(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-
-	n1.AwaitListening(t)
-
-	st := n1.MustStatus(t)
-	t.Logf("Status: %s", st.BackendState)
-
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `Program starting: `
-		if !env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Error(err)
-	}
-
+	n1.AwaitResponding(t)
 	n1.MustUp()
 
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
@@ -344,6 +346,76 @@ func TestAddPingRequest(t *testing.T) {
 	t.Error("all ping attempts failed")
 }
 
+// Issue 2434: when "down" (WantRunning false), tailscaled shouldn't
+// be connected to control.
+func TestNoControlConnWhenDown(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+
+	// Come up the first time.
+	n1.MustUp()
+	ip1 := n1.AwaitIP(t)
+	n1.AwaitRunning(t)
+
+	// Then bring it down and stop the daemon.
+	n1.MustDown()
+	d1.MustCleanShutdown(t)
+
+	env.LogCatcher.Reset()
+	d2 := n1.StartDaemon(t)
+	defer d2.Kill()
+	n1.AwaitResponding(t)
+
+	st := n1.MustStatus(t)
+	if got, want := st.BackendState, "Stopped"; got != want {
+		t.Fatalf("after restart, state = %q; want %q", got, want)
+	}
+
+	ip2 := n1.AwaitIP(t)
+	if ip1 != ip2 {
+		t.Errorf("IPs different: %q vs %q", ip1, ip2)
+	}
+
+	// The real test: verify our daemon doesn't have an HTTP request open.:
+	if n := env.Control.InServeMap(); n != 0 {
+		t.Errorf("in serve map = %d; want 0", n)
+	}
+
+	d2.MustCleanShutdown(t)
+}
+
+// Issue 2137: make sure Windows tailscaled works with the CLI alone,
+// without the GUI to kick off a Start.
+func TestOneNodeUpWindowsStyle(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+	n1.upFlagGOOS = "windows"
+
+	d1 := n1.StartDaemonAsIPNGOOS(t, "windows")
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp("--unattended")
+
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+}
+
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
@@ -420,9 +492,10 @@ func (e *testEnv) Close() error {
 type testNode struct {
 	env *testEnv
 
-	dir       string // temp dir for sock & state
-	sockFile  string
-	stateFile string
+	dir        string // temp dir for sock & state
+	sockFile   string
+	stateFile  string
+	upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI
 
 	mu        sync.Mutex
 	onLogLine []func([]byte)
@@ -432,10 +505,14 @@ type testNode struct {
 // The node is not started automatically.
 func newTestNode(t *testing.T, env *testEnv) *testNode {
 	dir := t.TempDir()
+	sockFile := filepath.Join(dir, "tailscale.sock")
+	if len(sockFile) >= 104 {
+		t.Fatalf("sockFile path %q (len %v) is too long, must be < 104", sockFile, len(sockFile))
+	}
 	return &testNode{
 		env:       env,
 		dir:       dir,
-		sockFile:  filepath.Join(dir, "tailscale.sock"),
+		sockFile:  sockFile,
 		stateFile: filepath.Join(dir, "tailscale.state"),
 	}
 }
@@ -460,6 +537,26 @@ func (n *testNode) diskPrefs(t testing.TB) *ipn.Prefs {
 	return p
 }
 
+// AwaitResponding waits for n's tailscaled to be up enough to be
+// responding, but doesn't wait for any particular state.
+func (n *testNode) AwaitResponding(t testing.TB) {
+	t.Helper()
+	n.AwaitListening(t)
+
+	st := n.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `Program starting: `
+		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // addLogLineHook registers a hook f to be called on each tailscaled
 // log line output.
 func (n *testNode) addLogLineHook(f func([]byte)) {
@@ -561,6 +658,10 @@ func (d *Daemon) MustCleanShutdown(t testing.TB) {
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
 func (n *testNode) StartDaemon(t testing.TB) *Daemon {
+	return n.StartDaemonAsIPNGOOS(t, runtime.GOOS)
+}
+
+func (n *testNode) StartDaemonAsIPNGOOS(t testing.TB, ipnGOOS string) *Daemon {
 	cmd := exec.Command(n.env.Binaries.Daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
@@ -571,6 +672,8 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
 		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
 		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
+		"TS_DEBUG_TAILSCALED_IPN_GOOS="+ipnGOOS,
+		"TS_LOGS_DIR="+t.TempDir(),
 	)
 	cmd.Stderr = &nodeOutputParser{n: n}
 	if *verboseTailscaled {
@@ -585,10 +688,15 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 	}
 }
 
-func (n *testNode) MustUp() {
+func (n *testNode) MustUp(extraArgs ...string) {
 	t := n.env.t
-	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
-	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
+	args := []string{
+		"up",
+		"--login-server=" + n.env.ControlServer.URL,
+	}
+	args = append(args, extraArgs...)
+	t.Logf("Running %v ...", args)
+	if err := n.Tailscale(args...).Run(); err != nil {
 		t.Fatalf("up: %v", err)
 	}
 }
@@ -620,7 +728,10 @@ func (n *testNode) AwaitIPs(t testing.TB) []netaddr.IP {
 	t.Helper()
 	var addrs []netaddr.IP
 	if err := tstest.WaitFor(20*time.Second, func() error {
-		out, err := n.Tailscale("ip").Output()
+		cmd := n.Tailscale("ip")
+		cmd.Stdout = nil // in case --verbose-tailscale was set
+		cmd.Stderr = nil // in case --verbose-tailscale was set
+		out, err := cmd.Output()
 		if err != nil {
 			return err
 		}
@@ -652,6 +763,7 @@ func (n *testNode) AwaitIP(t testing.TB) netaddr.IP {
 	return ips[0]
 }
 
+// AwaitRunning waits for n to reach the IPN state "Running".
 func (n *testNode) AwaitRunning(t testing.TB) {
 	t.Helper()
 	if err := tstest.WaitFor(20*time.Second, func() error {
@@ -674,11 +786,22 @@ func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
 	cmd := exec.Command(n.env.Binaries.CLI, "--socket="+n.sockFile)
 	cmd.Args = append(cmd.Args, arg...)
 	cmd.Dir = n.dir
+	cmd.Env = append(os.Environ(),
+		"TS_DEBUG_UP_FLAG_GOOS="+n.upFlagGOOS,
+		"TS_LOGS_DIR="+n.env.t.TempDir(),
+	)
+	if *verboseTailscale {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
 	return cmd
 }
 
 func (n *testNode) Status() (*ipnstate.Status, error) {
-	out, err := n.Tailscale("status", "--json").CombinedOutput()
+	cmd := n.Tailscale("status", "--json")
+	cmd.Stdout = nil // in case --verbose-tailscale was set
+	cmd.Stderr = nil // in case --verbose-tailscale was set
+	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
 	}

tstest/integration/tailscaled_deps_test_darwin.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/exec"
+	_ "os/signal"
+	_ "path/filepath"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -39,6 +39,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_linux.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_windows.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "golang.org/x/sys/windows"
+	_ "golang.org/x/sys/windows/svc"
+	_ "golang.org/x/sys/windows/svc/mgr"
+	_ "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	_ "inet.af/netaddr"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail/backoff"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wf"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/testcontrol/testcontrol.go

@@ -51,6 +51,7 @@ type Server struct {
 	mux         *http.ServeMux
 
 	mu            sync.Mutex
+	inServeMap    int
 	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        wgkey.Key
 	privKey       wgkey.Private
@@ -509,7 +510,22 @@ func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
 	return peersToUpdate
 }
 
+func (s *Server) incrInServeMap(delta int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inServeMap += delta
+}
+
+// InServeMap returns the number of clients currently in a MapRequest HTTP handler.
+func (s *Server) InServeMap() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.inServeMap
+}
+
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	s.incrInServeMap(1)
+	defer s.incrInServeMap(-1)
 	ctx := r.Context()
 
 	req := new(tailcfg.MapRequest)
@@ -636,6 +652,9 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		Domain:          string(user.Domain),
 		CollectServices: "true",
 		PacketFilter:    tailcfg.FilterAllowAll,
+		Debug: &tailcfg.Debug{
+			DisableUPnP: "true",
+		},
 	}
 	for _, p := range s.AllNodes() {
 		if p.StableID != node.StableID {

tstest/integration/vms/distros.go

@@ -2,24 +2,33 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux
-
 package vms
 
+import (
+	_ "embed"
+	"log"
+
+	"github.com/tailscale/hujson"
+)
+
+// go:generate go run ./gen
+
 type Distro struct {
-	name           string // amazon-linux
-	url            string // URL to a qcow2 image
-	sha256sum      string // hex-encoded sha256 sum of contents of URL
-	mem            int    // VM memory in megabytes
-	packageManager string // yum/apt/dnf/zypper
-	initSystem     string // systemd/openrc
+	Name           string // amazon-linux
+	URL            string // URL to a qcow2 image
+	SHA256Sum      string // hex-encoded sha256 sum of contents of URL
+	MemoryMegs     int    // VM memory in megabytes
+	PackageManager string // yum/apt/dnf/zypper
+	InitSystem     string // systemd/openrc
 }
 
 func (d *Distro) InstallPre() string {
-	switch d.packageManager {
+	switch d.PackageManager {
 	case "yum":
 		return ` - [ yum, update, gnupg2 ]
- - [ yum, "-y", install, iptables ]`
+ - [ yum, "-y", install, iptables ]
+ - [ sh, "-c", "printf '\n\nUseDNS no\n\n' | tee -a /etc/ssh/sshd_config" ]
+ - [ systemctl, restart, "sshd.service" ]`
 	case "zypper":
 		return ` - [ zypper, in, "-y", iptables ]`
 
@@ -38,58 +47,15 @@ func (d *Distro) InstallPre() string {
 	return ""
 }
 
-var distros = []Distro{
-	// NOTE(Xe): If you run into issues getting the autoconfig to work, run
-	// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
-	// client with a command like this:
-	//
-	//    $ vncviewer :0
-	//
-	// On NixOS you can get away with something like this:
-	//
-	//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
-	//
-	// Login as root with the password root. Then look in
-	// /var/log/cloud-init-output.log for what you messed up.
+//go:embed distros.hujson
+var distroData string
 
-	// NOTE(Xe): These images are not official images created by the Alpine Linux
-	// cloud team because the cloud team hasn't created any official images yet.
-	// These images were created under the guidance of the cloud team and contain
-	// few notable differences from what they would end up shipping. The Alpine
-	// Linux cloud team probably won't have official images up until a year or so
-	// after this comment is written (2021-06-11), but overall they will be
-	// compatible with these images. These images were created using the setup in
-	// this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
-	// these links.
-	{"alpine-3-13-5", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", 256, "apk", "openrc"},
-	{"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk", "openrc"},
+var Distros []Distro = func() []Distro {
+	var result []Distro
+	err := hujson.Unmarshal([]byte(distroData), &result)
+	if err != nil {
+		log.Fatalf("error decoding distros: %v", err)
+	}
 
-	// NOTE(Xe): All of the following images are official images straight from each
-	// distribution's official documentation.
-	{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum", "systemd"},
-	{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman", "systemd"},
-	{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum", "systemd"},
-	{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf", "systemd"},
-	{"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt", "systemd"},
-	{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt", "systemd"},
-	{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf", "systemd"},
-	{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper", "systemd"},
-	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"},
-	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"},
-	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"},
-	{"oracle-linux-7", "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2", "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb", 512, "yum", "systemd"},
-	{"oracle-linux-8", "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2", "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b", 768, "dnf", "systemd"},
-	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"},
-	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"},
-	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
-	{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"},
-	{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"},
-
-	// NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
-	// used here is actually the URL of the NixOS channel being built from and the
-	// shasum is meaningless. This `channel:name` syntax is documented at [1].
-	//
-	// [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
-	{"nixos-21-05", "channel:nixos-21.05", "lolfakesha", 512, "nix", "systemd"},
-	{"nixos-unstable", "channel:nixos-unstable", "lolfakesha", 512, "nix", "systemd"},
-}
+	return result
+}()

tstest/integration/vms/distros.hujson

@@ -0,0 +1,208 @@
+// NOTE(Xe): If you run into issues getting the autoconfig to work, run
+// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
+// client with a command like this:
+//
+//    $ vncviewer :0
+//
+// On NixOS you can get away with something like this:
+//
+//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
+//
+// Login as root with the password root. Then look in
+// /var/log/cloud-init-output.log for what you messed up.
+[
+	  // NOTE(Xe): These images are not official images created by the Alpine Linux
+	  // cloud team because the cloud team hasn't created any official images yet.
+	  // These images were created under the guidance of the cloud team and contain
+	  // few notable differences from what they would end up shipping. The Alpine
+	  // Linux cloud team probably won't have official images up until a year or so
+	  // after this comment is written (2021-06-11), but overall they will be
+	  // compatible with these images. These images were created using the setup in
+	  // this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
+	  // these links.
+    {
+        "Name": "alpine-3-13-5",
+        "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2",
+        "SHA256Sum": "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287",
+        "MemoryMegs": 256,
+        "PackageManager": "apk",
+        "InitSystem": "openrc"
+    },
+    {
+        "Name": "alpine-edge",
+        "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2",
+        "SHA256Sum": "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0",
+        "MemoryMegs": 256,
+        "PackageManager": "apk",
+        "InitSystem": "openrc"
+    },
+
+	  // NOTE(Xe): All of the following images are official images straight from each
+	  // distribution's official documentation.
+    {
+        "Name": "amazon-linux",
+        "URL": "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2",
+        "SHA256Sum": "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b",
+        "MemoryMegs": 512,
+        "PackageManager": "yum",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "arch",
+        "URL": "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2",
+        "SHA256Sum": "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27",
+        "MemoryMegs": 512,
+        "PackageManager": "pacman",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "centos-7",
+        "URL": "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c",
+        "SHA256Sum": "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87",
+        "MemoryMegs": 512,
+        "PackageManager": "yum",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "centos-8",
+        "URL": "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2",
+        "SHA256Sum": "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0",
+        "MemoryMegs": 768,
+        "PackageManager": "dnf",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "debian-9",
+        "URL": "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2",
+        "SHA256Sum": "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "debian-10",
+        "URL": "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2",
+        "SHA256Sum": "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0",
+        "MemoryMegs": 768,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "fedora-34",
+        "URL": "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2",
+        "SHA256Sum": "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea",
+        "MemoryMegs": 768,
+        "PackageManager": "dnf",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-leap-15-1",
+        "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2",
+        "SHA256Sum": "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-leap-15-2",
+        "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2",
+        "SHA256Sum": "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-leap-15-3",
+        "URL": "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2",
+        "SHA256Sum": "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "opensuse-tumbleweed",
+        "URL": "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2",
+        "SHA256Sum": "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f",
+        "MemoryMegs": 512,
+        "PackageManager": "zypper",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "oracle-linux-7",
+        "URL": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2",
+        "SHA256Sum": "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb",
+        "MemoryMegs": 512,
+        "PackageManager": "yum",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "oracle-linux-8",
+        "URL": "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2",
+        "SHA256Sum": "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b",
+        "MemoryMegs": 768,
+        "PackageManager": "dnf",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-16-04",
+        "URL": "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img",
+        "SHA256Sum": "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-18-04",
+        "URL": "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img",
+        "SHA256Sum": "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-20-04",
+        "URL": "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img",
+        "SHA256Sum": "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-20-10",
+        "URL": "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img",
+        "SHA256Sum": "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "ubuntu-21-04",
+        "URL": "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img",
+        "SHA256Sum": "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b",
+        "MemoryMegs": 512,
+        "PackageManager": "apt",
+        "InitSystem": "systemd"
+    },
+
+	  // NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
+	  // used here is actually the URL of the NixOS channel being built from and the
+	  // shasum is meaningless. This `channel:name` syntax is documented at [1].
+	  //
+	  // [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
+    {
+        "Name": "nixos-21-05",
+        "URL": "channel:nixos-21.05",
+        "SHA256Sum": "lolfakesha",
+        "MemoryMegs": 512,
+        "PackageManager": "nix",
+        "InitSystem": "systemd"
+    },
+    {
+        "Name": "nixos-unstable",
+        "URL": "channel:nixos-unstable",
+        "SHA256Sum": "lolfakesha",
+        "MemoryMegs": 512,
+        "PackageManager": "nix",
+        "InitSystem": "systemd"
+    }
+]

tstest/integration/vms/distros_test.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"testing"
+)
+
+func TestDistrosGotLoaded(t *testing.T) {
+	if len(Distros) == 0 {
+		t.Fatal("no distros were loaded")
+	}
+}

tstest/integration/vms/gen/test_codegen.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// build ignore
+
+package main
+
+import (
+	_ "embed"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/dave/jennifer/jen"
+	"github.com/iancoleman/strcase"
+	"tailscale.com/tstest/integration/vms"
+)
+
+func main() {
+	f := jen.NewFile("vms")
+	f.Comment("Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.")
+
+	ptr := jen.Op("*")
+
+	for i, d := range vms.Distros {
+		f.Func().
+			Id("TestRun" + strcase.ToCamel(d.Name)).
+			Params(jen.Id("t").Add(ptr).Qual("testing", "T")).
+			BlockFunc(func(g *jen.Group) {
+				g.Id("t").Dot("Parallel").Call()
+				g.Id("setupTests").Call(jen.Id("t"))
+				g.Id("testOneDistribution").Call(jen.Id("t"), jen.Lit(i), jen.Id("Distros").Index(jen.Lit(i)))
+			})
+	}
+
+	os.Remove("top_level_test.go")
+	fout, err := os.Create("top_level_test.go")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fout.Close()
+
+	fmt.Fprintf(fout, "// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.\n", time.Now().Year())
+	fout.WriteString("// Use of this source code is governed by a BSD-style\n")
+	fout.WriteString("// license that can be found in the LICENSE file.\n")
+	fout.WriteString("\n")
+	fout.WriteString("// +build linux\n\n")
+
+	err = f.Render(fout)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

tstest/integration/vms/harness_test.go

@@ -10,14 +10,19 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"log"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
+	"strconv"
+	"sync"
 	"testing"
 	"time"
 
-	"github.com/gliderlabs/ssh"
+	"golang.org/x/crypto/ssh"
 	"golang.org/x/net/proxy"
 	"inet.af/netaddr"
 	"tailscale.com/tstest/integration"
@@ -28,10 +33,102 @@ type Harness struct {
 	testerDialer   proxy.Dialer
 	testerDir      string
 	bins           *integration.Binaries
+	pubKey         string
 	signer         ssh.Signer
 	cs             *testcontrol.Server
 	loginServerURL string
 	testerV4       netaddr.IP
+	ipMu           *sync.Mutex
+	ipMap          map[string]ipMapping
+}
+
+func newHarness(t *testing.T) *Harness {
+	dir := t.TempDir()
+	bindHost := deriveBindhost(t)
+	ln, err := net.Listen("tcp", net.JoinHostPort(bindHost, "0"))
+	if err != nil {
+		t.Fatalf("can't make TCP listener: %v", err)
+	}
+	t.Cleanup(func() {
+		ln.Close()
+	})
+	t.Logf("host:port: %s", ln.Addr())
+
+	cs := &testcontrol.Server{}
+
+	derpMap := integration.RunDERPAndSTUN(t, t.Logf, bindHost)
+	cs.DERPMap = derpMap
+
+	var (
+		ipMu  sync.Mutex
+		ipMap = map[string]ipMapping{}
+	)
+
+	mux := http.NewServeMux()
+	mux.Handle("/", cs)
+
+	lc := &integration.LogCatcher{}
+	if *verboseLogcatcher {
+		lc.UseLogf(t.Logf)
+	}
+	mux.Handle("/c/", lc)
+
+	// This handler will let the virtual machines tell the host information about that VM.
+	// This is used to maintain a list of port->IP address mappings that are known to be
+	// working. This allows later steps to connect over SSH. This returns no response to
+	// clients because no response is needed.
+	mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) {
+		ipMu.Lock()
+		defer ipMu.Unlock()
+
+		name := path.Base(r.URL.Path)
+		host, _, _ := net.SplitHostPort(r.RemoteAddr)
+		port, err := strconv.Atoi(name)
+		if err != nil {
+			log.Panicf("bad port: %v", port)
+		}
+		distro := r.UserAgent()
+		ipMap[distro] = ipMapping{distro, port, host}
+		t.Logf("%s: %v", name, host)
+	})
+
+	hs := &http.Server{Handler: mux}
+	go hs.Serve(ln)
+
+	run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``)
+	pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub"))
+	if err != nil {
+		t.Fatalf("can't read ssh key: %v", err)
+	}
+
+	privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey"))
+	if err != nil {
+		t.Fatalf("can't read ssh private key: %v", err)
+	}
+
+	signer, err := ssh.ParsePrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("can't parse private key: %v", err)
+	}
+
+	loginServer := fmt.Sprintf("http://%s", ln.Addr())
+	t.Logf("loginServer: %s", loginServer)
+
+	bins := integration.BuildTestBinaries(t)
+
+	h := &Harness{
+		pubKey:         string(pubkey),
+		bins:           bins,
+		signer:         signer,
+		loginServerURL: loginServer,
+		cs:             cs,
+		ipMu:           &ipMu,
+		ipMap:          ipMap,
+	}
+
+	h.makeTestNode(t, bins, loginServer)
+
+	return h
 }
 
 func (h *Harness) Tailscale(t *testing.T, args ...string) []byte {

tstest/integration/vms/nixos_test.go

@@ -166,10 +166,14 @@ func copyUnit(t *testing.T, bins *integration.Binaries) {
 	}
 }
 
-func (h Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
+func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
+	if d.Name == "nixos-unstable" {
+		t.Skip("https://github.com/NixOS/nixpkgs/issues/131098")
+	}
+
 	copyUnit(t, h.bins)
 	dir := t.TempDir()
-	fname := filepath.Join(dir, d.name+".nix")
+	fname := filepath.Join(dir, d.Name+".nix")
 	fout, err := os.Create(fname)
 	if err != nil {
 		t.Fatal(err)
@@ -196,10 +200,10 @@ func (h Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
 	os.MkdirAll(outpath, 0755)
 
 	t.Cleanup(func() {
-		os.RemoveAll(filepath.Join(outpath, d.name)) // makes the disk image a candidate for GC
+		os.RemoveAll(filepath.Join(outpath, d.Name)) // makes the disk image a candidate for GC
 	})
 
-	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.name), "-c", fname)
+	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.Name), "-c", fname)
 	if *verboseNixOutput {
 		cmd.Stdout = logger.FuncWriter(t.Logf)
 		cmd.Stderr = logger.FuncWriter(t.Logf)
@@ -214,16 +218,16 @@ func (h Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
 		cmd.Stderr = fout
 		defer fout.Close()
 	}
-	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.url)
+	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.URL)
 	cmd.Dir = outpath
 	t.Logf("running %s %#v", "nixos-generate", cmd.Args)
 	if err := cmd.Run(); err != nil {
-		t.Fatalf("error while making NixOS image for %s: %v", d.name, err)
+		t.Fatalf("error while making NixOS image for %s: %v", d.Name, err)
 	}
 
 	if !*verboseNixOutput {
 		t.Log("done")
 	}
 
-	return filepath.Join(outpath, d.name, "nixos.qcow2")
+	return filepath.Join(outpath, d.Name, "nixos.qcow2")
 }

tstest/integration/vms/opensuse_leap_15_1_test.go

@@ -41,7 +41,7 @@ type openSUSELeap151MetaDataMeta struct {
 }
 
 func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
-	if d.name != "opensuse-leap-15-1" {
+	if d.Name != "opensuse-leap-15-1" {
 		return false
 	}
 
@@ -54,14 +54,14 @@ func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
 
 	metadata, err := json.Marshal(openSUSELeap151MetaData{
 		Zone:        "nova",
-		Hostname:    d.name,
+		Hostname:    d.Name,
 		LaunchIndex: "0",
 		Meta: openSUSELeap151MetaDataMeta{
 			Role:      "server",
 			DSMode:    "local",
 			Essential: "false",
 		},
-		Name: d.name,
+		Name: d.Name,
 		UUID: uuid.New().String(),
 	})
 	if err != nil {

tstest/integration/vms/top_level_test.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import "testing"
+
+// Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.
+func TestRunAlpine3135(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 0, Distros[0])
+}
+func TestRunAlpineEdge(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 1, Distros[1])
+}
+func TestRunAmazonLinux(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 2, Distros[2])
+}
+func TestRunArch(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 3, Distros[3])
+}
+func TestRunCentos7(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 4, Distros[4])
+}
+func TestRunCentos8(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 5, Distros[5])
+}
+func TestRunDebian9(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 6, Distros[6])
+}
+func TestRunDebian10(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 7, Distros[7])
+}
+func TestRunFedora34(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 8, Distros[8])
+}
+func TestRunOpensuseLeap151(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 9, Distros[9])
+}
+func TestRunOpensuseLeap152(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 10, Distros[10])
+}
+func TestRunOpensuseLeap153(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 11, Distros[11])
+}
+func TestRunOpensuseTumbleweed(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 12, Distros[12])
+}
+func TestRunOracleLinux7(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 13, Distros[13])
+}
+func TestRunOracleLinux8(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 14, Distros[14])
+}
+func TestRunUbuntu1604(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 15, Distros[15])
+}
+func TestRunUbuntu1804(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 16, Distros[16])
+}
+func TestRunUbuntu2004(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 17, Distros[17])
+}
+func TestRunUbuntu2010(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 18, Distros[18])
+}
+func TestRunUbuntu2104(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 19, Distros[19])
+}
+func TestRunNixos2105(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 20, Distros[20])
+}
+func TestRunNixosUnstable(t *testing.T) {
+	t.Parallel()
+	setupTests(t)
+	testOneDistribution(t, 21, Distros[21])
+}

tstest/integration/vms/vm_setup_test.go

@@ -35,7 +35,7 @@ import (
 // mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
 // to the testcontrol server. The function it returns is for killing the virtual
 // machine when it is time for it to die.
-func (h Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) {
+func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) {
 	t.Helper()
 
 	cdir, err := os.UserCacheDir()
@@ -53,17 +53,17 @@ func (h Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir strin
 	mkLayeredQcow(t, tdir, d, h.fetchDistro(t, d))
 	mkSeed(t, d, sshKey, hostURL, tdir, port)
 
-	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
+	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.Name+".qcow2"))
 
 	args := []string{
 		"-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off",
 		"-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port),
 		"-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25",
-		"-m", fmt.Sprint(d.mem),
+		"-m", fmt.Sprint(d.MemoryMegs),
 		"-boot", "c",
 		"-drive", driveArg,
-		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
-		"-smbios", "type=1,serial=ds=nocloud;h=" + d.name,
+		"-cdrom", filepath.Join(tdir, d.Name, "seed", "seed.iso"),
+		"-smbios", "type=1,serial=ds=nocloud;h=" + d.Name,
 	}
 
 	if *useVNC {
@@ -101,7 +101,7 @@ func (h Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir strin
 	t.Cleanup(func() {
 		err := cmd.Process.Kill()
 		if err != nil {
-			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
+			t.Errorf("can't kill %s (%d): %v", d.Name, cmd.Process.Pid, err)
 		}
 
 		cmd.Wait()
@@ -139,15 +139,15 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
 		d.PartSize = 64 * 1024 * 1024 // 64MB per part
 	})
 
-	t.Logf("fetching s3://%s/%s", bucketName, d.sha256sum)
+	t.Logf("fetching s3://%s/%s", bucketName, d.SHA256Sum)
 
 	_, err = dler.Download(fout, &s3.GetObjectInput{
 		Bucket: aws.String(bucketName),
-		Key:    aws.String(d.sha256sum),
+		Key:    aws.String(d.SHA256Sum),
 	})
 	if err != nil {
 		fout.Close()
-		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.sha256sum, err)
+		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.SHA256Sum, err)
 	}
 
 	err = fout.Close()
@@ -160,7 +160,7 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
 
 // fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
 // also validates the sha256 sum from a known good hash.
-func (h Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
+func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	t.Helper()
 
 	cdir, err := os.UserCacheDir()
@@ -169,17 +169,17 @@ func (h Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	}
 	cdir = filepath.Join(cdir, "tailscale", "vm-test")
 
-	if strings.HasPrefix(resultDistro.name, "nixos") {
+	if strings.HasPrefix(resultDistro.Name, "nixos") {
 		return h.makeNixOSImage(t, resultDistro, cdir)
 	}
 
-	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
+	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.SHA256Sum)
 
 	_, err = os.Stat(qcowPath)
 	if err == nil {
 		hash := checkCachedImageHash(t, resultDistro, cdir)
-		if hash != resultDistro.sha256sum {
-			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.name, qcowPath, resultDistro.sha256sum)
+		if hash != resultDistro.SHA256Sum {
+			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.Name, qcowPath, resultDistro.SHA256Sum)
 			err = errors.New("some fake non-nil error to force a redownload")
 
 			if err := os.Remove(qcowPath); err != nil {
@@ -189,26 +189,26 @@ func (h Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	}
 
 	if err != nil {
-		t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
+		t.Logf("downloading distro image %s to %s", resultDistro.URL, qcowPath)
 		fout, err := os.Create(qcowPath)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		if !fetchFromS3(t, fout, resultDistro) {
-			resp, err := http.Get(resultDistro.url)
+			resp, err := http.Get(resultDistro.URL)
 			if err != nil {
-				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
+				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.Name, resultDistro.URL, err)
 			}
 
 			if resp.StatusCode != http.StatusOK {
 				resp.Body.Close()
-				t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
+				t.Fatalf("%s replied %s", resultDistro.URL, resp.Status)
 			}
 
 			_, err = io.Copy(fout, resp.Body)
 			if err != nil {
-				t.Fatalf("download of %s failed: %v", resultDistro.url, err)
+				t.Fatalf("download of %s failed: %v", resultDistro.URL, err)
 			}
 
 			resp.Body.Close()
@@ -219,8 +219,8 @@ func (h Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 
 			hash := checkCachedImageHash(t, resultDistro, cdir)
 
-			if hash != resultDistro.sha256sum {
-				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.sha256sum, hash)
+			if hash != resultDistro.SHA256Sum {
+				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.SHA256Sum, hash)
 			}
 		}
 	}
@@ -231,7 +231,7 @@ func (h Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) {
 	t.Helper()
 
-	qcowPath := filepath.Join(cacheDir, "qcow2", d.sha256sum)
+	qcowPath := filepath.Join(cacheDir, "qcow2", d.SHA256Sum)
 
 	fin, err := os.Open(qcowPath)
 	if err != nil {
@@ -244,8 +244,8 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri
 	}
 	hash := hex.EncodeToString(hasher.Sum(nil))
 
-	if hash != d.sha256sum {
-		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.sha256sum)
+	if hash != d.SHA256Sum {
+		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.SHA256Sum)
 	}
 
 	gotHash = hash
@@ -253,9 +253,9 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri
 	return
 }
 
-func (h Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
+func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
 	bins := h.bins
-	if strings.HasPrefix(d.name, "nixos") {
+	if strings.HasPrefix(d.Name, "nixos") {
 		return
 	}
 
@@ -275,7 +275,7 @@ func (h Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
 	// TODO(Xe): revisit this assumption before it breaks the test.
 	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
 
-	switch d.initSystem {
+	switch d.InitSystem {
 	case "openrc":
 		mkdir(t, cli, "/etc/init.d")
 		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.openrc", "/etc/init.d/tailscaled")

tstest/integration/vms/vms_steps_test.go

@@ -39,7 +39,7 @@ func retry(t *testing.T, fn func() error) {
 	t.Fatalf("tried %d times, got: %v", tries, err)
 }
 
-func (h Harness) testPing(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) {
+func (h *Harness) testPing(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) {
 	var outp []byte
 	var err error
 	retry(t, func() error {
@@ -83,7 +83,7 @@ func getSession(t *testing.T, cli *ssh.Client) *ssh.Session {
 	return sess
 }
 
-func (h Harness) testOutgoingTCP(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) {
+func (h *Harness) testOutgoingTCP(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) {
 	const sendmsg = "this is a message that curl won't print"
 	ctx, cancel := context.WithCancel(context.Background())
 	s := &http.Server{

tstest/integration/vms/vms_test.go

@@ -11,12 +11,9 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
 	"net"
-	"net/http"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -33,7 +30,6 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/integration"
-	"tailscale.com/tstest/integration/testcontrol"
 	"tailscale.com/types/logger"
 )
 
@@ -62,20 +58,20 @@ func TestDownloadImages(t *testing.T) {
 
 	bins := integration.BuildTestBinaries(t)
 
-	for _, d := range distros {
+	for _, d := range Distros {
 		distro := d
-		t.Run(distro.name, func(t *testing.T) {
-			if !distroRex.Unwrap().MatchString(distro.name) {
-				t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex)
+		t.Run(distro.Name, func(t *testing.T) {
+			if !distroRex.Unwrap().MatchString(distro.Name) {
+				t.Skipf("distro name %q doesn't match regex: %s", distro.Name, distroRex)
 			}
 
-			if strings.HasPrefix(distro.name, "nixos") {
+			if strings.HasPrefix(distro.Name, "nixos") {
 				t.Skip("NixOS is built on the fly, no need to download it")
 			}
 
 			t.Parallel()
 
-			(Harness{bins: bins}).fetchDistro(t, distro)
+			(&Harness{bins: bins}).fetchDistro(t, distro)
 		})
 	}
 }
@@ -103,7 +99,7 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro, qcowBase string) {
 	run(t, tdir, "qemu-img", "create",
 		"-f", "qcow2",
 		"-o", "backing_file="+qcowBase,
-		filepath.Join(tdir, d.name+".qcow2"),
+		filepath.Join(tdir, d.Name+".qcow2"),
 	)
 }
 
@@ -117,7 +113,7 @@ var (
 func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 	t.Helper()
 
-	dir := filepath.Join(tdir, d.name, "seed")
+	dir := filepath.Join(tdir, d.Name, "seed")
 	os.MkdirAll(dir, 0700)
 
 	// make meta-data
@@ -132,7 +128,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			Hostname string
 		}{
 			ID:       "31337",
-			Hostname: d.name,
+			Hostname: d.Name,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -161,7 +157,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 		}{
 			SSHKey:     strings.TrimSpace(sshKey),
 			HostURL:    hostURL,
-			Hostname:   d.name,
+			Hostname:   d.Name,
 			Port:       port,
 			InstallPre: d.InstallPre(),
 			Password:   securePassword,
@@ -225,10 +221,11 @@ func getProbablyFreePortNumber() (int, error) {
 	return portNum, nil
 }
 
-// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
-// tailscale on it and then ensures that it connects to the network
-// successfully.
-func TestVMIntegrationEndToEnd(t *testing.T) {
+func setupTests(t *testing.T) {
+	ramsem.once.Do(func() {
+		ramsem.sem = semaphore.NewWeighted(int64(*vmRamLimit))
+	})
+
 	if !*runVMTests {
 		t.Skip("not running integration tests (need --run-vm-tests)")
 	}
@@ -244,139 +241,56 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test --v --timeout=60m --run-vm-tests'")
 		t.Fatalf("missing dependency: %v", err)
 	}
-
-	dir := t.TempDir()
-
-	rex := distroRex.Unwrap()
-
-	bindHost := deriveBindhost(t)
-	ln, err := net.Listen("tcp", net.JoinHostPort(bindHost, "0"))
-	if err != nil {
-		t.Fatalf("can't make TCP listener: %v", err)
-	}
-	defer ln.Close()
-	t.Logf("host:port: %s", ln.Addr())
-
-	cs := &testcontrol.Server{}
-
-	derpMap := integration.RunDERPAndSTUN(t, t.Logf, bindHost)
-	cs.DERPMap = derpMap
-
-	var (
-		ipMu  sync.Mutex
-		ipMap = map[string]ipMapping{}
-	)
-
-	mux := http.NewServeMux()
-	mux.Handle("/", cs)
-
-	lc := &integration.LogCatcher{}
-	if *verboseLogcatcher {
-		lc.UseLogf(t.Logf)
-	}
-	mux.Handle("/c/", lc)
-
-	// This handler will let the virtual machines tell the host information about that VM.
-	// This is used to maintain a list of port->IP address mappings that are known to be
-	// working. This allows later steps to connect over SSH. This returns no response to
-	// clients because no response is needed.
-	mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) {
-		ipMu.Lock()
-		defer ipMu.Unlock()
-
-		name := path.Base(r.URL.Path)
-		host, _, _ := net.SplitHostPort(r.RemoteAddr)
-		port, err := strconv.Atoi(name)
-		if err != nil {
-			log.Panicf("bad port: %v", port)
-		}
-		distro := r.UserAgent()
-		ipMap[distro] = ipMapping{distro, port, host}
-		t.Logf("%s: %v", name, host)
-	})
-
-	hs := &http.Server{Handler: mux}
-	go hs.Serve(ln)
-
-	run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``)
-	pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub"))
-	if err != nil {
-		t.Fatalf("can't read ssh key: %v", err)
-	}
-
-	privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey"))
-	if err != nil {
-		t.Fatalf("can't read ssh private key: %v", err)
-	}
-
-	signer, err := ssh.ParsePrivateKey(privateKey)
-	if err != nil {
-		t.Fatalf("can't parse private key: %v", err)
-	}
-
-	loginServer := fmt.Sprintf("http://%s", ln.Addr())
-	t.Logf("loginServer: %s", loginServer)
-
-	ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
-	bins := integration.BuildTestBinaries(t)
-
-	h := &Harness{
-		bins:           bins,
-		signer:         signer,
-		loginServerURL: loginServer,
-		cs:             cs,
-	}
-
-	h.makeTestNode(t, bins, loginServer)
-
-	t.Run("do", func(t *testing.T) {
-		for n, distro := range distros {
-			n, distro := n, distro
-			if rex.MatchString(distro.name) {
-				t.Logf("%s matches %s", distro.name, rex)
-			} else {
-				continue
-			}
-
-			t.Run(distro.name, func(t *testing.T) {
-				ctx, done := context.WithCancel(context.Background())
-				t.Cleanup(done)
-
-				t.Parallel()
-
-				dir := t.TempDir()
-
-				err := ramsem.Acquire(ctx, int64(distro.mem))
-				if err != nil {
-					t.Fatalf("can't acquire ram semaphore: %v", err)
-				}
-				defer ramsem.Release(int64(distro.mem))
-
-				h.mkVM(t, n, distro, string(pubkey), loginServer, dir)
-				var ipm ipMapping
-
-				t.Run("wait-for-start", func(t *testing.T) {
-					waiter := time.NewTicker(time.Second)
-					defer waiter.Stop()
-					var ok bool
-					for {
-						<-waiter.C
-						ipMu.Lock()
-						if ipm, ok = ipMap[distro.name]; ok {
-							ipMu.Unlock()
-							break
-						}
-						ipMu.Unlock()
-					}
-				})
-
-				h.testDistro(t, distro, ipm)
-			})
-		}
-	})
 }
 
-func (h Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
+var ramsem struct {
+	once sync.Once
+	sem  *semaphore.Weighted
+}
+
+func testOneDistribution(t *testing.T, n int, distro Distro) {
+	setupTests(t)
+
+	if distroRex.Unwrap().MatchString(distro.Name) {
+		t.Logf("%s matches %s", distro.Name, distroRex.Unwrap())
+	} else {
+		t.Skip("regex not matched")
+	}
+
+	ctx, done := context.WithCancel(context.Background())
+	t.Cleanup(done)
+
+	h := newHarness(t)
+	dir := t.TempDir()
+
+	err := ramsem.sem.Acquire(ctx, int64(distro.MemoryMegs))
+	if err != nil {
+		t.Fatalf("can't acquire ram semaphore: %v", err)
+	}
+	t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) })
+
+	h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir)
+	var ipm ipMapping
+
+	t.Run("wait-for-start", func(t *testing.T) {
+		waiter := time.NewTicker(time.Second)
+		defer waiter.Stop()
+		var ok bool
+		for {
+			<-waiter.C
+			h.ipMu.Lock()
+			if ipm, ok = h.ipMap[distro.Name]; ok {
+				h.ipMu.Unlock()
+				break
+			}
+			h.ipMu.Unlock()
+		}
+	})
+
+	h.testDistro(t, distro, ipm)
+}
+
+func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 	signer := h.signer
 	loginServer := h.loginServerURL
 
@@ -424,7 +338,7 @@ func (h Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 			&expect.BExp{R: `(\#)`},
 		}
 
-		switch d.initSystem {
+		switch d.InitSystem {
 		case "openrc":
 			// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
 			// of service readiness. If this sleep is removed then tailscale will not be
@@ -449,11 +363,32 @@ func (h Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 	})
 
 	t.Run("tailscale status", func(t *testing.T) {
-		runTestCommands(t, timeout, cli, []expect.Batcher{
-			&expect.BSnd{S: "sleep 5 && tailscale status\n"},
-			&expect.BExp{R: `100.64.0.1`},
-			&expect.BExp{R: `(\#)`},
-		})
+		dur := 100 * time.Millisecond
+		var outp []byte
+		var err error
+
+		// NOTE(Xe): retry `tailscale status` a few times until it works. When tailscaled
+		// starts with testcontrol sometimes there can be up to a few seconds where
+		// tailscaled is in an unknown state on these virtual machines. This exponential
+		// delay loop should delay long enough for tailscaled to be ready.
+		for count := 0; count < 10; count++ {
+			sess := getSession(t, cli)
+
+			outp, err = sess.CombinedOutput("tailscale status")
+			if err == nil {
+				if !strings.Contains(string(outp), "100.64.0.1") {
+					t.Log(string(outp))
+					t.Fatal("can't find tester IP")
+				}
+
+				return
+			}
+			time.Sleep(dur)
+			dur = dur * 2
+		}
+
+		t.Log(string(outp))
+		t.Fatalf("error: %v", err)
 	})
 
 	t.Run("dump routes", func(t *testing.T) {
@@ -581,25 +516,40 @@ func (h Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 		}
 		defer ln.Close()
 
-		sess, err := cli.NewSession()
-		if err != nil {
-			t.Fatalf("can't open session: %v", err)
-		}
-		defer sess.Close()
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
 
-		sess.Stdin = strings.NewReader("hi")
-		sess.Stdout = logger.FuncWriter(t.Logf)
-		sess.Stderr = logger.FuncWriter(t.Logf)
+		go func() {
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
 
-		_, port, _ := net.SplitHostPort(ln.LocalAddr().String())
+				sess, err := cli.NewSession()
+				if err != nil {
+					t.Errorf("can't open session: %v", err)
+					return
+				}
+				defer sess.Close()
 
-		cmd := fmt.Sprintf("/udp_tester -client %s\n", net.JoinHostPort("100.64.0.1", port))
-		time.Sleep(10 * time.Millisecond)
-		t.Logf("sending packet: %s", cmd)
-		err = sess.Run(cmd)
-		if err != nil {
-			t.Errorf("can't send UDP packet: %v", err)
-		}
+				sess.Stdin = strings.NewReader("hi")
+				sess.Stdout = logger.FuncWriter(t.Logf)
+				sess.Stderr = logger.FuncWriter(t.Logf)
+
+				_, port, _ := net.SplitHostPort(ln.LocalAddr().String())
+
+				cmd := fmt.Sprintf("/udp_tester -client %s\n", net.JoinHostPort("100.64.0.1", port))
+				t.Logf("sending packet: %s", cmd)
+				err = sess.Run(cmd)
+				if err != nil {
+					t.Logf("can't send UDP packet: %v", err)
+				}
+
+				time.Sleep(10 * time.Millisecond)
+			}
+		}()
 
 		t.Log("listening for packet")
 		n, _, err := ln.ReadFromUDP(buf)

tstime/mono/mono.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package mono provides fast monotonic time.
+// On most platforms, mono.Now is about 2x faster than time.Now.
+// However, time.Now is really fast, and nicer to use.
+//
+// For almost all purposes, you should use time.Now.
+//
+// Package mono exists because we get the current time multiple
+// times per network packet, at which point it makes a
+// measurable difference.
+package mono
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+	_ "unsafe" // for go:linkname
+)
+
+// Time is the number of nanoseconds elapsed since an unspecified reference start time.
+type Time int64
+
+// Now returns the current monotonic time.
+func Now() Time {
+	// On a newly started machine, the monotonic clock might be very near zero.
+	// Thus mono.Time(0).Before(mono.Now.Add(-time.Minute)) might yield true.
+	// The corresponding package time expression never does, if the wall clock is correct.
+	// Preserve this correspondence by increasing the "base" monotonic clock by a fair amount.
+	const baseOffset int64 = 1 << 55 // approximately 10,000 hours in nanoseconds
+	return Time(now() + baseOffset)
+}
+
+// Since returns the time elapsed since t.
+func Since(t Time) time.Duration {
+	return time.Duration(Now() - t)
+}
+
+// Sub returns t-n, the duration from n to t.
+func (t Time) Sub(n Time) time.Duration {
+	return time.Duration(t - n)
+}
+
+// Add returns t+d.
+func (t Time) Add(d time.Duration) Time {
+	return t + Time(d)
+}
+
+// After reports t > n, whether t is after n.
+func (t Time) After(n Time) bool {
+	return t > n
+}
+
+// After reports t < n, whether t is before n.
+func (t Time) Before(n Time) bool {
+	return t < n
+}
+
+// IsZero reports whether t == 0.
+func (t Time) IsZero() bool {
+	return t == 0
+}
+
+// StoreAtomic does an atomic store *t = new.
+func (t *Time) StoreAtomic(new Time) {
+	atomic.StoreInt64((*int64)(t), int64(new))
+}
+
+// LoadAtomic does an atomic load *t.
+func (t *Time) LoadAtomic() Time {
+	return Time(atomic.LoadInt64((*int64)(t)))
+}
+
+//go:linkname now runtime.nanotime1
+func now() int64
+
+// baseWall and baseMono are a pair of almost-identical times used to correlate a Time with a wall time.
+var (
+	baseWall time.Time
+	baseMono Time
+)
+
+func init() {
+	baseWall = time.Now()
+	baseMono = Now()
+}
+
+// String prints t, including an estimated equivalent wall clock.
+// This is best-effort only, for rough debugging purposes only.
+// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
+// Even in the best of circumstances, it may vary by a few milliseconds.
+func (t Time) String() string {
+	return fmt.Sprintf("mono.Time(ns=%d, estimated wall=%v)", int64(t), baseWall.Add(t.Sub(baseMono)).Truncate(0))
+}
+
+// MarshalJSON formats t for JSON as if it were a time.Time.
+// We format Time this way for backwards-compatibility.
+// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
+// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
+// Even in the best of circumstances, it may vary by a few milliseconds.
+func (t Time) MarshalJSON() ([]byte, error) {
+	var tt time.Time
+	if !t.IsZero() {
+		tt = baseWall.Add(t.Sub(baseMono)).Truncate(0)
+	}
+	return tt.MarshalJSON()
+}
+
+// UnmarshalJSON sets t according to data.
+// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	var tt time.Time
+	err := tt.UnmarshalJSON(data)
+	if err != nil {
+		return err
+	}
+	*t = Now().Add(-time.Since(tt))
+	return nil
+}

tstime/mono/mono_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mono
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNow(t *testing.T) {
+	start := Now()
+	time.Sleep(100 * time.Millisecond)
+	if elapsed := Since(start); elapsed < 100*time.Millisecond {
+		t.Errorf("short sleep: %v elapsed, want min %v", elapsed, 100*time.Millisecond)
+	}
+}
+
+func BenchmarkMonoNow(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Now()
+	}
+}
+
+func BenchmarkTimeNow(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		time.Now()
+	}
+}