derp: WIP notes on adding a flow type

very rough, uncompiled.

.github/workflows/go_generate.yml

@@ -1,34 +0,0 @@
-name: go generate
-
-on:
-  push:
-    branches:
-      - main
-      - "release-branch/*"
-  pull_request:
-    branches:
-      - "*"
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.16
-
-      - name: Check out code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: check 'go generate' is clean
-        run: |
-          mkdir gentools
-          go build -o gentools/stringer golang.org/x/tools/cmd/stringer
-          PATH="$PATH:$(pwd)/gentools" go generate ./...
-          echo
-          echo
-          git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1)

.github/workflows/xe-experimental-vm-test.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     paths:
       - "tstest/integration/vms/**"
+  push:
+    branches: [ main ]
   release:
     types: [ created ]
 

Dockerfile

@@ -42,7 +42,8 @@ FROM golang:1.16-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
-COPY go.mod go.sum ./
+COPY go.mod .
+COPY go.sum .
 RUN go mod download
 
 COPY . .

VERSION.txt

@@ -1 +1 @@
-1.13.0
+1.11.0

cmd/addlicense/main.go

@@ -1,77 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Program addlicense adds a license header to a file.
-// It is intended for use with 'go generate',
-// so it has a slightly weird usage.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"os/exec"
-)
-
-var (
-	year = flag.Int("year", 0, "copyright year")
-	file = flag.String("file", "", "file to modify")
-)
-
-func usage() {
-	fmt.Fprintf(os.Stderr, `
-usage: addlicense -year YEAR -file FILE <subcommand args...>
-`[1:])
-
-	flag.PrintDefaults()
-	fmt.Fprintf(os.Stderr, `
-addlicense adds a Tailscale license to the beginning of file,
-using year as the copyright year.
-
-It is intended for use with 'go generate', so it also runs a subcommand,
-which presumably creates the file.
-
-Sample usage:
-
-addlicense -year 2021 -file pull_strings.go stringer -type=pull
-`[1:])
-	os.Exit(2)
-}
-
-func main() {
-	flag.Usage = usage
-	flag.Parse()
-	if len(flag.Args()) == 0 {
-		flag.Usage()
-	}
-	cmd := exec.Command(flag.Arg(0), flag.Args()[1:]...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	check(err)
-	b, err := os.ReadFile(*file)
-	check(err)
-	f, err := os.OpenFile(*file, os.O_TRUNC|os.O_WRONLY, 0644)
-	check(err)
-	_, err = fmt.Fprintf(f, license, *year)
-	check(err)
-	_, err = f.Write(b)
-	check(err)
-	err = f.Close()
-	check(err)
-}
-
-func check(err error) {
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-}
-
-var license = `
-// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-`[1:]

cmd/derper/mesh.go

@@ -9,9 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net"
 	"strings"
-	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -41,34 +39,6 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return err
 	}
 	c.MeshKey = s.MeshKey()
-
-	// For meshed peers within a region, connect via VPC addresses.
-	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
-		host, port, err := net.SplitHostPort(addr)
-		if err != nil {
-			return nil, err
-		}
-		var d net.Dialer
-		var r net.Resolver
-		if port == "443" && strings.HasSuffix(host, ".tailscale.com") {
-			base := strings.TrimSuffix(host, ".tailscale.com")
-			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
-			defer cancel()
-			vpcHost := base + "-vpc.tailscale.com"
-			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
-			if len(ips) > 0 {
-				vpcAddr := net.JoinHostPort(ips[0].String(), port)
-				c, err := d.DialContext(subCtx, network, vpcAddr)
-				if err == nil {
-					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
-					return c, nil
-				}
-				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
-			}
-		}
-		return d.DialContext(ctx, network, addr)
-	})
-
 	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
 	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)

cmd/derpprobe/derpprobe.go

@@ -211,13 +211,6 @@ func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.D
 	}
 	defer toc.Close()
 
-	// Wait a bit for from's node to hear about to existing on the
-	// other node in the region, in the case where the two nodes
-	// are different.
-	if from.Name != to.Name {
-		time.Sleep(100 * time.Millisecond) // pretty arbitrary
-	}
-
 	// Make a random packet
 	pkt := make([]byte, 8)
 	crand.Read(pkt)

cmd/tailscale/cli/status.go

@@ -23,7 +23,6 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/util/dnsname"
 )
 
@@ -194,7 +193,7 @@ func runStatus(ctx context.Context, args []string) error {
 //
 // TODO: have the server report this bool instead.
 func peerActive(ps *ipnstate.PeerStatus) bool {
-	return !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
+	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
 }
 
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {

cmd/tailscale/cli/up.go

@@ -51,14 +51,7 @@ flag is also used.
 	Exec:    runUp,
 }
 
-func effectiveGOOS() string {
-	if v := os.Getenv("TS_DEBUG_UP_FLAG_GOOS"); v != "" {
-		return v
-	}
-	return runtime.GOOS
-}
-
-var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
+var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
@@ -70,13 +63,13 @@ func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
@@ -334,7 +327,7 @@ func runUp(ctx context.Context, args []string) error {
 		}
 	}
 
-	prefs, err := prefsFromUpArgs(upArgs, warnf, st, effectiveGOOS())
+	prefs, err := prefsFromUpArgs(upArgs, warnf, st, runtime.GOOS)
 	if err != nil {
 		fatalf("%s", err)
 	}
@@ -351,7 +344,7 @@ func runUp(ctx context.Context, args []string) error {
 	}
 
 	env := upCheckEnv{
-		goos:          effectiveGOOS(),
+		goos:          runtime.GOOS,
 		user:          os.Getenv("USER"),
 		flagSet:       upFlagSet,
 		upArgs:        upArgs,
@@ -391,7 +384,7 @@ func runUp(ctx context.Context, args []string) error {
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
-				switch effectiveGOOS() {
+				switch runtime.GOOS {
 				case "windows":
 					msg += " (Tailscale service in use by other user?)"
 				default:
@@ -465,7 +458,7 @@ func runUp(ctx context.Context, args []string) error {
 		// Windows service (~tailscaled) is the one that computes the
 		// StateKey based on the connection identity. So for now, just
 		// do as the Windows GUI's always done:
-		if effectiveGOOS() == "windows" {
+		if runtime.GOOS == "windows" {
 			// The Windows service will set this as needed based
 			// on our connection's identity.
 			opts.StateKey = ""

cmd/tailscale/depaware.txt

@@ -49,8 +49,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
-     💣 tailscale.com/tstime/mono                                    from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/derp+

cmd/tailscaled/debug.go

@@ -14,23 +14,18 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/url"
 	"os"
-	"strings"
 	"time"
 
-	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
-	"tailscale.com/net/portmapper"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
-	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -38,7 +33,6 @@ var debugArgs struct {
 	monitor   bool
 	getURL    string
 	derpCheck string
-	portmap   bool
 }
 
 var debugModeFunc = debugMode // so it can be addressable
@@ -46,7 +40,6 @@ var debugModeFunc = debugMode // so it can be addressable
 func debugMode(args []string) error {
 	fs := flag.NewFlagSet("debug", flag.ExitOnError)
 	fs.BoolVar(&debugArgs.monitor, "monitor", false, "If true, run link monitor forever. Precludes all other options.")
-	fs.BoolVar(&debugArgs.portmap, "portmap", false, "If true, run portmap debugging. Precludes all other options.")
 	fs.StringVar(&debugArgs.getURL, "get-url", "", "If non-empty, fetch provided URL.")
 	fs.StringVar(&debugArgs.derpCheck, "derp", "", "if non-empty, test a DERP ping via named region code")
 	if err := fs.Parse(args); err != nil {
@@ -62,9 +55,6 @@ func debugMode(args []string) error {
 	if debugArgs.monitor {
 		return runMonitor(ctx)
 	}
-	if debugArgs.portmap {
-		return debugPortmap(ctx)
-	}
 	if debugArgs.getURL != "" {
 		return getURL(ctx, debugArgs.getURL)
 	}
@@ -201,83 +191,3 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 	log.Printf("ok")
 	return err
 }
-
-func debugPortmap(ctx context.Context) error {
-	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
-	defer cancel()
-
-	portmapper.VerboseLogs = true
-
-	done := make(chan bool, 1)
-
-	var c *portmapper.Client
-	logf := log.Printf
-	c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), func() {
-		logf("portmapping changed.")
-		logf("have mapping: %v", c.HaveMapping())
-
-		if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
-			logf("cb: mapping: %v", ext)
-			select {
-			case done <- true:
-			default:
-			}
-			return
-		}
-		logf("cb: no mapping")
-	})
-	linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
-	if err != nil {
-		return err
-	}
-
-	gatewayAndSelfIP := func() (gw, self netaddr.IP, ok bool) {
-		if v := os.Getenv("TS_DEBUG_GW_SELF"); strings.Contains(v, "/") {
-			i := strings.Index(v, "/")
-			gw = netaddr.MustParseIP(v[:i])
-			self = netaddr.MustParseIP(v[i+1:])
-			return gw, self, true
-		}
-		return linkMon.GatewayAndSelfIP()
-	}
-
-	c.SetGatewayLookupFunc(gatewayAndSelfIP)
-
-	gw, selfIP, ok := gatewayAndSelfIP()
-	if !ok {
-		logf("no gateway or self IP; %v", linkMon.InterfaceState())
-		return nil
-	}
-	logf("gw=%v; self=%v", gw, selfIP)
-
-	res, err := c.Probe(ctx)
-	if err != nil {
-		return fmt.Errorf("Probe: %v", err)
-	}
-	logf("Probe: %+v", res)
-
-	if !res.PCP && !res.PMP && !res.UPnP {
-		logf("no portmapping services available")
-		return nil
-	}
-
-	uc, err := net.ListenPacket("udp", "0.0.0.0:0")
-	if err != nil {
-		return err
-	}
-	defer uc.Close()
-	c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
-
-	if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
-		logf("mapping: %v", ext)
-	} else {
-		logf("no mapping")
-	}
-
-	select {
-	case <-done:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}

cmd/tailscaled/depaware.txt

@@ -130,8 +130,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
-     💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
-        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
@@ -145,7 +143,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
-     💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver

cmd/tailscaled/tailscaled.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/go-multierror/multierror"
-	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
@@ -46,6 +45,15 @@ import (
 	"tailscale.com/wgengine/router"
 )
 
+// globalStateKey is the ipn.StateKey that tailscaled loads on
+// startup.
+//
+// We have to support multiple state keys for other OSes (Windows in
+// particular), but right now Unix daemons run with a single
+// node-global state. To keep open the option of having per-user state
+// later, the global state key doesn't look like a username.
+const globalStateKey = "_daemon"
+
 // defaultTunName returns the default tun device name for the platform.
 func defaultTunName() string {
 	switch runtime.GOOS {
@@ -159,28 +167,6 @@ func main() {
 	}
 }
 
-func ipnServerOpts() (o ipnserver.Options) {
-	// Allow changing the OS-specific IPN behavior for tests
-	// so we can e.g. test Windows-specific behaviors on Linux.
-	goos := os.Getenv("TS_DEBUG_TAILSCALED_IPN_GOOS")
-	if goos == "" {
-		goos = runtime.GOOS
-	}
-
-	o.Port = 41112
-	o.StatePath = args.statepath
-	o.SocketPath = args.socketpath // even for goos=="windows", for tests
-
-	switch goos {
-	default:
-		o.SurviveDisconnects = true
-		o.AutostartStateKey = ipn.GlobalDaemonStateKey
-	case "windows":
-		// Not those.
-	}
-	return o
-}
-
 func run() error {
 	var err error
 
@@ -210,9 +196,6 @@ func run() error {
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
-		if os.Getenv("TS_PLEASE_PANIC") != "" {
-			panic("TS_PLEASE_PANIC asked us to panic")
-		}
 		dns.Cleanup(logf, args.tunname)
 		router.Cleanup(logf, args.tunname)
 		return nil
@@ -288,8 +271,14 @@ func run() error {
 		}
 	}()
 
-	opts := ipnServerOpts()
-	opts.DebugMux = debugMux
+	opts := ipnserver.Options{
+		SocketPath:         args.socketpath,
+		Port:               41112,
+		StatePath:          args.statepath,
+		AutostartStateKey:  globalStateKey,
+		SurviveDisconnects: runtime.GOOS != "windows",
+		DebugMux:           debugMux,
+	}
 	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
@@ -352,7 +341,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			return nil, false, err
 		}
 		conf.Tun = dev
-		r, err := router.New(logf, dev, linkMon)
+		r, err := router.New(logf, dev)
 		if err != nil {
 			dev.Close()
 			return nil, false, err

cmd/tailscaled/tailscaled_windows.go

@@ -168,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			return nil, fmt.Errorf("TUN: %w", err)
 		}
-		r, err := router.New(logf, dev, nil)
+		r, err := router.New(logf, dev)
 		if err != nil {
 			dev.Close()
 			return nil, fmt.Errorf("router: %w", err)
@@ -233,6 +233,12 @@ func startIPNServer(ctx context.Context, logid string) error {
 		}
 	}()
 
+	opts := ipnserver.Options{
+		Port:               41112,
+		SurviveDisconnects: false,
+		StatePath:          args.statepath,
+	}
+
 	// getEngine is called by ipnserver to get the engine. It's
 	// not called concurrently and is not called again once it
 	// successfully returns an engine.
@@ -257,7 +263,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
+	err := ipnserver.Run(ctx, logf, logid, getEngine, opts)
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -29,8 +29,8 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/creack/pty"
 	"github.com/gliderlabs/ssh"
+	"github.com/kr/pty"
 	gossh "golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"

control/controlclient/debug.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"regexp"
 	"runtime"
 	"strconv"
 	"time"
@@ -41,82 +42,28 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 	}
 }
 
+var reHexArgs = regexp.MustCompile(`\b0x[0-9a-f]+\b`)
+
 // scrubbedGoroutineDump returns the list of all current goroutines, but with the actual
 // values of arguments scrubbed out, lest it contain some private key material.
 func scrubbedGoroutineDump() []byte {
-	var buf []byte
-	// Grab stacks multiple times into increasingly larger buffer sizes
-	// to minimize the risk that we blow past our iOS memory limit.
-	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
-		buf = make([]byte, size)
-		buf = buf[:runtime.Stack(buf, true)]
-		if len(buf) < size {
-			// It fit.
-			break
-		}
-	}
-	return scrubHex(buf)
-}
+	buf := make([]byte, 1<<20)
+	buf = buf[:runtime.Stack(buf, true)]
 
-func scrubHex(buf []byte) []byte {
 	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
-
-	foreachHexAddress(buf, func(in []byte) {
+	return reHexArgs.ReplaceAllFunc(buf, func(in []byte) []byte {
 		if string(in) == "0x0" {
-			return
+			return in
 		}
 		if v, ok := saw[string(in)]; ok {
-			for i := range in {
-				in[i] = '_'
-			}
-			copy(in, v)
-			return
+			return v
 		}
-		inStr := string(in)
 		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
-		for i := range in {
-			in[i] = '_'
-		}
 		if err != nil {
-			in[0] = '?'
-			return
+			return []byte("??")
 		}
 		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
-		saw[inStr] = v
-		copy(in, v)
+		saw[string(in)] = v
+		return v
 	})
-	return buf
-}
-
-var ohx = []byte("0x")
-
-// foreachHexAddress calls f with each subslice of b that matches
-// regexp `0x[0-9a-f]*`.
-func foreachHexAddress(b []byte, f func([]byte)) {
-	for len(b) > 0 {
-		i := bytes.Index(b, ohx)
-		if i == -1 {
-			return
-		}
-		b = b[i:]
-		hx := hexPrefix(b)
-		f(hx)
-		b = b[len(hx):]
-	}
-}
-
-func hexPrefix(b []byte) []byte {
-	for i, c := range b {
-		if i < 2 {
-			continue
-		}
-		if !isHexByte(c) {
-			return b[:i]
-		}
-	}
-	return b
-}
-
-func isHexByte(b byte) bool {
-	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
 }

control/controlclient/debug_test.go

@@ -9,22 +9,3 @@ import "testing"
 func TestScrubbedGoroutineDump(t *testing.T) {
 	t.Logf("Got:\n%s\n", scrubbedGoroutineDump())
 }
-
-func TestScrubHex(t *testing.T) {
-	tests := []struct {
-		in, want string
-	}{
-		{"foo", "foo"},
-		{"", ""},
-		{"0x", "?_"},
-		{"0x001 and same 0x001", "v1%1_ and same v1%1_"},
-		{"0x008 and same 0x008", "v1%0_ and same v1%0_"},
-		{"0x001 and diff 0x002", "v1%1_ and diff v2%2_"},
-	}
-	for _, tt := range tests {
-		got := scrubHex([]byte(tt.in))
-		if string(got) != tt.want {
-			t.Errorf("for input:\n%s\n\ngot:\n%s\n\nwant:\n%s\n", tt.in, got, tt.want)
-		}
-	}
-}

control/controlclient/direct.go

@@ -220,13 +220,6 @@ func packageType() string {
 		// Using tailscaled or IPNExtension?
 		exe, _ := os.Executable()
 		return filepath.Base(exe)
-	case "linux":
-		// Report whether this is in a snap.
-		// See https://snapcraft.io/docs/environment-variables
-		// We just look at two somewhat arbitrarily.
-		if os.Getenv("SNAP_NAME") != "" && os.Getenv("SNAP") != "" {
-			return "snap"
-		}
 	}
 	return ""
 }

derp/derp_server.go

@@ -36,7 +36,6 @@ import (
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
-	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
@@ -119,8 +118,6 @@ type Server struct {
 	curClients                   expvar.Int
 	curHomeClients               expvar.Int // ones with preferred
 	clientsReplaced              expvar.Int
-	clientsReplaceLimited        expvar.Int
-	clientsReplaceSleeping       expvar.Int
 	unknownFrames                expvar.Int
 	homeMovesIn                  expvar.Int // established clients announce home server moves in
 	homeMovesOut                 expvar.Int // established clients announce home server moves out
@@ -167,7 +164,7 @@ type PacketForwarder interface {
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
 // It is a defined type so that non-net connections can be used.
 type Conn interface {
-	io.WriteCloser
+	io.Closer
 
 	// The *Deadline methods follow the semantics of net.Conn.
 
@@ -349,28 +346,14 @@ func (s *Server) initMetacert() {
 func (s *Server) MetaCert() []byte { return s.metaCert }
 
 // registerClient notes that client c is now authenticated and ready for packets.
-//
-// If c's public key was already connected with a different
-// connection, the prior one is closed, unless it's fighting rapidly
-// with another client with the same key, in which case the returned
-// ok is false, and the caller should wait the provided duration
-// before trying again.
-func (s *Server) registerClient(c *sclient) (ok bool, d time.Duration) {
+// If c's public key was already connected with a different connection, the prior one is closed.
+func (s *Server) registerClient(c *sclient) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	old := s.clients[c.key]
 	if old == nil {
 		c.logf("adding connection")
 	} else {
-		// Take over the old rate limiter, discarding the one
-		// our caller just made.
-		c.replaceLimiter = old.replaceLimiter
-		if rr := c.replaceLimiter.ReserveN(timeNow(), 1); rr.OK() {
-			if d := rr.DelayFrom(timeNow()); d > 0 {
-				s.clientsReplaceLimited.Add(1)
-				return false, d
-			}
-		}
 		s.clientsReplaced.Add(1)
 		c.logf("adding connection, replacing %s", old.remoteAddr)
 		go old.nc.Close()
@@ -382,7 +365,6 @@ func (s *Server) registerClient(c *sclient) (ok bool, d time.Duration) {
 	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
-	return true, 0
 }
 
 // broadcastPeerStateChangeLocked enqueues a message to all watchers
@@ -470,9 +452,8 @@ func (s *Server) addWatcher(c *sclient) {
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
-	br := brw.Reader
+	br, bw := brw.Reader, brw.Writer
 	nc.SetDeadline(time.Now().Add(10 * time.Second))
-	bw := &lazyBufioWriter{w: nc, lbw: brw.Writer}
 	if err := s.sendServerKey(bw); err != nil {
 		return fmt.Errorf("send server key: %v", err)
 	}
@@ -509,14 +490,8 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
 		peerGone:       make(chan key.Public),
 		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
-
-		// Allow kicking out previous connections once a
-		// minute, with a very high burst of 100. Once a
-		// minute is less than the client's 2 minute
-		// inactivity timeout.
-		replaceLimiter: rate.NewLimiter(rate.Every(time.Minute), 100),
+		flows:          make(map[key.Public]*flow),
 	}
-
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
 	}
@@ -524,18 +499,10 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		c.info = *clientInfo
 	}
 
-	for {
-		ok, d := s.registerClient(c)
-		if ok {
-			break
-		}
-		s.clientsReplaceSleeping.Add(1)
-		timeSleep(d)
-		s.clientsReplaceSleeping.Add(-1)
-	}
+	s.registerClient(c)
 	defer s.unregisterClient(c)
 
-	err = s.sendServerInfo(c.bw, clientKey)
+	err = s.sendServerInfo(bw, clientKey)
 	if err != nil {
 		return fmt.Errorf("send server info: %v", err)
 	}
@@ -543,12 +510,6 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	return c.run(ctx)
 }
 
-// for testing
-var (
-	timeSleep = time.Sleep
-	timeNow   = time.Now
-)
-
 // run serves the client until there's an error.
 // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error.
 func (c *sclient) run(ctx context.Context) error {
@@ -576,6 +537,8 @@ func (c *sclient) run(ctx context.Context) error {
 			}
 			return fmt.Errorf("client %x: readFrameHeader: %w", c.key, err)
 		}
+		// read c.crapIsDirty atomic bool, do some flow cleanup
+
 		switch ft {
 		case frameNotePreferred:
 			err = c.handleFrameNotePreferred(ft, fl)
@@ -705,14 +668,41 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	}
 
 	var fwd PacketForwarder
-	s.mu.Lock()
-	dst := s.clients[dstKey]
-	if dst == nil {
-		fwd = s.clientsMesh[dstKey]
-	} else {
-		s.notePeerSendLocked(c.key, dst)
+	var dst *sclient
+
+	var f *flow
+	var ok bool
+	if f, ok = c.flows[dstKey]; ok {
+		if f.dst != nil {
+			select {
+			case <-f.dst.done:
+				flow.close()
+			default:
+				dst = f.dst
+			}
+		}
 	}
-	s.mu.Unlock()
+	if dst == nil {
+		s.mu.Lock()
+		dst = s.clients[dstKey]
+		if dst == nil {
+			fwd = s.clientsMesh[dstKey]
+		} else {
+			s.notePeerSendLocked(c.key, dst)
+		}
+		s.mu.Unlock()
+	}
+	if f == nil {
+		f = &flow{
+			c:   c,
+			dst: dst,
+		}
+		f.idleTimer = time.AfterFunc(f.onIdle, time.Minute)
+		c.flows[dstKey] = f
+	}
+	f.pkts++
+	f.bytes += int64(len(contents))
+	f.markActive()
 
 	if dst == nil {
 		if fwd != nil {
@@ -738,7 +728,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 // dropReason is why we dropped a DERP frame.
 type dropReason int
 
-//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason
+//go:generate stringer -type=dropReason -trimprefix=dropReason
 
 const (
 	dropReasonUnknownDest      dropReason = iota // unknown destination pubkey
@@ -847,20 +837,18 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	return nil
 }
 
-func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
+func (s *Server) sendServerKey(bw *bufio.Writer) error {
 	buf := make([]byte, 0, len(magic)+len(s.publicKey))
 	buf = append(buf, magic...)
 	buf = append(buf, s.publicKey[:]...)
-	err := writeFrame(lw.bw(), frameServerKey, buf)
-	lw.Flush() // redundant (no-op) flush to release bufio.Writer
-	return err
+	return writeFrame(bw, frameServerKey, buf)
 }
 
 type serverInfo struct {
 	Version int `json:"version,omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
+func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 	var nonce [24]byte
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
@@ -871,7 +859,7 @@ func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error
 	}
 
 	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
+	if err := writeFrameHeader(bw, frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(nonce[:]); err != nil {
@@ -974,6 +962,26 @@ func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, d
 	return srcKey, dstKey, contents, nil
 }
 
+type flow struct {
+	c   *sclient // source of flow
+	dst *sclient // non-nil for local client; nil means use forwarder
+
+	// Various stats:
+	pkts  int64
+	bytes int64
+
+	lastLog   time.Time // etc
+	idleTimer *time.Timer
+}
+
+func (f *flow) markActive() {
+	f.idleTimer.Reset(time.Minute)
+}
+
+func (f *flow) onIdle() {
+	panic("TODOXXX")
+}
+
 // sclient is a client connection to the server.
 //
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
@@ -994,18 +1002,14 @@ type sclient struct {
 	meshUpdate     chan struct{}   // write request to write peerStateChange
 	canMesh        bool            // clientInfo had correct mesh token for inter-region routing
 
-	// replaceLimiter controls how quickly two connections with
-	// the same client key can kick each other off the server by
-	// taking over ownership of a key.
-	replaceLimiter *rate.Limiter
-
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
 	connectedAt time.Time
 	preferred   bool
+	flows       map[key.Public]*flow
 
 	// Owned by sender, not thread-safe.
-	bw *lazyBufioWriter
+	bw *bufio.Writer
 
 	// Guarded by s.mu
 	//
@@ -1167,14 +1171,14 @@ func (c *sclient) setWriteDeadline() {
 // sendKeepAlive sends a keep-alive frame, without flushing.
 func (c *sclient) sendKeepAlive() error {
 	c.setWriteDeadline()
-	return writeFrameHeader(c.bw.bw(), frameKeepAlive, 0)
+	return writeFrameHeader(c.bw, frameKeepAlive, 0)
 }
 
 // sendPeerGone sends a peerGone frame, without flushing.
 func (c *sclient) sendPeerGone(peer key.Public) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw, framePeerGone, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1184,7 +1188,7 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 // sendPeerPresent sends a peerPresent frame, without flushing.
 func (c *sclient) sendPeerPresent(peer key.Public) error {
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1257,11 +1261,11 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	if withKey {
 		pktLen += len(srcKey)
 	}
-	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
+	if err = writeFrameHeader(c.bw, frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw.bw(), &srcKey)
+		err := writePublicKey(c.bw, &srcKey)
 		if err != nil {
 			return err
 		}
@@ -1398,8 +1402,6 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
-	m.Set("clients_replace_limited", &s.clientsReplaceLimited)
-	m.Set("gauge_clients_replace_sleeping", &s.clientsReplaceSleeping)
 	m.Set("bytes_received", &s.bytesRecv)
 	m.Set("bytes_sent", &s.bytesSent)
 	m.Set("packets_dropped", &s.packetsDropped)
@@ -1576,45 +1578,3 @@ func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(minTimeBetweenLogs)
 	}
 }
-
-var bufioWriterPool = &sync.Pool{
-	New: func() interface{} {
-		return bufio.NewWriterSize(ioutil.Discard, 2<<10)
-	},
-}
-
-// lazyBufioWriter is a bufio.Writer-like wrapping writer that lazily
-// allocates its actual bufio.Writer from a sync.Pool, releasing it to
-// the pool upon flush.
-//
-// We do this to reduce memory overhead; most DERP connections are
-// idle and the idle bufio.Writers were 30% of overall memory usage.
-type lazyBufioWriter struct {
-	w   io.Writer     // underlying
-	lbw *bufio.Writer // lazy; nil means it needs an associated buffer
-}
-
-func (w *lazyBufioWriter) bw() *bufio.Writer {
-	if w.lbw == nil {
-		w.lbw = bufioWriterPool.Get().(*bufio.Writer)
-		w.lbw.Reset(w.w)
-	}
-	return w.lbw
-}
-
-func (w *lazyBufioWriter) Available() int { return w.bw().Available() }
-
-func (w *lazyBufioWriter) Write(p []byte) (int, error) { return w.bw().Write(p) }
-
-func (w *lazyBufioWriter) Flush() error {
-	if w.lbw == nil {
-		return nil
-	}
-	err := w.lbw.Flush()
-
-	w.lbw.Reset(ioutil.Discard)
-	bufioWriterPool.Put(w.lbw)
-	w.lbw = nil
-
-	return err
-}

derp/derp_test.go

@@ -23,7 +23,6 @@ import (
 	"testing"
 	"time"
 
-	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -850,136 +849,6 @@ func TestClientSendPong(t *testing.T) {
 
 }
 
-func TestServerReplaceClients(t *testing.T) {
-	defer func() {
-		timeSleep = time.Sleep
-		timeNow = time.Now
-	}()
-
-	var (
-		mu     sync.Mutex
-		now    = time.Unix(123, 0)
-		sleeps int
-		slept  time.Duration
-	)
-	timeSleep = func(d time.Duration) {
-		mu.Lock()
-		defer mu.Unlock()
-		sleeps++
-		slept += d
-		now = now.Add(d)
-	}
-	timeNow = func() time.Time {
-		mu.Lock()
-		defer mu.Unlock()
-		return now
-	}
-
-	serverPrivateKey := newPrivateKey(t)
-	var logger logger.Logf = logger.Discard
-	const debug = false
-	if debug {
-		logger = t.Logf
-	}
-
-	s := NewServer(serverPrivateKey, logger)
-	defer s.Close()
-
-	priv := newPrivateKey(t)
-
-	ln, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ln.Close()
-
-	connNum := 0
-	connect := func() *Client {
-		connNum++
-		cout, err := net.Dial("tcp", ln.Addr().String())
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		cin, err := ln.Accept()
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		brwServer := bufio.NewReadWriter(bufio.NewReader(cin), bufio.NewWriter(cin))
-		go s.Accept(cin, brwServer, fmt.Sprintf("test-client-%d", connNum))
-
-		brw := bufio.NewReadWriter(bufio.NewReader(cout), bufio.NewWriter(cout))
-		c, err := NewClient(priv, cout, brw, logger)
-		if err != nil {
-			t.Fatalf("client %d: %v", connNum, err)
-		}
-		return c
-	}
-
-	wantVar := func(v *expvar.Int, want int64) {
-		t.Helper()
-		if got := v.Value(); got != want {
-			t.Errorf("got %d; want %d", got, want)
-		}
-	}
-
-	wantClosed := func(c *Client) {
-		t.Helper()
-		for {
-			m, err := c.Recv()
-			if err != nil {
-				t.Logf("got expected error: %v", err)
-				return
-			}
-			switch m.(type) {
-			case ServerInfoMessage:
-				continue
-			default:
-				t.Fatalf("client got %T; wanted an error", m)
-			}
-		}
-	}
-
-	c1 := connect()
-	waitConnect(t, c1)
-	c2 := connect()
-	waitConnect(t, c2)
-	wantVar(&s.clientsReplaced, 1)
-	wantClosed(c1)
-
-	for i := 0; i < 100+5; i++ {
-		c := connect()
-		defer c.nc.Close()
-		if s.clientsReplaceLimited.Value() == 0 && i < 90 {
-			continue
-		}
-		t.Logf("for %d: replaced=%d, limited=%d, sleeping=%d", i,
-			s.clientsReplaced.Value(),
-			s.clientsReplaceLimited.Value(),
-			s.clientsReplaceSleeping.Value(),
-		)
-	}
-
-	mu.Lock()
-	defer mu.Unlock()
-	if sleeps == 0 {
-		t.Errorf("no sleeps")
-	}
-	if slept == 0 {
-		t.Errorf("total sleep duration was 0")
-	}
-}
-
-func TestLimiter(t *testing.T) {
-	rl := rate.NewLimiter(rate.Every(time.Minute), 100)
-	for i := 0; i < 200; i++ {
-		r := rl.Reserve()
-		d := r.Delay()
-		t.Logf("i=%d, allow=%v, d=%v", i, r.OK(), d)
-	}
-}
-
 func BenchmarkSendRecv(b *testing.B) {
 	for _, size := range []int{10, 100, 1000, 10000} {
 		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })

derp/derphttp/derphttp_client.go

@@ -54,7 +54,6 @@ type Client struct {
 
 	privateKey key.Private
 	logf       logger.Logf
-	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
 	// Either url or getRegion is non-nil:
 	url       *url.URL
@@ -364,26 +363,14 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	return c.client, c.connGen, nil
 }
 
-// SetURLDialer sets the dialer to use for dialing URLs.
-// This dialer is only use for clients created with NewClient, not NewRegionClient.
-// If unset or nil, the default dialer is used.
-//
-// The primary use for this is the derper mesh mode to connect to each
-// other over a VPC network.
-func (c *Client) SetURLDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
-	c.dialer = dialer
-}
-
 func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 	host := c.url.Hostname()
-	if c.dialer != nil {
-		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
-	}
 	hostOrIP := host
+
 	dialer := netns.NewDialer()
 
 	if c.DNSCache != nil {
-		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
+		ip, _, err := c.DNSCache.LookupIP(ctx, host)
 		if err == nil {
 			hostOrIP = ip.String()
 		}

derp/dropreason_string.go

@@ -1,7 +1,3 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
 // Code generated by "stringer -type=dropReason -trimprefix=dropReason"; DO NOT EDIT.
 
 package derp

go.mod

@@ -7,8 +7,6 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
-	github.com/creack/pty v1.1.9
-	github.com/dave/jennifer v1.4.1
 	github.com/frankban/quicktest v1.13.0
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
@@ -18,10 +16,10 @@ require (
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
-	github.com/iancoleman/strcase v0.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
+	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
@@ -31,8 +29,7 @@ require (
 	github.com/pkg/sftp v1.13.0
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
-	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 // indirect
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
@@ -46,7 +43,7 @@ require (
 	golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0
 	golang.zx2c4.com/wireguard/windows v0.3.16
 	honnef.co/go/tools v0.1.4
-	inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1
+	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
 	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756

go.sum

@@ -82,13 +82,12 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/daixiang0/gci v0.2.7 h1:bosLNficubzJZICsVzxuyNc6oAbdz0zcqLG2G/RxtY4=
 github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
-github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
-github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -297,8 +296,6 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
-github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
-github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -358,6 +355,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -581,10 +580,12 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
-github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
-github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
-github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
+github.com/tailscale/goupnp v1.0.1-0.20210629174436-7df6c9efe30c h1:F+pROyGPs+9wdB7jBPHr9IZEF8SKj9YUCFFShnyLNZM=
+github.com/tailscale/goupnp v1.0.1-0.20210629174436-7df6c9efe30c/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683 h1:ZXmZQuVebYllEJL/dpttpIDGx723ezC5GJkHIu0YKrM=
+github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 h1:AIJ8AF9O7jBmCwilP0ydwJMIzW5dw48Us8f3hLJhYBY=
+github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -691,6 +692,7 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -737,6 +739,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
@@ -955,8 +958,8 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
-inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
+inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=

ipn/ipnlocal/local.go

@@ -95,7 +95,7 @@ type LocalBackend struct {
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
 
-	filterHash deephash.Sum
+	filterHash string
 
 	// The mutex protects the following elements.
 	mu             sync.Mutex
@@ -179,7 +179,6 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		gotPortPollRes: make(chan struct{}),
 	}
 	b.statusChanged = sync.NewCond(&b.statusLock)
-	b.e.SetStatusCallback(b.setWgengineStatus)
 
 	linkMon := e.GetLinkMonitor()
 	b.prevIfState = linkMon.InterfaceState()
@@ -215,15 +214,6 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
-// b.mu must be held.
-func (b *LocalBackend) maybePauseControlClientLocked() {
-	if b.cc == nil {
-		return
-	}
-	networkUp := b.prevIfState.AnyInterfaceUp()
-	b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
-}
-
 // linkChange is our link monitor callback, called whenever the network changes.
 // major is whether ifst is different than earlier.
 func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
@@ -232,7 +222,11 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	hadPAC := b.prevIfState.HasPAC()
 	b.prevIfState = ifst
-	b.maybePauseControlClientLocked()
+
+	networkUp := ifst.AnyInterfaceUp()
+	if b.cc != nil {
+		go b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
+	}
 
 	// If the PAC-ness of the network changed, reconfig wireguard+route to
 	// add/remove subnets.
@@ -611,8 +605,8 @@ func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 
 	if cc != nil {
 		cc.UpdateEndpoints(0, s.LocalAddrs)
-		b.stateMachine()
 	}
+	b.stateMachine()
 
 	b.statusLock.Lock()
 	b.statusChanged.Broadcast()
@@ -869,6 +863,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	cc.SetStatusFunc(b.setClientStatus)
+	b.e.SetStatusCallback(b.setWgengineStatus)
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
 	b.mu.Lock()
@@ -944,7 +939,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets, _ := localNetsB.IPSet()
 	logNets, _ := logNetsB.IPSet()
 
-	changed := deephash.Update(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -984,44 +979,6 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 	tsaddr.TailscaleULARange(),
 }
 
-// internalAndExternalInterfaces splits interface routes into "internal"
-// and "external" sets. Internal routes are those of virtual ethernet
-// network interfaces used by guest VMs and containers, such as WSL and
-// Docker.
-//
-// Given that "internal" routes don't leave the device, we choose to
-// trust them more, allowing access to them when an Exit Node is enabled.
-func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
-	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
-		if tsaddr.IsTailscaleIP(pfx.IP()) {
-			return
-		}
-		if pfx.IsSingleIP() {
-			return
-		}
-		if runtime.GOOS == "windows" {
-			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
-			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
-			//
-			// This includes WSL2 vEthernet.
-			// Importantly: by default WSL2 /etc/resolv.conf points to
-			// a stub resolver running on the host vEthernet IP.
-			// So enabling exit nodes with the default tailnet
-			// configuration breaks WSL2 DNS without this.
-			mac := iface.Interface.HardwareAddr
-			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
-				internal = append(internal, pfx)
-				return
-			}
-		}
-		external = append(external, pfx)
-	}); err != nil {
-		return nil, nil, err
-	}
-
-	return internal, external, nil
-}
-
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
@@ -2157,21 +2114,18 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
-		internalIPs, externalIPs, err := internalAndExternalInterfaces()
-		if err != nil {
-			b.logf("failed to discover interface ips: %v", err)
-		}
 		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
-			rs.LocalRoutes = internalIPs // unconditionally allow access to guest VM networks
+			// Only allow local lan access on linux machines for now.
+			ips, _, err := interfaceRoutes()
+			if err != nil {
+				b.logf("failed to discover interface ips: %v", err)
+			}
 			if prefs.ExitNodeAllowLANAccess {
-				rs.LocalRoutes = append(rs.LocalRoutes, externalIPs...)
-				if len(externalIPs) != 0 {
-					b.logf("allowing exit node access to internal IPs: %v", internalIPs)
-				}
+				rs.LocalRoutes = ips.Prefixes()
 			} else {
 				// Explicitly add routes to the local network so that we do not
 				// leak any traffic.
-				rs.Routes = append(rs.Routes, externalIPs...)
+				rs.Routes = append(rs.Routes, ips.Prefixes()...)
 			}
 		}
 	}
@@ -2200,18 +2154,6 @@ func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *ipn.Prefs) {
 	}
 	if v := prefs.OSVersion; v != "" {
 		hi.OSVersion = v
-
-		// The Android app annotates when Google Play Services
-		// aren't available by tacking on a string to the
-		// OSVersion. Promote that to the Hostinfo.Package
-		// field instead, rather than adding a new pref, as
-		// this applyPrefsToHostinfo mechanism is mostly
-		// abused currently. TODO(bradfitz): instead let
-		// frontends update Hostinfo, without using Prefs.
-		if runtime.GOOS == "android" && strings.HasSuffix(v, " [nogoogle]") {
-			hi.Package = "nogoogle"
-			hi.OSVersion = strings.TrimSuffix(v, " [nogoogle]")
-		}
 	}
 	if m := prefs.DeviceModel; m != "" {
 		hi.DeviceModel = m
@@ -2231,7 +2173,9 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	oldState := b.state
 	b.state = newState
 	prefs := b.prefs
+	cc := b.cc
 	netMap := b.netMap
+	networkUp := b.prevIfState.AnyInterfaceUp()
 	activeLogin := b.activeLogin
 	authURL := b.authURL
 	if newState == ipn.Running {
@@ -2241,7 +2185,6 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		// Transitioning away from running.
 		b.closePeerAPIListenersLocked()
 	}
-	b.maybePauseControlClientLocked()
 	b.mu.Unlock()
 
 	if oldState == newState {
@@ -2252,6 +2195,10 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
+	if cc != nil {
+		cc.SetPaused((newState == ipn.Stopped && netMap != nil) || !networkUp)
+	}
+
 	switch newState {
 	case ipn.NeedsLogin:
 		systemd.Status("Needs login: %s", authURL)
@@ -2512,7 +2459,6 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		b.logf("active login: %v", login)
 		b.activeLogin = login
 	}
-	b.maybePauseControlClientLocked()
 
 	// Determine if file sharing is enabled
 	fs := hasCapability(nm, tailcfg.CapabilityFileSharing)
@@ -2784,18 +2730,17 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 
-	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
+			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
+			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
 		}
 		if !on {
-			return fmt.Errorf("%s is disabled.%s", key, suffix)
+			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
 		}
 	}
 	return nil

ipn/ipnlocal/state_test.go

@@ -354,7 +354,7 @@ func TestStateMachine(t *testing.T) {
 	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		c.Assert([]string{"Shutdown", "unpause", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -389,7 +389,7 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause"})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
@@ -414,7 +414,7 @@ func TestStateMachine(t *testing.T) {
 		// We're still not logged in so there's nothing we can do
 		// with it. (And empirically, it's providing an empty list
 		// of endpoints.)
-		c.Assert([]string{"UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
 		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
 	}
@@ -440,7 +440,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, url2, false, nil)
 	{
 		// BUG: UpdateEndpoints again, this is getting silly.
-		c.Assert([]string{"UpdateEndpoints", "unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 
 		// This time, backend should emit it to the UI right away,
 		// because the UI is anxiously awaiting a new URL to visit.
@@ -470,7 +470,7 @@ func TestStateMachine(t *testing.T) {
 		// wait until it gets into Starting.
 		// TODO: (Currently this test doesn't detect that bug, but
 		// it's visible in the logs)
-		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -483,7 +483,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(1)
 	// BUG: the real controlclient sends LoginFinished with every
 	// notification while it's in StateAuthenticated, but not StateSynced.
-	// It should send it exactly once, or every time we're authenticated,
+	// We should send it exactly once, or every time we're authenticated,
 	// but the current code is brittle.
 	// (ie. I suspect it would be better to change false->true in send()
 	// below, and do the same in the real controlclient.)
@@ -492,7 +492,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
@@ -534,7 +534,7 @@ func TestStateMachine(t *testing.T) {
 		nn := notifies.drain(2)
 		// BUG: UpdateEndpoints isn't needed here.
 		// BUG: Login isn't needed here. We never logged out.
-		c.Assert([]string{"Login", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -543,8 +543,7 @@ func TestStateMachine(t *testing.T) {
 	}
 
 	// Test the fast-path frontend reconnection.
-	// This one is very finicky, so we have to force State==Running
-	// or it won't use the fast path.
+	// This one is very finicky, so we have to force State==Running.
 	// TODO: actually get to State==Running, rather than cheating.
 	//  That'll require spinning up a fake DERP server and putting it in
 	//  the netmap.
@@ -571,7 +570,7 @@ func TestStateMachine(t *testing.T) {
 	b.Logout()
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"pause", "StartLogout", "pause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
@@ -588,7 +587,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
@@ -604,7 +603,7 @@ func TestStateMachine(t *testing.T) {
 		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
-		c.Assert([]string{"StartLogout", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -618,7 +617,8 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -632,7 +632,7 @@ func TestStateMachine(t *testing.T) {
 	// I guess, since that's supposed to be synchronous.
 	{
 		notifies.drain(0)
-		c.Assert([]string{"Logout", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -646,7 +646,8 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -677,7 +678,7 @@ func TestStateMachine(t *testing.T) {
 		// BUG: We already called Shutdown(), no need to do it again.
 		// BUG: Way too soon for UpdateEndpoints.
 		// BUG: don't unpause because we're not logged in.
-		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -702,7 +703,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -742,8 +743,9 @@ func TestStateMachine(t *testing.T) {
 		//  on startup, otherwise UIs can't show the node list, login
 		//  name, etc when in state ipn.Stopped.
 		//  Arguably they shouldn't try. But they currently do.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+
 		nn := notifies.drain(2)
-		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
@@ -752,25 +754,6 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
 	}
 
-	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
-	// the first netmap. Simulate that netmap being received, after which
-	// it should pause us, to avoid wasting CPU retrieving unnecessarily
-	// additional netmap updates.
-	//
-	// TODO: really the various GUIs and prefs should be refactored to
-	//  not require the netmap structure at all when starting while
-	//  !WantRunning. That would remove the need for this (or contacting
-	//  the control server at all when stopped).
-	t.Logf("\n\nStart4 -> netmap")
-	notifies.expect(0)
-	cc.send(nil, "", true, &netmap.NetworkMap{
-		MachineStatus: tailcfg.MachineAuthorized,
-	})
-	{
-		notifies.drain(0)
-		c.Assert([]string{"pause", "pause"}, qt.DeepEquals, cc.getCalls())
-	}
-
 	// Request connection.
 	// The state machine didn't call Login() earlier, so now it needs to.
 	t.Logf("\n\nWantRunning4 -> true")
@@ -797,7 +780,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -807,27 +790,25 @@ func TestStateMachine(t *testing.T) {
 	// We want to try logging in as a different user, while Stopped.
 	// First, start the login process (without logging out first).
 	t.Logf("\n\nLoginDifferent")
-	notifies.expect(1)
+	notifies.expect(2)
 	b.StartLoginInteractive()
 	url3 := "http://localhost:1/3"
 	cc.send(nil, url3, false, nil)
 	{
-		nn := notifies.drain(1)
+		nn := notifies.drain(2)
 		// It might seem like WantRunning should switch to true here,
 		// but that would be risky since we already have a valid
 		// user account. It might try to reconnect to the old account
 		// before the new one is ready. So no change yet.
-		//
-		// Because the login hasn't yet completed, the old login
-		// is still valid, so it's correct that we stay paused.
-		c.Assert([]string{"Login", "pause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 	}
 
-	// Now, let's complete the interactive login, using a different
-	// user account than before. WantRunning changes to true after an
-	// interactive login, so we end up unpaused.
+	// Now, let's say the interactive login completed, using a different
+	// user account than before.
 	t.Logf("\n\nLoginDifferent URL visited")
 	notifies.expect(3)
 	cc.persist.LoginName = "user3"
@@ -836,13 +817,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		// BUG: pause() being called here is a bad sign.
-		//  It means that either the state machine ran at least once
-		//  with the old netmap, or it ran with the new login+netmap
-		//  and !WantRunning. But since it's a fresh and successful
-		//  new login, WantRunning is true, so there was never a
-		//  reason to pause().
-		c.Assert([]string{"pause", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -861,7 +836,7 @@ func TestStateMachine(t *testing.T) {
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() ourselves.
-		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(1)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -880,7 +855,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))

ipn/ipnstate/ipnstate.go

@@ -20,7 +20,6 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/util/dnsname"
 )
@@ -91,7 +90,7 @@ type PeerStatus struct {
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
-	LastWrite     mono.Time // time last packet sent
+	LastWrite     time.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -321,7 +320,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	f("<tr><th>Peer</th><th>OS</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Connection</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
-	now := mono.Now()
+	now := time.Now()
 
 	var peers []*PeerStatus
 	for _, peer := range st.Peers() {
@@ -379,7 +378,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 		f("<td>")
 
 		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
+		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
 		if active {
 			if ps.Relay != "" && ps.CurAddr == "" {
 				f("relay <b>%s</b>", html.EscapeString(ps.Relay))

logpolicy/logpolicy.go

@@ -128,13 +128,6 @@ func (l logWriter) Write(buf []byte) (int, error) {
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
 func logsDir(logf logger.Logf) string {
-	if d := os.Getenv("TS_LOGS_DIR"); d != "" {
-		fi, err := os.Stat(d)
-		if err == nil && fi.IsDir() {
-			return d
-		}
-	}
-
 	// STATE_DIRECTORY is set by systemd 240+ but we support older
 	// systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237.
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")
@@ -187,10 +180,6 @@ func runningUnderSystemd() bool {
 	return false
 }
 
-func redirectStderrToLogPanics() bool {
-	return runningUnderSystemd() || os.Getenv("TS_PLEASE_PANIC") != ""
-}
-
 // tryFixLogStateLocation is a temporary fixup for
 // https://github.com/tailscale/tailscale/issues/247 . We accidentally
 // wrote logging state files to /, and then later to $CACHE_DIRECTORY
@@ -439,14 +428,9 @@ func New(collection string) *Policy {
 		c.HTTPC = &http.Client{Transport: newLogtailTransport(u.Host)}
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{
-		ReplaceStderr: redirectStderrToLogPanics(),
-	})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
-		if filchBuf.OrigStderr != nil {
-			c.Stderr = filchBuf.OrigStderr
-		}
 	}
 	lw := logtail.NewLogger(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here

logtail/logtail.go

@@ -118,10 +118,9 @@ type Logger struct {
 	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
-	explainedRaw   bool
 
 	shutdownStart chan struct{} // closed when shutdown begins
-	shutdownDone  chan struct{} // closed when shutdown complete
+	shutdownDone  chan struct{} // closd when shutdown complete
 }
 
 // SetVerbosityLevel controls the verbosity level that should be
@@ -231,14 +230,6 @@ func (l *Logger) drainPending() (res []byte) {
 			// outside of the logtail logger. Encode it.
 			// Do not add a client time, as it could have been
 			// been written a long time ago.
-			if !l.explainedRaw {
-				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
-				fmt.Fprintf(l.stderr, "RAW-STDERR: *** Lines prefixed with RAW-STDERR below bypassed logtail and probably come from a previous run of the program\n")
-				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
-				fmt.Fprintf(l.stderr, "RAW-STDERR:\n")
-				l.explainedRaw = true
-			}
-			fmt.Fprintf(l.stderr, "RAW-STDERR: %s", b)
 			b = l.encodeText(b, true)
 		}
 

net/dns/config.go

@@ -5,12 +5,9 @@
 package dns
 
 import (
-	"bufio"
-	"fmt"
 	"sort"
 
 	"inet.af/netaddr"
-	"tailscale.com/net/dns/resolver"
 	"tailscale.com/util/dnsname"
 )
 
@@ -41,20 +38,6 @@ type Config struct {
 	Hosts map[dnsname.FQDN][]netaddr.IP
 }
 
-// WriteToBufioWriter write a debug version of c for logs to w, omitting
-// spammy stuff like *.arpa entries and replacing it with a total count.
-func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
-	w.WriteString("{DefaultResolvers:")
-	resolver.WriteIPPorts(w, c.DefaultResolvers)
-
-	w.WriteString(" Routes:")
-	resolver.WriteRoutes(w, c.Routes)
-
-	fmt.Fprintf(w, " SearchDomains:%v", c.SearchDomains)
-	fmt.Fprintf(w, " Hosts:%v", len(c.Hosts))
-	w.WriteString("}")
-}
-
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {

net/dns/ini.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build windows
-
 package dns
 
 import (

net/dns/ini_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build windows
-
 package dns
 
 import (

net/dns/manager.go

@@ -5,7 +5,6 @@
 package dns
 
 import (
-	"bufio"
 	"runtime"
 	"time"
 
@@ -51,18 +50,14 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, li
 }
 
 func (m *Manager) Set(cfg Config) error {
-	m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) {
-		cfg.WriteToBufioWriter(w)
-	}))
+	m.logf("Set: %+v", cfg)
 
 	rcfg, ocfg, err := m.compileConfig(cfg)
 	if err != nil {
 		return err
 	}
 
-	m.logf("Resolvercfg: %v", logger.ArgWriter(func(w *bufio.Writer) {
-		rcfg.WriteToBufioWriter(w)
-	}))
+	m.logf("Resolvercfg: %+v", rcfg)
 	m.logf("OScfg: %+v", ocfg)
 
 	if err := m.resolver.SetConfig(rcfg); err != nil {

net/dns/manager_windows.go

@@ -293,7 +293,7 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 
 		t0 = time.Now()
-		m.logf("running ipconfig /flushdns ...")
+		m.logf("running ipconfig /registerdns ...")
 		cmd = exec.Command("ipconfig", "/flushdns")
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
 		err = cmd.Run()

net/dns/resolver/doh_test.go

@@ -44,9 +44,7 @@ func TestDoH(t *testing.T) {
 		t.Fatal("no known DoH")
 	}
 
-	f := &forwarder{
-		dohSem: make(chan struct{}, 10),
-	}
+	f := new(forwarder)
 
 	for ip := range knownDoH {
 		t.Run(ip.String(), func(t *testing.T) {
@@ -83,16 +81,3 @@ func TestDoH(t *testing.T) {
 		})
 	}
 }
-
-func TestDoHV6Fallback(t *testing.T) {
-	for ip, base := range knownDoH {
-		if ip.Is4() {
-			ip6, ok := dohV6(base)
-			if !ok {
-				t.Errorf("no v6 DoH known for %v", ip)
-			} else if !ip6.Is6() {
-				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
-			}
-		}
-	}
-}

net/dns/resolver/forwarder.go

@@ -16,8 +16,6 @@ import (
 	"math/rand"
 	"net"
 	"net/http"
-	"runtime"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -41,11 +39,6 @@ const (
 	// connections open to DNS-over-HTTPs servers. This is pretty
 	// arbitrary.
 	dohTransportTimeout = 30 * time.Second
-
-	// wellKnownHostBackupDelay is how long to artificially delay upstream
-	// DNS queries to the "fallback" DNS server IP for a known provider
-	// (e.g. how long to wait to query Google's 8.8.4.4 after 8.8.8.8).
-	wellKnownHostBackupDelay = 200 * time.Millisecond
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
@@ -125,7 +118,6 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		return
 	}
 
-	// https://datatracker.ietf.org/doc/html/rfc6891#section-6.1.2
 	opt := packet[len(packet)-optFixedBytes:]
 
 	if opt[0] != 0 {
@@ -142,8 +134,8 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		// Be conservative and don't touch unknown versions.
 		return
 	}
-	// Ignore flags in opt[6:9]
-	if binary.BigEndian.Uint16(opt[9:11]) != 0 {
+	// Ignore flags in opt[7:9]
+	if binary.BigEndian.Uint16(opt[10:12]) != 0 {
 		// RDLEN must be 0 (no variable length data). We're at the end of the
 		// packet so this should be 0 anyway)..
 		return
@@ -159,20 +151,7 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 
 type route struct {
 	Suffix    dnsname.FQDN
-	Resolvers []resolverAndDelay
-}
-
-// resolverAndDelay is an upstream DNS resolver and a delay for how
-// long to wait before querying it.
-type resolverAndDelay struct {
-	// ipp is the upstream resolver.
-	ipp netaddr.IPPort
-
-	// startDelay is an amount to delay this resolver at
-	// start. It's used when, say, there are four Google or
-	// Cloudflare DNS IPs (two IPv4 + two IPv6) and we don't want
-	// to race all four at once.
-	startDelay time.Duration
+	Resolvers []netaddr.IPPort
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
@@ -180,7 +159,6 @@ type forwarder struct {
 	logf    logger.Logf
 	linkMon *monitor.Mon
 	linkSel ForwardLinkSelector
-	dohSem  chan struct{}
 
 	ctx       context.Context    // good until Close
 	ctxCancel context.CancelFunc // closes ctx
@@ -202,18 +180,11 @@ func init() {
 }
 
 func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
-	maxDoHInFlight := 1000 // effectively unlimited
-	if runtime.GOOS == "ios" {
-		// No HTTP/2 on iOS yet (for size reasons), so DoH is
-		// pricier.
-		maxDoHInFlight = 10
-	}
 	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		linkMon:   linkMon,
 		linkSel:   linkSel,
 		responses: responses,
-		dohSem:    make(chan struct{}, maxDoHInFlight),
 	}
 	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
 	return f
@@ -224,84 +195,7 @@ func (f *forwarder) Close() error {
 	return nil
 }
 
-// resolversWithDelays maps from a set of DNS server ip:ports (currently
-// the port is always 53) to a slice of a type that included a
-// startDelay. So if ipps contains e.g. four Google DNS IPs (two IPv4
-// + twoIPv6), this function partition adds delays to some.
-func resolversWithDelays(ipps []netaddr.IPPort) []resolverAndDelay {
-	type hostAndFam struct {
-		host string // some arbitrary string representing DNS host (currently the DoH base)
-		bits uint8  // either 32 or 128 for IPv4 vs IPv6s address family
-	}
-
-	// Track how many of each known resolver host are in the list,
-	// per address family.
-	total := map[hostAndFam]int{}
-
-	rr := make([]resolverAndDelay, len(ipps))
-	for _, ipp := range ipps {
-		ip := ipp.IP()
-		if host, ok := knownDoH[ip]; ok {
-			total[hostAndFam{host, ip.BitLen()}]++
-		}
-	}
-
-	done := map[hostAndFam]int{}
-	for i, ipp := range ipps {
-		ip := ipp.IP()
-		var startDelay time.Duration
-		if host, ok := knownDoH[ip]; ok {
-			key4 := hostAndFam{host, 32}
-			key6 := hostAndFam{host, 128}
-			switch {
-			case ip.Is4():
-				if done[key4] > 0 {
-					startDelay += wellKnownHostBackupDelay
-				}
-			case ip.Is6():
-				total4 := total[key4]
-				if total4 >= 2 {
-					// If we have two IPv4 IPs of the same provider
-					// already in the set, delay the IPv6 queries
-					// until halfway through the timeout (so wait
-					// 2.5 seconds). Even the network is IPv6-only,
-					// the DoH dialer will fallback to IPv6
-					// immediately anyway.
-					startDelay = responseTimeout / 2
-				} else if total4 == 1 {
-					startDelay += wellKnownHostBackupDelay
-				}
-				if done[key6] > 0 {
-					startDelay += wellKnownHostBackupDelay
-				}
-			}
-			done[hostAndFam{host, ip.BitLen()}]++
-		}
-		rr[i] = resolverAndDelay{
-			ipp:        ipp,
-			startDelay: startDelay,
-		}
-	}
-	return rr
-}
-
-// setRoutes sets the routes to use for DNS forwarding. It's called by
-// Resolver.SetConfig on reconfig.
-//
-// The memory referenced by routesBySuffix should not be modified.
-func (f *forwarder) setRoutes(routesBySuffix map[dnsname.FQDN][]netaddr.IPPort) {
-	routes := make([]route, 0, len(routesBySuffix))
-	for suffix, ipps := range routesBySuffix {
-		routes = append(routes, route{
-			Suffix:    suffix,
-			Resolvers: resolversWithDelays(ipps),
-		})
-	}
-	// Sort from longest prefix to shortest.
-	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
-	})
-
+func (f *forwarder) setRoutes(routes []route) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.routes = routes
@@ -349,16 +243,7 @@ func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client,
 				if !strings.HasPrefix(netw, "tcp") {
 					return nil, fmt.Errorf("unexpected network %q", netw)
 				}
-				c, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
-				// If v4 failed, try an equivalent v6 also in the time remaining.
-				if err != nil && ctx.Err() == nil {
-					if ip6, ok := dohV6(urlBase); ok && ip.Is4() {
-						if c6, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip6.String(), "443")); err == nil {
-							return c6, nil
-						}
-					}
-				}
-				return c, err
+				return nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
 			},
 		},
 	}
@@ -368,22 +253,7 @@ func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client,
 
 const dohType = "application/dns-message"
 
-func (f *forwarder) releaseDoHSem() { <-f.dohSem }
-
 func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client, packet []byte) ([]byte, error) {
-	// Bound the number of HTTP requests in flight. This primarily
-	// matters for iOS where we're very memory constrained and
-	// HTTP requests are heavier on iOS where we don't include
-	// HTTP/2 for binary size reasons (as binaries on iOS linked
-	// with Go code cost memory proportional to the binary size,
-	// for reasons not fully understood).
-	select {
-	case f.dohSem <- struct{}{}:
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-	defer f.releaseDoHSem()
-
 	req, err := http.NewRequestWithContext(ctx, "POST", urlBase, bytes.NewReader(packet))
 	if err != nil {
 		return nil, err
@@ -413,19 +283,22 @@ func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client,
 //
 // send expects the reply to have the same txid as txidOut.
 //
-func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPort) ([]byte, error) {
-	ip := dst.IP()
-
+// The provided closeOnCtxDone lets send register values to Close if
+// the caller's ctx expires. This avoids send from allocating its own
+// waiting goroutine to interrupt the ReadFrom, as memory is tight on
+// iOS and we want the number of pending DNS lookups to be bursty
+// without too much associated goroutine/memory cost.
+func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
 	// Upgrade known DNS IPs to DoH (DNS-over-HTTPs).
-	if urlBase, dc, ok := f.getDoHClient(ip); ok {
-		res, err := f.sendDoH(ctx, urlBase, dc, fq.packet)
+	if urlBase, dc, ok := f.getDoHClient(dst.IP()); ok {
+		res, err := f.sendDoH(ctx, urlBase, dc, packet)
 		if err == nil || ctx.Err() != nil {
 			return res, err
 		}
-		f.logf("DoH error from %v: %v", ip, err)
+		f.logf("DoH error from %v: %v", dst.IP, err)
 	}
 
-	ln, err := f.packetListener(ip)
+	ln, err := f.packetListener(dst.IP())
 	if err != nil {
 		return nil, err
 	}
@@ -436,10 +309,10 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPo
 	}
 	defer conn.Close()
 
-	fq.closeOnCtxDone.Add(conn)
-	defer fq.closeOnCtxDone.Remove(conn)
+	closeOnCtxDone.Add(conn)
+	defer closeOnCtxDone.Remove(conn)
 
-	if _, err := conn.WriteTo(fq.packet, dst.UDPAddr()); err != nil {
+	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
@@ -468,7 +341,7 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPo
 	}
 	out = out[:n]
 	txid := getTxID(out)
-	if txid != fq.txid {
+	if txid != txidOut {
 		return nil, errors.New("txid doesn't match")
 	}
 
@@ -491,7 +364,7 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPo
 }
 
 // resolvers returns the resolvers to use for domain.
-func (f *forwarder) resolvers(domain dnsname.FQDN) []resolverAndDelay {
+func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
 	f.mu.Lock()
 	routes := f.routes
 	f.mu.Unlock()
@@ -503,30 +376,6 @@ func (f *forwarder) resolvers(domain dnsname.FQDN) []resolverAndDelay {
 	return nil
 }
 
-// forwardQuery is information and state about a forwarded DNS query that's
-// being sent to 1 or more upstreams.
-//
-// In the case of racing against multiple equivalent upstreams
-// (e.g. Google or CloudFlare's 4 DNS IPs: 2 IPv4 + 2 IPv6), this type
-// handles racing them more intelligently than just blasting away 4
-// queries at once.
-type forwardQuery struct {
-	txid   txid
-	packet []byte
-
-	// closeOnCtxDone lets send register values to Close if the
-	// caller's ctx expires. This avoids send from allocating its
-	// own waiting goroutine to interrupt the ReadFrom, as memory
-	// is tight on iOS and we want the number of pending DNS
-	// lookups to be bursty without too much associated
-	// goroutine/memory cost.
-	closeOnCtxDone *closePool
-
-	// TODO(bradfitz): add race delay state:
-	// mu sync.Mutex
-	// ...
-}
-
 // forward forwards the query to all upstream nameservers and returns the first response.
 func (f *forwarder) forward(query packet) error {
 	domain, err := nameFromQuery(query.bs)
@@ -534,6 +383,7 @@ func (f *forwarder) forward(query packet) error {
 		return err
 	}
 
+	txid := getTxID(query.bs)
 	clampEDNSSize(query.bs, maxResponseBytes)
 
 	resolvers := f.resolvers(domain)
@@ -541,12 +391,8 @@ func (f *forwarder) forward(query packet) error {
 		return errNoUpstreams
 	}
 
-	fq := &forwardQuery{
-		txid:           getTxID(query.bs),
-		packet:         query.bs,
-		closeOnCtxDone: new(closePool),
-	}
-	defer fq.closeOnCtxDone.Close()
+	closeOnCtxDone := new(closePool)
+	defer closeOnCtxDone.Close()
 
 	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
 	defer cancel()
@@ -557,18 +403,9 @@ func (f *forwarder) forward(query packet) error {
 		firstErr error
 	)
 
-	for _, rr := range resolvers {
-		go func(rr resolverAndDelay) {
-			if rr.startDelay > 0 {
-				timer := time.NewTimer(rr.startDelay)
-				select {
-				case <-timer.C:
-				case <-ctx.Done():
-					timer.Stop()
-					return
-				}
-			}
-			resb, err := f.send(ctx, fq, rr.ipp)
+	for _, ipp := range resolvers {
+		go func(ipp netaddr.IPPort) {
+			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
 			if err != nil {
 				mu.Lock()
 				defer mu.Unlock()
@@ -581,7 +418,7 @@ func (f *forwarder) forward(query packet) error {
 			case resc <- resb:
 			default:
 			}
-		}(rr)
+		}(ipp)
 	}
 
 	select {
@@ -672,22 +509,7 @@ func (p *closePool) Close() error {
 
 var knownDoH = map[netaddr.IP]string{}
 
-var dohIPsOfBase = map[string][]netaddr.IP{}
-
-func addDoH(ipStr, base string) {
-	ip := netaddr.MustParseIP(ipStr)
-	knownDoH[ip] = base
-	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
-}
-
-func dohV6(base string) (ip netaddr.IP, ok bool) {
-	for _, ip := range dohIPsOfBase[base] {
-		if ip.Is6() {
-			return ip, true
-		}
-	}
-	return ip, false
-}
+func addDoH(ip, base string) { knownDoH[netaddr.MustParseIP(ip)] = base }
 
 func init() {
 	// Cloudflare
@@ -697,16 +519,16 @@ func init() {
 	addDoH("2606:4700:4700::1001", "https://cloudflare-dns.com/dns-query")
 
 	// Cloudflare -Malware
-	addDoH("1.1.1.2", "https://security.cloudflare-dns.com/dns-query")
-	addDoH("1.0.0.2", "https://security.cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1112", "https://security.cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1002", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("1.1.1.2", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.2", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1112", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1002", "https://cloudflare-dns.com/dns-query")
 
 	// Cloudflare -Malware -Adult
-	addDoH("1.1.1.3", "https://family.cloudflare-dns.com/dns-query")
-	addDoH("1.0.0.3", "https://family.cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1113", "https://family.cloudflare-dns.com/dns-query")
-	addDoH("2606:4700:4700::1003", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("1.1.1.3", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.3", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1113", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1003", "https://cloudflare-dns.com/dns-query")
 
 	// Google
 	addDoH("8.8.8.8", "https://dns.google/dns-query")

net/dns/resolver/forwarder_test.go

@@ -1,90 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package resolver
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-
-	"inet.af/netaddr"
-)
-
-func (rr resolverAndDelay) String() string {
-	return fmt.Sprintf("%v+%v", rr.ipp, rr.startDelay)
-}
-
-func TestResolversWithDelays(t *testing.T) {
-	// query
-	q := func(ss ...string) (ipps []netaddr.IPPort) {
-		for _, s := range ss {
-			ipps = append(ipps, netaddr.MustParseIPPort(s))
-		}
-		return
-	}
-	// output
-	o := func(ss ...string) (rr []resolverAndDelay) {
-		for _, s := range ss {
-			var d time.Duration
-			if i := strings.Index(s, "+"); i != -1 {
-				var err error
-				d, err = time.ParseDuration(s[i+1:])
-				if err != nil {
-					panic(fmt.Sprintf("parsing duration in %q: %v", s, err))
-				}
-				s = s[:i]
-			}
-			rr = append(rr, resolverAndDelay{
-				ipp:        netaddr.MustParseIPPort(s),
-				startDelay: d,
-			})
-		}
-		return
-	}
-
-	tests := []struct {
-		name string
-		in   []netaddr.IPPort
-		want []resolverAndDelay
-	}{
-		{
-			name: "unknown-no-delays",
-			in:   q("1.2.3.4:53", "2.3.4.5:53"),
-			want: o("1.2.3.4:53", "2.3.4.5:53"),
-		},
-		{
-			name: "google-all-ipv4",
-			in:   q("8.8.8.8:53", "8.8.4.4:53"),
-			want: o("8.8.8.8:53", "8.8.4.4:53+200ms"),
-		},
-		{
-			name: "google-only-ipv6",
-			in:   q("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
-			want: o("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53+200ms"),
-		},
-		{
-			name: "google-all-four",
-			in:   q("8.8.8.8:53", "8.8.4.4:53", "[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
-			want: o("8.8.8.8:53", "8.8.4.4:53+200ms", "[2001:4860:4860::8888]:53+2.5s", "[2001:4860:4860::8844]:53+2.7s"),
-		},
-		{
-			name: "quad9-one-v4-one-v6",
-			in:   q("9.9.9.9:53", "[2620:fe::fe]:53"),
-			want: o("9.9.9.9:53", "[2620:fe::fe]:53+200ms"),
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := resolversWithDelays(tt.in)
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("got %v; want %v", got, tt.want)
-			}
-		})
-	}
-
-}

net/dns/resolver/tsdns.go

@@ -7,10 +7,8 @@
 package resolver
 
 import (
-	"bufio"
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"runtime"
 	"sort"
 	"strings"
@@ -81,74 +79,6 @@ type Config struct {
 	LocalDomains []dnsname.FQDN
 }
 
-// WriteToBufioWriter write a debug version of c for logs to w, omitting
-// spammy stuff like *.arpa entries and replacing it with a total count.
-func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
-	w.WriteString("{Routes:")
-	WriteRoutes(w, c.Routes)
-	fmt.Fprintf(w, " Hosts:%v LocalDomains:[", len(c.Hosts))
-	space := false
-	arpa := 0
-	for _, d := range c.LocalDomains {
-		if strings.HasSuffix(string(d), ".arpa.") {
-			arpa++
-			continue
-		}
-		if space {
-			w.WriteByte(' ')
-		}
-		w.WriteString(string(d))
-		space = true
-	}
-	w.WriteString("]")
-	if arpa > 0 {
-		fmt.Fprintf(w, "+%darpa", arpa)
-	}
-	w.WriteString("}")
-}
-
-// WriteIPPorts writes vv to w.
-func WriteIPPorts(w *bufio.Writer, vv []netaddr.IPPort) {
-	w.WriteByte('[')
-	var b []byte
-	for i, v := range vv {
-		if i > 0 {
-			w.WriteByte(' ')
-		}
-		b = v.AppendTo(b[:0])
-		w.Write(b)
-	}
-	w.WriteByte(']')
-}
-
-// WriteRoutes writes routes to w, omitting *.arpa routes and instead
-// summarizing how many of them there were.
-func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]netaddr.IPPort) {
-	var kk []dnsname.FQDN
-	arpa := 0
-	for k := range routes {
-		if strings.HasSuffix(string(k), ".arpa.") {
-			arpa++
-			continue
-		}
-		kk = append(kk, k)
-	}
-	sort.Slice(kk, func(i, j int) bool { return kk[i] < kk[j] })
-	w.WriteByte('{')
-	for i, k := range kk {
-		if i > 0 {
-			w.WriteByte(' ')
-		}
-		w.WriteString(string(k))
-		w.WriteByte(':')
-		WriteIPPorts(w, routes[k])
-	}
-	w.WriteByte('}')
-	if arpa > 0 {
-		fmt.Fprintf(w, "+%darpa", arpa)
-	}
-}
-
 // Resolver is a DNS resolver for nodes on the Tailscale network,
 // associating them with domain names of the form <mynode>.<mydomain>.<root>.
 // If it is asked to resolve a domain that is not of that form,
@@ -208,6 +138,7 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		r.saveConfigForTests(cfg)
 	}
 
+	routes := make([]route, 0, len(cfg.Routes))
 	reverse := make(map[netaddr.IP]dnsname.FQDN, len(cfg.Hosts))
 
 	for host, ips := range cfg.Hosts {
@@ -216,7 +147,18 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		}
 	}
 
-	r.forwarder.setRoutes(cfg.Routes)
+	for suffix, ips := range cfg.Routes {
+		routes = append(routes, route{
+			Suffix:    suffix,
+			Resolvers: ips,
+		})
+	}
+	// Sort from longest prefix to shortest.
+	sort.Slice(routes, func(i, j int) bool {
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
+	})
+
+	r.forwarder.setRoutes(routes)
 
 	r.mu.Lock()
 	defer r.mu.Unlock()

net/dns/resolver/tsdns_test.go

@@ -931,11 +931,8 @@ func TestAllocs(t *testing.T) {
 		query []byte
 		want  int
 	}{
-		// Name lowercasing, response slice created by dns.NewBuilder,
-		// and closure allocation from go call.
-		// (Closure allocation only happens when using new register ABI,
-		// which is amd64 with Go 1.17, and probably more platforms later.)
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 3},
+		// Name lowercasing and response slice created by dns.NewBuilder.
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
 		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}

net/dnscache/dnscache.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// TODO(bradfitz): update this code to use netaddr more
-
 // Package dnscache contains a minimal DNS cache that makes a bunch of
 // assumptions that are only valid for us. Not recommended for general use.
 package dnscache
@@ -80,9 +78,8 @@ type Resolver struct {
 }
 
 type ipCacheEntry struct {
-	ip      net.IP       // either v4 or v6
-	ip6     net.IP       // nil if no v4 or no v6
-	allIPs  []net.IPAddr // 1+ v4 and/or v6
+	ip      net.IP // either v4 or v6
+	ip6     net.IP // nil if no v4 or no v6
 	expires time.Time
 }
 
@@ -108,82 +105,81 @@ var debug, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DNS_CACHE"))
 //
 // If err is nil, ip will be non-nil. The v6 address may be nil even
 // with a nil error.
-func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, allIPs []net.IPAddr, err error) {
+func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, err error) {
 	if ip := net.ParseIP(host); ip != nil {
 		if ip4 := ip.To4(); ip4 != nil {
-			return ip4, nil, []net.IPAddr{{IP: ip4}}, nil
+			return ip4, nil, nil
 		}
 		if debug {
 			log.Printf("dnscache: %q is an IP", host)
 		}
-		return ip, nil, []net.IPAddr{{IP: ip}}, nil
+		return ip, nil, nil
 	}
 
-	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
+	if ip, ip6, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q = %v (cached)", host, ip)
 		}
-		return ip, ip6, allIPs, nil
+		return ip, ip6, nil
 	}
 
-	type ipRes struct {
+	type ipPair struct {
 		ip, ip6 net.IP
-		allIPs  []net.IPAddr
 	}
 	ch := r.sf.DoChan(host, func() (interface{}, error) {
-		ip, ip6, allIPs, err := r.lookupIP(host)
+		ip, ip6, err := r.lookupIP(host)
 		if err != nil {
 			return nil, err
 		}
-		return ipRes{ip, ip6, allIPs}, nil
+		return ipPair{ip, ip6}, nil
 	})
 	select {
 	case res := <-ch:
 		if res.Err != nil {
 			if r.UseLastGood {
-				if ip, ip6, allIPs, ok := r.lookupIPCacheExpired(host); ok {
+				if ip, ip6, ok := r.lookupIPCacheExpired(host); ok {
 					if debug {
 						log.Printf("dnscache: %q using %v after error", host, ip)
 					}
-					return ip, ip6, allIPs, nil
+					return ip, ip6, nil
 				}
 			}
 			if debug {
 				log.Printf("dnscache: error resolving %q: %v", host, res.Err)
 			}
-			return nil, nil, nil, res.Err
+			return nil, nil, res.Err
 		}
-		r := res.Val.(ipRes)
-		return r.ip, r.ip6, r.allIPs, nil
+		pair := res.Val.(ipPair)
+		return pair.ip, pair.ip6, nil
 	case <-ctx.Done():
 		if debug {
 			log.Printf("dnscache: context done while resolving %q: %v", host, ctx.Err())
 		}
-		return nil, nil, nil, ctx.Err()
+		return nil, nil, ctx.Err()
 	}
 }
 
-func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
+func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok && ent.expires.After(time.Now()) {
-		return ent.ip, ent.ip6, ent.allIPs, true
+		return ent.ip, ent.ip6, true
 	}
-	return nil, nil, nil, false
+	return nil, nil, false
 }
 
-func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
+func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok {
-		return ent.ip, ent.ip6, ent.allIPs, true
+		return ent.ip, ent.ip6, true
 	}
-	return nil, nil, nil, false
+	return nil, nil, false
 }
 
 func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	if r.UseLastGood {
-		if _, _, _, ok := r.lookupIPCacheExpired(host); ok {
+		if _, _, ok := r.lookupIPCacheExpired(host); ok {
 			// If we have some previous good value for this host,
 			// don't give this DNS lookup much time. If we're in a
 			// situation where the user's DNS server is unreachable
@@ -198,12 +194,12 @@ func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	return 10 * time.Second
 }
 
-func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, err error) {
-	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
+func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
+	if ip, ip6, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q found in cache as %v", host, ip)
 		}
-		return ip, ip6, allIPs, nil
+		return ip, ip6, nil
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), r.lookupTimeoutForHost(host))
@@ -222,10 +218,10 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, e
 		}
 	}
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 	if len(ips) == 0 {
-		return nil, nil, nil, fmt.Errorf("no IPs for %q found", host)
+		return nil, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
 	have4 := false
@@ -244,12 +240,12 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, e
 			}
 		}
 	}
-	r.addIPCache(host, ip, ip6, ips, r.ttl())
-	return ip, ip6, ips, nil
+	r.addIPCache(host, ip, ip6, r.ttl())
+	return ip, ip6, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr, d time.Duration) {
-	if naIP, _ := netaddr.FromStdIP(ip); naIP.IsPrivate() {
+func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
+	if isPrivateIP(ip) {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
 		if debug {
@@ -267,14 +263,27 @@ func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr,
 	if r.ipCache == nil {
 		r.ipCache = make(map[string]ipCacheEntry)
 	}
-	r.ipCache[host] = ipCacheEntry{
-		ip:      ip,
-		ip6:     ip6,
-		allIPs:  allIPs,
-		expires: time.Now().Add(d),
-	}
+	r.ipCache[host] = ipCacheEntry{ip: ip, ip6: ip6, expires: time.Now().Add(d)}
 }
 
+func mustCIDR(s string) *net.IPNet {
+	_, ipNet, err := net.ParseCIDR(s)
+	if err != nil {
+		panic(err)
+	}
+	return ipNet
+}
+
+func isPrivateIP(ip net.IP) bool {
+	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
+}
+
+var (
+	private1 = mustCIDR("10.0.0.0/8")
+	private2 = mustCIDR("172.16.0.0/12")
+	private3 = mustCIDR("192.168.0.0/16")
+)
+
 type DialContextFunc func(ctx context.Context, network, address string) (net.Conn, error)
 
 // Dialer returns a wrapped DialContext func that uses the provided dnsCache.
@@ -296,129 +305,39 @@ func Dialer(fwd DialContextFunc, dnsCache *Resolver) DialContextFunc {
 				// Return with original error
 				return
 			}
-			if c, err := raceDial(ctx, fwd, network, ips, port); err == nil {
-				retConn = c
-				ret = nil
-				return
-			}
-		}()
-
-		ip, ip6, allIPs, err := dnsCache.LookupIP(ctx, host)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve %q: %w", host, err)
-		}
-		i4s := v4addrs(allIPs)
-		if len(i4s) < 2 {
-			dst := net.JoinHostPort(ip.String(), port)
-			if debug {
-				log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
-			}
-			c, err := fwd(ctx, network, dst)
-			if err == nil || ctx.Err() != nil || ip6 == nil {
-				return c, err
-			}
-			// Fall back to trying IPv6.
-			dst = net.JoinHostPort(ip6.String(), port)
-			return fwd(ctx, network, dst)
-		}
-
-		// Multiple IPv4 candidates, and 0+ IPv6.
-		ipsToTry := append(i4s, v6addrs(allIPs)...)
-		return raceDial(ctx, fwd, network, ipsToTry, port)
-	}
-}
-
-// fallbackDelay is how long to wait between trying subsequent
-// addresses when multiple options are available.
-// 300ms is the same as Go's Happy Eyeballs fallbackDelay value.
-const fallbackDelay = 300 * time.Millisecond
-
-// raceDial tries to dial port on each ip in ips, starting a new race
-// dial every fallbackDelay apart, returning whichever completes first.
-func raceDial(ctx context.Context, fwd DialContextFunc, network string, ips []netaddr.IP, port string) (net.Conn, error) {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	type res struct {
-		c   net.Conn
-		err error
-	}
-	resc := make(chan res)           // must be unbuffered
-	failBoost := make(chan struct{}) // best effort send on dial failure
-
-	go func() {
-		for i, ip := range ips {
-			if i != 0 {
-				timer := time.NewTimer(fallbackDelay)
-				select {
-				case <-timer.C:
-				case <-failBoost:
-					timer.Stop()
-				case <-ctx.Done():
-					timer.Stop()
+			for _, ip := range ips {
+				dst := net.JoinHostPort(ip.String(), port)
+				if c, err := fwd(ctx, network, dst); err == nil {
+					retConn = c
+					ret = nil
 					return
 				}
 			}
-			go func(ip netaddr.IP) {
-				c, err := fwd(ctx, network, net.JoinHostPort(ip.String(), port))
-				if err != nil {
-					// Best effort wake-up a pending dial.
-					// e.g. IPv4 dials failing quickly on an IPv6-only system.
-					// In that case we don't want to wait 300ms per IPv4 before
-					// we get to the IPv6 addresses.
-					select {
-					case failBoost <- struct{}{}:
-					default:
-					}
-				}
-				select {
-				case resc <- res{c, err}:
-				case <-ctx.Done():
-					if c != nil {
-						c.Close()
-					}
-				}
-			}(ip)
-		}
-	}()
+		}()
 
-	var firstErr error
-	var fails int
-	for {
-		select {
-		case r := <-resc:
-			if r.c != nil {
-				return r.c, nil
-			}
-			fails++
-			if firstErr == nil {
-				firstErr = r.err
-			}
-			if fails == len(ips) {
-				return nil, firstErr
-			}
-		case <-ctx.Done():
-			return nil, ctx.Err()
+		ip, ip6, err := dnsCache.LookupIP(ctx, host)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve %q: %w", host, err)
 		}
-	}
-}
-
-func v4addrs(aa []net.IPAddr) (ret []netaddr.IP) {
-	for _, a := range aa {
-		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is4() {
-			ret = append(ret, ip)
+		dst := net.JoinHostPort(ip.String(), port)
+		if debug {
+			log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
 		}
-	}
-	return ret
-}
-
-func v6addrs(aa []net.IPAddr) (ret []netaddr.IP) {
-	for _, a := range aa {
-		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is6() {
-			ret = append(ret, ip)
+		c, err := fwd(ctx, network, dst)
+		if err == nil || ctx.Err() != nil || ip6 == nil {
+			return c, err
 		}
+		// Fall back to trying IPv6.
+		// TODO(bradfitz): this is a primarily for IPv6-only
+		// hosts; it's not supposed to be a real Happy
+		// Eyeballs implementation. We should use the net
+		// package's implementation of that by plumbing this
+		// dnscache impl into net.Dialer.Resolver.Dial and
+		// unmarshal/marshal DNS queries/responses to the net
+		// package. This works for v6-only hosts for now.
+		dst = net.JoinHostPort(ip6.String(), port)
+		return fwd(ctx, network, dst)
 	}
-	return ret
 }
 
 var errTLSHandshakeTimeout = errors.New("timeout doing TLS handshake")

net/dnscache/dnscache_test.go

@@ -5,29 +5,24 @@
 package dnscache
 
 import (
-	"context"
-	"flag"
 	"net"
 	"testing"
-	"time"
 )
 
-var dialTest = flag.String("dial-test", "", "if non-empty, addr:port to test dial")
+func TestIsPrivateIP(t *testing.T) {
+	tests := []struct {
+		ip   string
+		want bool
+	}{
+		{"10.1.2.3", true},
+		{"172.16.1.100", true},
+		{"192.168.1.1", true},
+		{"1.2.3.4", false},
+	}
 
-func TestDialer(t *testing.T) {
-	if *dialTest == "" {
-		t.Skip("skipping; --dial-test is blank")
+	for _, test := range tests {
+		if got := isPrivateIP(net.ParseIP(test.ip)); got != test.want {
+			t.Errorf("isPrivateIP(%q)=%v, want %v", test.ip, got, test.want)
+		}
 	}
-	r := new(Resolver)
-	var std net.Dialer
-	dialer := Dialer(std.DialContext, r)
-	t0 := time.Now()
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	c, err := dialer(ctx, "tcp", *dialTest)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Logf("dialed in %v", time.Since(t0))
-	c.Close()
 }

net/dnsfallback/dns-fallback-servers.json

@@ -90,25 +90,18 @@
 			"RegionName": "r4",
 			"Nodes": [
 				{
-					"Name": "4c",
+					"Name": "4a",
 					"RegionID": 4,
-					"HostName": "derp4c.tailscale.com",
-					"IPv4": "134.122.77.138",
-					"IPv6": "2a03:b0c0:3:d0::1501:6001"
+					"HostName": "derp4.tailscale.com",
+					"IPv4": "167.172.182.26",
+					"IPv6": "2a03:b0c0:3:e0::36e:9001"
 				},
 				{
-					"Name": "4d",
+					"Name": "4b",
 					"RegionID": 4,
-					"HostName": "derp4d.tailscale.com",
-					"IPv4": "134.122.94.167",
-					"IPv6": "2a03:b0c0:3:d0::1501:b001"
-				},
-				{
-					"Name": "4e",
-					"RegionID": 4,
-					"HostName": "derp4e.tailscale.com",
-					"IPv4": "134.122.74.153",
-					"IPv6": "2a03:b0c0:3:d0::29:9001"
+					"HostName": "derp4b.tailscale.com",
+					"IPv4": "157.230.25.0",
+					"IPv6": "2a03:b0c0:3:e0::58f:3001"
 				}
 			]
 		},
@@ -160,25 +153,11 @@
 			"RegionName": "r8",
 			"Nodes": [
 				{
-					"Name": "8b",
+					"Name": "8a",
 					"RegionID": 8,
-					"HostName": "derp8b.tailscale.com",
-					"IPv4": "46.101.74.201",
-					"IPv6": "2a03:b0c0:1:d0::ec1:e001"
-				},
-				{
-					"Name": "8c",
-					"RegionID": 8,
-					"HostName": "derp8c.tailscale.com",
-					"IPv4": "206.189.16.32",
-					"IPv6": "2a03:b0c0:1:d0::e1f:4001"
-				},
-				{
-					"Name": "8d",
-					"RegionID": 8,
-					"HostName": "derp8d.tailscale.com",
-					"IPv4": "178.62.44.132",
-					"IPv6": "2a03:b0c0:1:d0::e08:e001"
+					"HostName": "derp8.tailscale.com",
+					"IPv4": "167.71.139.179",
+					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
 				}
 			]
 		},

net/interfaces/interfaces.go

@@ -134,7 +134,7 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 					// but their OS supports IPv6 so they have an fe80::
 					// address. We don't want to report all of those
 					// IPv6 LL to Control.
-				} else if ip.Is6() && ip.IsPrivate() {
+				} else if ip.Is6() && tsaddr.IsULA(ip) {
 					// Google Cloud Run uses NAT with IPv6 Unique
 					// Local Addresses to provide IPv6 connectivity.
 					ula6 = append(ula6, ip)
@@ -479,7 +479,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
 		ip := pfx.IP()
-		if ip.IsPrivate() {
+		if isPrivateIP(ip) {
 			if privateIP == "" {
 				privateIP = ip.String()
 			}
@@ -519,15 +519,21 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
-		if gateway.IsPrivate() && ip.IsPrivate() {
-			myIP = ip
-			ok = true
-			return
+		for _, prefix := range privatev4s {
+			if prefix.Contains(gateway) && prefix.Contains(ip) {
+				myIP = ip
+				ok = true
+				return
+			}
 		}
 	})
 	return gateway, myIP, !myIP.IsZero()
 }
 
+func isPrivateIP(ip netaddr.IP) bool {
+	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
+}
+
 // isUsableV4 reports whether ip is a usable IPv4 address which could
 // conceivably be used to get Internet connectivity. Globally routable and
 // private IPv4 addresses are always Usable, and link local 169.254.x.x
@@ -548,11 +554,23 @@ func isUsableV4(ip netaddr.IP) bool {
 // (fc00::/7) are in some environments used with address translation.
 func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
-		(ip.Is6() && ip.IsPrivate() && !tsaddr.TailscaleULARange().Contains(ip))
+		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
+}
+
+func mustCIDR(s string) netaddr.IPPrefix {
+	prefix, err := netaddr.ParseIPPrefix(s)
+	if err != nil {
+		panic(err)
+	}
+	return prefix
 }
 
 var (
-	v6Global1 = netaddr.MustParseIPPrefix("2000::/3")
+	private1   = mustCIDR("10.0.0.0/8")
+	private2   = mustCIDR("172.16.0.0/12")
+	private3   = mustCIDR("192.168.0.0/16")
+	privatev4s = []netaddr.IPPrefix{private1, private2, private3}
+	v6Global1  = mustCIDR("2000::/3")
 )
 
 // anyInterestingIP reports whether pfxs contains any IP that matches

net/interfaces/interfaces_darwin_test.go

@@ -73,7 +73,7 @@ func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
 			return nil
 		}
 		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
-		if err == nil && ip.IsPrivate() {
+		if err == nil && isPrivateIP(ip) {
 			ret = ip
 			// We've found what we're looking for.
 			return errStopReadingNetstatTable

net/interfaces/interfaces_linux.go

@@ -72,7 +72,7 @@ func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
 			return nil // ignore error, skip line and keep going
 		}
 		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
-		if ip.IsPrivate() {
+		if isPrivateIP(ip) {
 			ret = ip
 		}
 		return nil

net/interfaces/interfaces_test.go

@@ -58,8 +58,6 @@ func TestIsUsableV6(t *testing.T) {
 		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
 		{"Link Local", "fe80::1", false},
 		{"Global", "2602::1", true},
-		{"IPv4 public", "192.0.2.1", false},
-		{"IPv4 private", "192.168.1.1", false},
 	}
 
 	for _, test := range tests {

net/interfaces/interfaces_windows.go

@@ -93,7 +93,7 @@ func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
 		}
 	}
 
-	if !ret.IsZero() && !ret.IsPrivate() {
+	if !ret.IsZero() && !isPrivateIP(ret) {
 		// Default route has a non-private gateway
 		return netaddr.IP{}, false
 	}

net/portmapper/disabled_stubs.go

@@ -1,25 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ios
-// (https://github.com/tailscale/tailscale/issues/2495)
-
-package portmapper
-
-import (
-	"context"
-
-	"inet.af/netaddr"
-)
-
-type upnpClient interface{}
-
-func (c *Client) getUPnPPortMapping(
-	ctx context.Context,
-	gw netaddr.IP,
-	internal netaddr.IPPort,
-	prevPort uint16,
-) (external netaddr.IPPort, ok bool) {
-	return netaddr.IPPort{}, false
-}

net/portmapper/portmapper.go

@@ -14,11 +14,9 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"net/http"
 	"sync"
 	"time"
 
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netns"
@@ -64,11 +62,8 @@ type Client struct {
 	pmpPubIPTime time.Time  // time pmpPubIP last verified
 	pmpLastEpoch uint32
 
-	pcpSawTime time.Time // time we last saw PCP was available
-
-	uPnPSawTime    time.Time         // time we last saw UPnP was available
-	uPnPMeta       uPnPDiscoResponse // Location header from UPnP UDP discovery response
-	uPnPHTTPClient *http.Client      // nil until needed
+	pcpSawTime  time.Time // time we last saw PCP was available
+	uPnPSawTime time.Time // time we last saw UPnP was available
 
 	localPort uint16
 
@@ -275,7 +270,7 @@ func IsNoMappingError(err error) bool {
 
 var (
 	ErrNoPortMappingServices = errors.New("no port mapping services were found")
-	ErrGatewayRange          = errors.New("skipping portmap; gateway range likely lacks support")
+	ErrGatewayNotFound       = errors.New("failed to look up gateway address")
 )
 
 // GetCachedMappingOrStartCreatingOne quickly returns with our current cached portmapping, if any.
@@ -336,7 +331,7 @@ func (c *Client) createMapping() {
 func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return netaddr.IPPort{}, NoMappingError{ErrGatewayRange}
+		return netaddr.IPPort{}, NoMappingError{ErrGatewayNotFound}
 	}
 
 	c.mu.Lock()
@@ -545,7 +540,7 @@ type ProbeResult struct {
 func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return res, ErrGatewayRange
+		return res, ErrGatewayNotFound
 	}
 	defer func() {
 		if err == nil {
@@ -565,9 +560,27 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	defer cancel()
 	defer closeCloserOnContextDone(ctx, uc)()
 
+	if c.sawUPnPRecently() {
+		res.UPnP = true
+	} else {
+		hasUPnP := make(chan bool, 1)
+		defer func() {
+			res.UPnP = <-hasUPnP
+		}()
+		go func() {
+			client, err := getUPnPClient(ctx, gw)
+			if err == nil && client != nil {
+				hasUPnP <- true
+				c.mu.Lock()
+				c.uPnPSawTime = time.Now()
+				c.mu.Unlock()
+			}
+			close(hasUPnP)
+		}()
+	}
+
 	pcpAddr := netaddr.IPPortFrom(gw, pcpPort).UDPAddr()
 	pmpAddr := netaddr.IPPortFrom(gw, pmpPort).UDPAddr()
-	upnpAddr := netaddr.IPPortFrom(gw, upnpPort).UDPAddr()
 
 	// Don't send probes to services that we recently learned (for
 	// the same gw/myIP) are available. See
@@ -582,16 +595,11 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	} else {
 		uc.WriteTo(pcpAnnounceRequest(myIP), pcpAddr)
 	}
-	if c.sawUPnPRecently() {
-		res.UPnP = true
-	} else {
-		uc.WriteTo(uPnPPacket, upnpAddr)
-	}
 
 	buf := make([]byte, 1500)
 	pcpHeard := false // true when we get any PCP response
 	for {
-		if pcpHeard && res.PMP && res.UPnP {
+		if pcpHeard && res.PMP {
 			// Nothing more to discover.
 			return res, nil
 		}
@@ -604,19 +612,6 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 		}
 		port := addr.(*net.UDPAddr).Port
 		switch port {
-		case upnpPort:
-			if mem.Contains(mem.B(buf[:n]), mem.S(":InternetGatewayDevice:")) {
-				meta, err := parseUPnPDiscoResponse(buf[:n])
-				if err != nil {
-					c.logf("unrecognized UPnP discovery response; ignoring")
-				}
-				// log.Printf("UPnP reply %+v, %q", meta, buf[:n])
-				res.UPnP = true
-				c.mu.Lock()
-				c.uPnPSawTime = time.Now()
-				c.uPnPMeta = meta
-				c.mu.Unlock()
-			}
 		case pcpPort: // same as pmpPort
 			if pres, ok := parsePCPResponse(buf[:n]); ok {
 				if pres.OpCode == pcpOpReply|pcpOpAnnounce {
@@ -729,13 +724,3 @@ func parsePCPResponse(b []byte) (res pcpResponse, ok bool) {
 }
 
 var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
-
-const (
-	upnpPort = 1900
-)
-
-var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
-	"HOST: 239.255.255.250:1900\r\n" +
-	"ST: ssdp:all\r\n" +
-	"MAN: \"ssdp:discover\"\r\n" +
-	"MX: 2\r\n\r\n")

net/portmapper/upnp.go

@@ -2,33 +2,19 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !ios
-// (https://github.com/tailscale/tailscale/issues/2495)
-
 package portmapper
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"fmt"
-	"log"
-	"math/rand"
-	"net/http"
 	"net/url"
 	"time"
 
-	"github.com/tailscale/goupnp"
 	"github.com/tailscale/goupnp/dcps/internetgateway2"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlknobs"
-	"tailscale.com/net/netns"
 )
 
-// VerboseLogs controls verbose debug logging.
-// It exists for use by "tailscaled debug --portmap".
-var VerboseLogs bool
-
 // References:
 //
 // WANIP Connection v2: http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v2-Service.pdf
@@ -54,8 +40,7 @@ func (u *upnpMapping) Release(ctx context.Context) {
 }
 
 // upnpClient is an interface over the multiple different clients exported by goupnp,
-// exposing the functions we need for portmapping. Those clients are auto-generated from XML-specs,
-// which is why they're not very idiomatic.
+// exposing the functions we need for portmapping. They are auto-generated from XML-specs.
 type upnpClient interface {
 	AddPortMapping(
 		ctx context.Context,
@@ -88,7 +73,7 @@ type upnpClient interface {
 		// greater than 0. From the spec, it appears if it is set to 0, it will switch to using
 		// 604800 seconds, but not sure why this is desired. The recommended time is 3600 seconds.
 		leaseDurationSec uint32,
-	) error
+	) (err error)
 
 	DeletePortMapping(ctx context.Context, remoteHost string, externalPort uint16, protocol string) error
 	GetExternalIPAddress(ctx context.Context) (externalIPAddress string, err error)
@@ -101,10 +86,6 @@ const tsPortMappingDesc = "tailscale-portmap"
 // addAnyPortMapping abstracts over different UPnP client connections, calling the available
 // AddAnyPortMapping call if available for WAN IP connection v2, otherwise defaulting to the old
 // behavior of calling AddPortMapping with port = 0 to specify a wildcard port.
-// It returns the new external port (which may not be identical to the external port specified),
-// or an error.
-//
-// TODO(bradfitz): also returned the actual lease duration obtained. and check it regularly.
 func addAnyPortMapping(
 	ctx context.Context,
 	upnp upnpClient,
@@ -126,9 +107,6 @@ func addAnyPortMapping(
 			uint32(leaseDuration.Seconds()),
 		)
 	}
-	for externalPort == 0 {
-		externalPort = uint16(rand.Intn(65535))
-	}
 	err = upnp.AddPortMapping(
 		ctx,
 		"",
@@ -140,79 +118,54 @@ func addAnyPortMapping(
 		tsPortMappingDesc,
 		uint32(leaseDuration.Seconds()),
 	)
-	return externalPort, err
+	return internalPort, err
 }
 
-// getUPnPClient gets a client for interfacing with UPnP, ignoring the underlying protocol for
+// getUPnPClients gets a client for interfacing with UPnP, ignoring the underlying protocol for
 // now.
 // Adapted from https://github.com/huin/goupnp/blob/master/GUIDE.md.
-//
-// The gw is the detected gateway.
-//
-// The meta is the most recently parsed UDP discovery packet response
-// from the Internet Gateway Device.
-//
-// The provided ctx is not retained in the returned upnpClient, but
-// its associated HTTP client is (if set via goupnp.WithHTTPClient).
-func getUPnPClient(ctx context.Context, gw netaddr.IP, meta uPnPDiscoResponse) (upnpClient, error) {
+func getUPnPClient(ctx context.Context, gw netaddr.IP) (upnpClient, error) {
 	if controlknobs.DisableUPnP() {
 		return nil, nil
 	}
-
-	if meta.Location == "" {
-		return nil, nil
-	}
-
-	if VerboseLogs {
-		log.Printf("fetching %v", meta.Location)
-	}
-	u, err := url.Parse(meta.Location)
-	if err != nil {
-		return nil, err
-	}
-
-	ipp, err := netaddr.ParseIPPort(u.Host)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected host %q in %q", u.Host, meta.Location)
-	}
-	if ipp.IP() != gw {
-		return nil, fmt.Errorf("UPnP discovered root %q does not match gateway IP %v; ignoring UPnP",
-			meta.Location, gw)
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+	ctx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
 	defer cancel()
+	// Attempt to connect over the multiple available connection types concurrently,
+	// returning the fastest.
 
-	// This part does a network fetch.
-	root, err := goupnp.DeviceByURL(ctx, u)
+	// TODO(jknodt): this url seems super brittle? maybe discovery is better but this is faster
+	u, err := url.Parse(fmt.Sprintf("http://%s:5000/rootDesc.xml", gw))
 	if err != nil {
 		return nil, err
 	}
 
-	// These parts don't do a network fetch.
-	// Pick the best service type available.
-	if cc, _ := internetgateway2.NewWANIPConnection2ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
-		return cc[0], nil
-	}
-	if cc, _ := internetgateway2.NewWANIPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
-		return cc[0], nil
-	}
-	if cc, _ := internetgateway2.NewWANPPPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
-		return cc[0], nil
-	}
-	return nil, nil
-}
-
-func (c *Client) upnpHTTPClientLocked() *http.Client {
-	if c.uPnPHTTPClient == nil {
-		c.uPnPHTTPClient = &http.Client{
-			Transport: &http.Transport{
-				DialContext:     netns.NewDialer().DialContext,
-				IdleConnTimeout: 2 * time.Second, // LAN is cheap
-			},
+	clients := make(chan upnpClient, 3)
+	go func() {
+		var err error
+		ip1Clients, err := internetgateway2.NewWANIPConnection1ClientsByURL(ctx, u)
+		if err == nil && len(ip1Clients) > 0 {
+			clients <- ip1Clients[0]
 		}
+	}()
+	go func() {
+		ip2Clients, err := internetgateway2.NewWANIPConnection2ClientsByURL(ctx, u)
+		if err == nil && len(ip2Clients) > 0 {
+			clients <- ip2Clients[0]
+		}
+	}()
+	go func() {
+		ppp1Clients, err := internetgateway2.NewWANPPPConnection1ClientsByURL(ctx, u)
+		if err == nil && len(ppp1Clients) > 0 {
+			clients <- ppp1Clients[0]
+		}
+	}()
+
+	select {
+	case client := <-clients:
+		return client, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
 	}
-	return c.uPnPHTTPClient
 }
 
 // getUPnPPortMapping attempts to create a port-mapping over the UPnP protocol. On success,
@@ -237,17 +190,11 @@ func (c *Client) getUPnPPortMapping(
 	var err error
 	c.mu.Lock()
 	oldMapping, ok := c.mapping.(*upnpMapping)
-	meta := c.uPnPMeta
-	httpClient := c.upnpHTTPClientLocked()
 	c.mu.Unlock()
 	if ok && oldMapping != nil {
 		client = oldMapping.client
 	} else {
-		ctx := goupnp.WithHTTPClient(ctx, httpClient)
-		client, err = getUPnPClient(ctx, gw, meta)
-		if VerboseLogs {
-			log.Printf("getUPnPClient: %T, %v", client, err)
-		}
+		client, err = getUPnPClient(ctx, gw)
 		if err != nil {
 			return netaddr.IPPort{}, false
 		}
@@ -265,17 +212,11 @@ func (c *Client) getUPnPPortMapping(
 		internal.IP().String(),
 		time.Second*pmpMapLifetimeSec,
 	)
-	if VerboseLogs {
-		log.Printf("addAnyPortMapping: %v, %v", newPort, err)
-	}
 	if err != nil {
 		return netaddr.IPPort{}, false
 	}
 	// TODO cache this ip somewhere?
 	extIP, err := client.GetExternalIPAddress(ctx)
-	if VerboseLogs {
-		log.Printf("client.GetExternalIPAddress: %v, %v", extIP, err)
-	}
 	if err != nil {
 		// TODO this doesn't seem right
 		return netaddr.IPPort{}, false
@@ -296,18 +237,3 @@ func (c *Client) getUPnPPortMapping(
 	c.localPort = newPort
 	return upnp.external, true
 }
-
-type uPnPDiscoResponse struct {
-	Location string
-}
-
-// parseUPnPDiscoResponse parses a UPnP HTTP-over-UDP discovery response.
-func parseUPnPDiscoResponse(body []byte) (uPnPDiscoResponse, error) {
-	var r uPnPDiscoResponse
-	res, err := http.ReadResponse(bufio.NewReaderSize(bytes.NewReader(body), 128), nil)
-	if err != nil {
-		return r, err
-	}
-	r.Location = res.Header.Get("Location")
-	return r, nil
-}

net/portmapper/upnp_test.go

@@ -1,95 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package portmapper
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"reflect"
-	"testing"
-
-	"inet.af/netaddr"
-)
-
-// Google Wifi
-const (
-	googleWifiUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nUSN: uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece::urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nEXT:\r\nSERVER: Linux/5.4.0-1034-gcp UPnP/1.1 MiniUPnPd/1.9\r\nLOCATION: http://192.168.86.1:5000/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1\r\nBOOTID.UPNP.ORG: 1\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
-
-	googleWifiRootDescXML = `<?xml version="1.0"?>
-<root xmlns="urn:schemas-upnp-org:device-1-0"><specVersion><major>1</major><minor>0</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:2</deviceType><friendlyName>OnHub</friendlyName><manufacturer>Google</manufacturer><manufacturerURL>http://google.com/</manufacturerURL><modelDescription>Wireless Router</modelDescription><modelName>OnHub</modelName><modelNumber>1</modelNumber><modelURL>https://on.google.com/hub/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:Layer3Forwarding1</serviceId><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL><SCPDURL>/L3F.xml</SCPDURL></service><service><serviceType>urn:schemas-upnp-org:service:DeviceProtection:1</serviceType><serviceId>urn:upnp-org:serviceId:DeviceProtection1</serviceId><controlURL>/ctl/DP</controlURL><eventSubURL>/evt/DP</eventSubURL><SCPDURL>/DP.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:2</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ecf</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL><SCPDURL>/WANCfg.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:2</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ec0</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:2</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL><SCPDURL>/WANIPCn.xml</SCPDURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>http://testwifi.here/</presentationURL></device></root>`
-)
-
-// pfSense 2.5.0-RELEASE / FreeBSD 12.2-STABLE
-const (
-	pfSenseUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nUSN: uuid:bee7052b-49e8-3597-b545-55a1e38ac11::urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nEXT:\r\nSERVER: FreeBSD/12.2-STABLE UPnP/1.1 MiniUPnPd/2.2.1\r\nLOCATION: http://192.168.1.1:2189/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1627958564\r\nBOOTID.UPNP.ORG: 1627958564\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
-
-	pfSenseRootDescXML = `<?xml version="1.0"?>
-<root xmlns="urn:schemas-upnp-org:device-1-0" configId="1337"><specVersion><major>1</major><minor>1</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:1</deviceType><friendlyName>FreeBSD router</friendlyName><manufacturer>FreeBSD</manufacturer><manufacturerURL>http://www.freebsd.org/</manufacturerURL><modelDescription>FreeBSD router</modelDescription><modelName>FreeBSD router</modelName><modelNumber>2.5.0-RELEASE</modelNumber><modelURL>http://www.freebsd.org/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac11</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:L3Forwarding1</serviceId><SCPDURL>/L3F.xml</SCPDURL><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:1</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac12</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><SCPDURL>/WANCfg.xml</SCPDURL><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:1</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac13</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:1</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><SCPDURL>/WANIPCn.xml</SCPDURL><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>https://192.168.1.1/</presentationURL></device></root>`
-)
-
-func TestParseUPnPDiscoResponse(t *testing.T) {
-	tests := []struct {
-		name    string
-		headers string
-		want    uPnPDiscoResponse
-	}{
-		{"google", googleWifiUPnPDisco, uPnPDiscoResponse{
-			Location: "http://192.168.86.1:5000/rootDesc.xml",
-		}},
-		{"pfsense", pfSenseUPnPDisco, uPnPDiscoResponse{
-			Location: "http://192.168.1.1:2189/rootDesc.xml",
-		}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseUPnPDiscoResponse([]byte(tt.headers))
-			if err != nil {
-				t.Fatal(err)
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("unexpected result:\n got: %+v\nwant: %+v\n", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestGetUPnPClient(t *testing.T) {
-	tests := []struct {
-		name    string
-		xmlBody string
-		want    string
-	}{
-		{"google", googleWifiRootDescXML, "*internetgateway2.WANIPConnection2"},
-		{"pfsense", pfSenseRootDescXML, "*internetgateway2.WANIPConnection1"},
-		// TODO(bradfitz): find a PPP one in the wild
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if r.RequestURI == "/rootDesc.xml" {
-					io.WriteString(w, tt.xmlBody)
-					return
-				}
-				http.NotFound(w, r)
-			}))
-			defer ts.Close()
-			gw, _ := netaddr.FromStdIP(ts.Listener.Addr().(*net.TCPAddr).IP)
-			c, err := getUPnPClient(context.Background(), gw, uPnPDiscoResponse{
-				Location: ts.URL + "/rootDesc.xml",
-			})
-			if err != nil {
-				t.Fatal(err)
-			}
-			got := fmt.Sprintf("%T", c)
-			if got != tt.want {
-				t.Errorf("got %v; want %v", got, tt.want)
-			}
-		})
-	}
-}

net/tsaddr/tsaddr.go

@@ -105,6 +105,11 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	return netaddr.IPFrom16(ret)
 }
 
+func IsULA(ip netaddr.IP) bool {
+	ulaRange.Do(func() { mustPrefix(&ulaRange.v, "fc00::/7") })
+	return ulaRange.v.Contains(ip)
+}
+
 func mustPrefix(v *netaddr.IPPrefix, prefix string) {
 	var err error
 	*v, err = netaddr.ParseIPPrefix(prefix)

net/tsaddr/tsaddr_test.go

@@ -43,6 +43,28 @@ func TestCGNATRange(t *testing.T) {
 	}
 }
 
+func TestIsUla(t *testing.T) {
+	tests := []struct {
+		name string
+		ip   string
+		want bool
+	}{
+		{"first ULA", "fc00::1", true},
+		{"not ULA", "fb00::1", false},
+		{"Tailscale", "fd7a:115c:a1e0::1", true},
+		{"Cloud Run", "fddf:3978:feb1:d745::1", true},
+		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
+		{"Link Local", "fe80::1", false},
+		{"Global", "2602::1", false},
+	}
+
+	for _, test := range tests {
+		if got := IsULA(netaddr.MustParseIP(test.ip)); got != test.want {
+			t.Errorf("IsULA(%s) = %v, want %v", test.name, got, test.want)
+		}
+	}
+}
+
 func TestNewContainsIPFunc(t *testing.T) {
 	f := NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.0.0.0/8")})
 	if f(netaddr.MustParseIP("8.8.8.8")) {

net/tstun/wrap.go

@@ -18,7 +18,6 @@ import (
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -65,7 +64,7 @@ type Wrapper struct {
 
 	closeOnce sync.Once
 
-	lastActivityAtomic mono.Time // time of last send or receive
+	lastActivityAtomic int64 // unix seconds of last send or receive
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 
@@ -154,10 +153,9 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		// a goroutine should not block when setting it, even with no listeners.
 		bufferConsumed: make(chan struct{}, 1),
 		closed:         make(chan struct{}),
-		// outbound can be unbuffered; the buffer is an optimization.
-		outbound:     make(chan tunReadResult, 1),
-		eventsUpDown: make(chan tun.Event),
-		eventsOther:  make(chan tun.Event),
+		outbound:       make(chan tunReadResult),
+		eventsUpDown:   make(chan tun.Event),
+		eventsOther:    make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
@@ -166,7 +164,6 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 	go tun.pumpEvents()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
-	tun.noteActivity()
 
 	return tun
 }
@@ -366,15 +363,16 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 
 // noteActivity records that there was a read or write at the current time.
 func (t *Wrapper) noteActivity() {
-	t.lastActivityAtomic.StoreAtomic(mono.Now())
+	atomic.StoreInt64(&t.lastActivityAtomic, time.Now().Unix())
 }
 
 // IdleDuration reports how long it's been since the last read or write to this device.
 //
-// Its value should only be presumed accurate to roughly 10ms granularity.
-// If there's never been activity, the duration is since the wrapper was created.
+// Its value is only accurate to roughly second granularity.
+// If there's never been activity, the duration is since 1970.
 func (t *Wrapper) IdleDuration() time.Duration {
-	return mono.Since(t.lastActivityAtomic.LoadAtomic())
+	sec := atomic.LoadInt64(&t.lastActivityAtomic)
+	return time.Since(time.Unix(sec, 0))
 }
 
 func (t *Wrapper) Read(buf []byte, offset int) (int, error) {

net/tstun/wrap_test.go

@@ -10,13 +10,13 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"unsafe"
 
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -335,9 +335,9 @@ func TestFilter(t *testing.T) {
 				// data was actually filtered.
 				// If it stays zero, nothing made it through
 				// to the wrapped TUN.
-				tun.lastActivityAtomic.StoreAtomic(0)
+				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
 				_, err = tun.Write(tt.data, 0)
-				filtered = tun.lastActivityAtomic.LoadAtomic() == 0
+				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)
@@ -416,7 +416,7 @@ func TestAtomic64Alignment(t *testing.T) {
 	}
 
 	c := new(Wrapper)
-	c.lastActivityAtomic.StoreAtomic(mono.Now())
+	atomic.StoreInt64(&c.lastActivityAtomic, 123)
 }
 
 func TestPeerAPIBypass(t *testing.T) {

safesocket/safesocket_darwin.go

@@ -53,16 +53,15 @@ func localTCPPortAndTokenDarwin() (port int, token string, err error) {
 	// The current process is running outside the sandbox, so use
 	// lsof to find the IPNExtension:
 
-	cmd := exec.Command("lsof",
+	out, err := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
 		fmt.Sprintf("-u%d", os.Getuid()), // process of same user only
 		"-c", "IPNExtension",             // starting with IPNExtension
 		"-F", // machine-readable output
-	)
-	out, err := cmd.Output()
+	).Output()
 	if err != nil {
-		return 0, "", fmt.Errorf("failed to run '%s' looking for IPNExtension: %w", cmd, err)
+		return 0, "", fmt.Errorf("failed to run lsof looking for IPNExtension: %w", err)
 	}
 	bs := bufio.NewScanner(bytes.NewReader(out))
 	subStr := []byte(".tailscale.ipn.macos/sameuserproof-")

safesocket/unixsocket.go

@@ -28,15 +28,16 @@ func connect(path string, port uint16) (net.Conn, error) {
 	pipe, err := net.Dial("unix", path)
 	if err != nil {
 		if runtime.GOOS == "darwin" {
-			extConn, extErr := connectMacOSAppSandbox()
-			if extErr != nil {
-				return nil, fmt.Errorf("safesocket: failed to connect to %v: %v; failed to connect to Tailscale IPNExtension: %v", path, err, extErr)
+			extConn, err := connectMacOSAppSandbox()
+			if err != nil {
+				log.Printf("safesocket: failed to connect to Tailscale IPNExtension: %v", err)
+			} else {
+				return extConn, nil
 			}
-			return extConn, nil
 		}
 		return nil, err
 	}
-	return pipe, nil
+	return pipe, err
 }
 
 // TODO(apenwarr): handle magic cookie auth

scripts/check_license_headers.sh

@@ -38,6 +38,9 @@ for file in $(find $1 -name '*.go' -not -path '*/.git/*'); do
         $1/wgengine/router/ifconfig_windows.go)
             # WireGuard copyright.
         ;;
+		*_string.go)
+			# Generated file from go:generate stringer
+		;;
         *)
             header="$(head -3 $file)"
             if ! check_file "$header"; then

syncs/syncs.go

@@ -79,16 +79,6 @@ func (b *AtomicBool) Set(v bool) {
 	atomic.StoreInt32((*int32)(b), n)
 }
 
-// Swap sets b to v and reports whether it changed.
-func (b *AtomicBool) Swap(v bool) (changed bool) {
-	var n int32
-	if v {
-		n = 1
-	}
-	old := atomic.SwapInt32((*int32)(b), n)
-	return old != n
-}
-
 func (b *AtomicBool) Get() bool {
 	return atomic.LoadInt32((*int32)(b)) != 0
 }

tailcfg/tailcfg.go

@@ -164,12 +164,6 @@ type Node struct {
 	Hostinfo   Hostinfo
 	Created    time.Time
 
-	// PrimaryRoutes are the routes from AllowedIPs that this node
-	// is currently the primary subnet router for, as determined
-	// by the control plane. It does not include the self address
-	// values from Addresses that are in AllowedIPs.
-	PrimaryRoutes []netaddr.IPPrefix `json:",omitempty"`
-
 	// LastSeen is when the node was last online. It is not
 	// updated when Online is true. It is nil if the current
 	// node doesn't have permission to know, or the node
@@ -1148,7 +1142,6 @@ func (n *Node) Equal(n2 *Node) bool {
 		eqBoolPtr(n.Online, n2.Online) &&
 		eqCIDRs(n.Addresses, n2.Addresses) &&
 		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
-		eqCIDRs(n.PrimaryRoutes, n2.PrimaryRoutes) &&
 		eqStrings(n.Endpoints, n2.Endpoints) &&
 		n.DERP == n2.DERP &&
 		n.Hostinfo.Equal(&n2.Hostinfo) &&

tailcfg/tailcfg_clone.go

@@ -49,7 +49,6 @@ func (src *Node) Clone() *Node {
 	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
 	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
 	dst.Hostinfo = *src.Hostinfo.Clone()
-	dst.PrimaryRoutes = append(src.PrimaryRoutes[:0:0], src.PrimaryRoutes...)
 	if dst.LastSeen != nil {
 		dst.LastSeen = new(time.Time)
 		*dst.LastSeen = *src.LastSeen
@@ -80,7 +79,6 @@ var _NodeNeedsRegeneration = Node(struct {
 	DERP                    string
 	Hostinfo                Hostinfo
 	Created                 time.Time
-	PrimaryRoutes           []netaddr.IPPrefix
 	LastSeen                *time.Time
 	Online                  *bool
 	KeepAlive               bool

tailcfg/tailcfg_test.go

@@ -194,8 +194,7 @@ func TestNodeEqual(t *testing.T) {
 		"ID", "StableID", "Name", "User", "Sharer",
 		"Key", "KeyExpiry", "Machine", "DiscoKey",
 		"Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo",
-		"Created", "PrimaryRoutes",
-		"LastSeen", "Online", "KeepAlive", "MachineAuthorized",
+		"Created", "LastSeen", "Online", "KeepAlive", "MachineAuthorized",
 		"Capabilities",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
 	}

tempfork/wireguard-windows/firewall/zsyscall_windows.go

@@ -1,3 +1,5 @@
+// +build windows
+
 // Code generated by 'go generate'; DO NOT EDIT.
 
 package firewall

tstest/integration/gen_deps.go

@@ -12,25 +12,16 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
-	"os"
 	"os/exec"
 )
 
 func main() {
-	for _, goos := range []string{"windows", "linux", "darwin", "freebsd", "openbsd"} {
-		generate(goos)
-	}
-}
-
-func generate(goos string) {
 	var x struct {
 		Imports []string
 	}
-	cmd := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled")
-	cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH=amd64")
-	j, err := cmd.Output()
+	j, err := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled").Output()
 	if err != nil {
-		log.Fatalf("GOOS=%s GOARCH=amd64 %s: %v", goos, cmd, err)
+		log.Fatal(err)
 	}
 	if err := json.Unmarshal(j, &x); err != nil {
 		log.Fatal(err)
@@ -55,8 +46,7 @@ import (
 	}
 	fmt.Fprintf(&out, ")\n")
 
-	filename := fmt.Sprintf("tailscaled_deps_test_%s.go", goos)
-	err = ioutil.WriteFile(filename, out.Bytes(), 0644)
+	err = ioutil.WriteFile("tailscaled_deps_test.go", out.Bytes(), 0644)
 	if err != nil {
 		log.Fatal(err)
 	}

tstest/integration/integration.go

@@ -205,13 +205,6 @@ func (lc *LogCatcher) logsString() string {
 	return lc.buf.String()
 }
 
-// Reset clears the buffered logs from memory.
-func (lc *LogCatcher) Reset() {
-	lc.mu.Lock()
-	defer lc.mu.Unlock()
-	lc.buf.Reset()
-}
-
 func (lc *LogCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var body io.Reader = r.Body
 	if r.Header.Get("Content-Encoding") == "zstd" {

tstest/integration/integration_test.go

@@ -42,7 +42,6 @@ import (
 
 var (
 	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
-	verboseTailscale  = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging")
 )
 
 var mainError atomic.Value // of error
@@ -73,9 +72,29 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-	n1.AwaitResponding(t)
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `Program starting: `
+		if !env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
 	n1.MustUp()
 
+	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
+		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
+		time.Sleep(d)
+	}
+
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
 	n1.AwaitRunning(t)
 
@@ -84,40 +103,6 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
-func TestCollectPanic(t *testing.T) {
-	t.Parallel()
-	bins := BuildTestBinaries(t)
-
-	env := newTestEnv(t, bins)
-	defer env.Close()
-
-	n := newTestNode(t, env)
-
-	cmd := exec.Command(n.env.Binaries.Daemon, "--cleanup")
-	cmd.Env = append(os.Environ(),
-		"TS_PLEASE_PANIC=1",
-		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
-	)
-	got, _ := cmd.CombinedOutput() // we expect it to fail, ignore err
-	t.Logf("initial run: %s", got)
-
-	// Now we run it again, and on start, it will upload the logs to logcatcher.
-	cmd = exec.Command(n.env.Binaries.Daemon, "--cleanup")
-	cmd.Env = append(os.Environ(), "TS_LOG_TARGET="+n.env.LogCatcherServer.URL)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		t.Fatalf("cleanup failed: %v: %q", err, out)
-	}
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `panic`
-		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Fatal(err)
-	}
-}
-
 // test Issue 2321: Start with UpdatePrefs should save prefs to disk
 func TestStateSavedOnStart(t *testing.T) {
 	t.Parallel()
@@ -130,7 +115,22 @@ func TestStateSavedOnStart(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-	n1.AwaitResponding(t)
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `Program starting: `
+		if !env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
 	n1.MustUp()
 
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
@@ -346,76 +346,6 @@ func TestAddPingRequest(t *testing.T) {
 	t.Error("all ping attempts failed")
 }
 
-// Issue 2434: when "down" (WantRunning false), tailscaled shouldn't
-// be connected to control.
-func TestNoControlConnWhenDown(t *testing.T) {
-	t.Parallel()
-	bins := BuildTestBinaries(t)
-
-	env := newTestEnv(t, bins)
-	defer env.Close()
-
-	n1 := newTestNode(t, env)
-
-	d1 := n1.StartDaemon(t)
-	defer d1.Kill()
-	n1.AwaitResponding(t)
-
-	// Come up the first time.
-	n1.MustUp()
-	ip1 := n1.AwaitIP(t)
-	n1.AwaitRunning(t)
-
-	// Then bring it down and stop the daemon.
-	n1.MustDown()
-	d1.MustCleanShutdown(t)
-
-	env.LogCatcher.Reset()
-	d2 := n1.StartDaemon(t)
-	defer d2.Kill()
-	n1.AwaitResponding(t)
-
-	st := n1.MustStatus(t)
-	if got, want := st.BackendState, "Stopped"; got != want {
-		t.Fatalf("after restart, state = %q; want %q", got, want)
-	}
-
-	ip2 := n1.AwaitIP(t)
-	if ip1 != ip2 {
-		t.Errorf("IPs different: %q vs %q", ip1, ip2)
-	}
-
-	// The real test: verify our daemon doesn't have an HTTP request open.:
-	if n := env.Control.InServeMap(); n != 0 {
-		t.Errorf("in serve map = %d; want 0", n)
-	}
-
-	d2.MustCleanShutdown(t)
-}
-
-// Issue 2137: make sure Windows tailscaled works with the CLI alone,
-// without the GUI to kick off a Start.
-func TestOneNodeUpWindowsStyle(t *testing.T) {
-	t.Parallel()
-	bins := BuildTestBinaries(t)
-
-	env := newTestEnv(t, bins)
-	defer env.Close()
-
-	n1 := newTestNode(t, env)
-	n1.upFlagGOOS = "windows"
-
-	d1 := n1.StartDaemonAsIPNGOOS(t, "windows")
-	defer d1.Kill()
-	n1.AwaitResponding(t)
-	n1.MustUp("--unattended")
-
-	t.Logf("Got IP: %v", n1.AwaitIP(t))
-	n1.AwaitRunning(t)
-
-	d1.MustCleanShutdown(t)
-}
-
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
@@ -492,10 +422,9 @@ func (e *testEnv) Close() error {
 type testNode struct {
 	env *testEnv
 
-	dir        string // temp dir for sock & state
-	sockFile   string
-	stateFile  string
-	upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI
+	dir       string // temp dir for sock & state
+	sockFile  string
+	stateFile string
 
 	mu        sync.Mutex
 	onLogLine []func([]byte)
@@ -505,14 +434,10 @@ type testNode struct {
 // The node is not started automatically.
 func newTestNode(t *testing.T, env *testEnv) *testNode {
 	dir := t.TempDir()
-	sockFile := filepath.Join(dir, "tailscale.sock")
-	if len(sockFile) >= 104 {
-		t.Fatalf("sockFile path %q (len %v) is too long, must be < 104", sockFile, len(sockFile))
-	}
 	return &testNode{
 		env:       env,
 		dir:       dir,
-		sockFile:  sockFile,
+		sockFile:  filepath.Join(dir, "tailscale.sock"),
 		stateFile: filepath.Join(dir, "tailscale.state"),
 	}
 }
@@ -537,26 +462,6 @@ func (n *testNode) diskPrefs(t testing.TB) *ipn.Prefs {
 	return p
 }
 
-// AwaitResponding waits for n's tailscaled to be up enough to be
-// responding, but doesn't wait for any particular state.
-func (n *testNode) AwaitResponding(t testing.TB) {
-	t.Helper()
-	n.AwaitListening(t)
-
-	st := n.MustStatus(t)
-	t.Logf("Status: %s", st.BackendState)
-
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `Program starting: `
-		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Fatal(err)
-	}
-}
-
 // addLogLineHook registers a hook f to be called on each tailscaled
 // log line output.
 func (n *testNode) addLogLineHook(f func([]byte)) {
@@ -658,10 +563,6 @@ func (d *Daemon) MustCleanShutdown(t testing.TB) {
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
 func (n *testNode) StartDaemon(t testing.TB) *Daemon {
-	return n.StartDaemonAsIPNGOOS(t, runtime.GOOS)
-}
-
-func (n *testNode) StartDaemonAsIPNGOOS(t testing.TB, ipnGOOS string) *Daemon {
 	cmd := exec.Command(n.env.Binaries.Daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
@@ -672,8 +573,6 @@ func (n *testNode) StartDaemonAsIPNGOOS(t testing.TB, ipnGOOS string) *Daemon {
 		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
 		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
 		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
-		"TS_DEBUG_TAILSCALED_IPN_GOOS="+ipnGOOS,
-		"TS_LOGS_DIR="+t.TempDir(),
 	)
 	cmd.Stderr = &nodeOutputParser{n: n}
 	if *verboseTailscaled {
@@ -688,15 +587,10 @@ func (n *testNode) StartDaemonAsIPNGOOS(t testing.TB, ipnGOOS string) *Daemon {
 	}
 }
 
-func (n *testNode) MustUp(extraArgs ...string) {
+func (n *testNode) MustUp() {
 	t := n.env.t
-	args := []string{
-		"up",
-		"--login-server=" + n.env.ControlServer.URL,
-	}
-	args = append(args, extraArgs...)
-	t.Logf("Running %v ...", args)
-	if err := n.Tailscale(args...).Run(); err != nil {
+	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
+	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
 		t.Fatalf("up: %v", err)
 	}
 }
@@ -728,10 +622,7 @@ func (n *testNode) AwaitIPs(t testing.TB) []netaddr.IP {
 	t.Helper()
 	var addrs []netaddr.IP
 	if err := tstest.WaitFor(20*time.Second, func() error {
-		cmd := n.Tailscale("ip")
-		cmd.Stdout = nil // in case --verbose-tailscale was set
-		cmd.Stderr = nil // in case --verbose-tailscale was set
-		out, err := cmd.Output()
+		out, err := n.Tailscale("ip").Output()
 		if err != nil {
 			return err
 		}
@@ -763,7 +654,6 @@ func (n *testNode) AwaitIP(t testing.TB) netaddr.IP {
 	return ips[0]
 }
 
-// AwaitRunning waits for n to reach the IPN state "Running".
 func (n *testNode) AwaitRunning(t testing.TB) {
 	t.Helper()
 	if err := tstest.WaitFor(20*time.Second, func() error {
@@ -786,22 +676,11 @@ func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
 	cmd := exec.Command(n.env.Binaries.CLI, "--socket="+n.sockFile)
 	cmd.Args = append(cmd.Args, arg...)
 	cmd.Dir = n.dir
-	cmd.Env = append(os.Environ(),
-		"TS_DEBUG_UP_FLAG_GOOS="+n.upFlagGOOS,
-		"TS_LOGS_DIR="+n.env.t.TempDir(),
-	)
-	if *verboseTailscale {
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-	}
 	return cmd
 }
 
 func (n *testNode) Status() (*ipnstate.Status, error) {
-	cmd := n.Tailscale("status", "--json")
-	cmd.Stdout = nil // in case --verbose-tailscale was set
-	cmd.Stderr = nil // in case --verbose-tailscale was set
-	out, err := cmd.CombinedOutput()
+	out, err := n.Tailscale("status", "--json").CombinedOutput()
 	if err != nil {
 		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
 	}

tstest/integration/tailscaled_deps_test.go

@@ -39,7 +39,6 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
-	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_darwin.go

@@ -1,61 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Code generated by gen_deps.go; DO NOT EDIT.
-
-package integration
-
-import (
-	// And depend on a bunch of tailscaled innards, for Go's test caching.
-	// Otherwise cmd/go never sees that we depend on these packages'
-	// transitive deps when we run "go install tailscaled" in a child
-	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/exec"
-	_ "os/signal"
-	_ "path/filepath"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
-	_ "tailscale.com/derp/derphttp"
-	_ "tailscale.com/ipn"
-	_ "tailscale.com/ipn/ipnserver"
-	_ "tailscale.com/logpolicy"
-	_ "tailscale.com/net/dns"
-	_ "tailscale.com/net/interfaces"
-	_ "tailscale.com/net/portmapper"
-	_ "tailscale.com/net/socks5/tssocks"
-	_ "tailscale.com/net/tshttpproxy"
-	_ "tailscale.com/net/tstun"
-	_ "tailscale.com/paths"
-	_ "tailscale.com/tailcfg"
-	_ "tailscale.com/types/flagtype"
-	_ "tailscale.com/types/key"
-	_ "tailscale.com/types/logger"
-	_ "tailscale.com/util/osshare"
-	_ "tailscale.com/version"
-	_ "tailscale.com/version/distro"
-	_ "tailscale.com/wgengine"
-	_ "tailscale.com/wgengine/monitor"
-	_ "tailscale.com/wgengine/netstack"
-	_ "tailscale.com/wgengine/router"
-	_ "time"
-)

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -1,59 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Code generated by gen_deps.go; DO NOT EDIT.
-
-package integration
-
-import (
-	// And depend on a bunch of tailscaled innards, for Go's test caching.
-	// Otherwise cmd/go never sees that we depend on these packages'
-	// transitive deps when we run "go install tailscaled" in a child
-	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
-	_ "tailscale.com/derp/derphttp"
-	_ "tailscale.com/ipn"
-	_ "tailscale.com/ipn/ipnserver"
-	_ "tailscale.com/logpolicy"
-	_ "tailscale.com/net/dns"
-	_ "tailscale.com/net/interfaces"
-	_ "tailscale.com/net/portmapper"
-	_ "tailscale.com/net/socks5/tssocks"
-	_ "tailscale.com/net/tshttpproxy"
-	_ "tailscale.com/net/tstun"
-	_ "tailscale.com/paths"
-	_ "tailscale.com/tailcfg"
-	_ "tailscale.com/types/flagtype"
-	_ "tailscale.com/types/key"
-	_ "tailscale.com/types/logger"
-	_ "tailscale.com/util/osshare"
-	_ "tailscale.com/version"
-	_ "tailscale.com/version/distro"
-	_ "tailscale.com/wgengine"
-	_ "tailscale.com/wgengine/monitor"
-	_ "tailscale.com/wgengine/netstack"
-	_ "tailscale.com/wgengine/router"
-	_ "time"
-)

tstest/integration/tailscaled_deps_test_linux.go

@@ -1,59 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Code generated by gen_deps.go; DO NOT EDIT.
-
-package integration
-
-import (
-	// And depend on a bunch of tailscaled innards, for Go's test caching.
-	// Otherwise cmd/go never sees that we depend on these packages'
-	// transitive deps when we run "go install tailscaled" in a child
-	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
-	_ "tailscale.com/derp/derphttp"
-	_ "tailscale.com/ipn"
-	_ "tailscale.com/ipn/ipnserver"
-	_ "tailscale.com/logpolicy"
-	_ "tailscale.com/net/dns"
-	_ "tailscale.com/net/interfaces"
-	_ "tailscale.com/net/portmapper"
-	_ "tailscale.com/net/socks5/tssocks"
-	_ "tailscale.com/net/tshttpproxy"
-	_ "tailscale.com/net/tstun"
-	_ "tailscale.com/paths"
-	_ "tailscale.com/tailcfg"
-	_ "tailscale.com/types/flagtype"
-	_ "tailscale.com/types/key"
-	_ "tailscale.com/types/logger"
-	_ "tailscale.com/util/osshare"
-	_ "tailscale.com/version"
-	_ "tailscale.com/version/distro"
-	_ "tailscale.com/wgengine"
-	_ "tailscale.com/wgengine/monitor"
-	_ "tailscale.com/wgengine/netstack"
-	_ "tailscale.com/wgengine/router"
-	_ "time"
-)

tstest/integration/tailscaled_deps_test_windows.go

@@ -1,66 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Code generated by gen_deps.go; DO NOT EDIT.
-
-package integration
-
-import (
-	// And depend on a bunch of tailscaled innards, for Go's test caching.
-	// Otherwise cmd/go never sees that we depend on these packages'
-	// transitive deps when we run "go install tailscaled" in a child
-	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
-	_ "golang.org/x/sys/windows"
-	_ "golang.org/x/sys/windows/svc"
-	_ "golang.org/x/sys/windows/svc/mgr"
-	_ "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
-	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
-	_ "tailscale.com/derp/derphttp"
-	_ "tailscale.com/ipn"
-	_ "tailscale.com/ipn/ipnserver"
-	_ "tailscale.com/logpolicy"
-	_ "tailscale.com/logtail/backoff"
-	_ "tailscale.com/net/dns"
-	_ "tailscale.com/net/interfaces"
-	_ "tailscale.com/net/portmapper"
-	_ "tailscale.com/net/socks5/tssocks"
-	_ "tailscale.com/net/tshttpproxy"
-	_ "tailscale.com/net/tstun"
-	_ "tailscale.com/paths"
-	_ "tailscale.com/tailcfg"
-	_ "tailscale.com/types/flagtype"
-	_ "tailscale.com/types/key"
-	_ "tailscale.com/types/logger"
-	_ "tailscale.com/util/osshare"
-	_ "tailscale.com/version"
-	_ "tailscale.com/version/distro"
-	_ "tailscale.com/wf"
-	_ "tailscale.com/wgengine"
-	_ "tailscale.com/wgengine/monitor"
-	_ "tailscale.com/wgengine/netstack"
-	_ "tailscale.com/wgengine/router"
-	_ "time"
-)

tstest/integration/testcontrol/testcontrol.go

@@ -51,7 +51,6 @@ type Server struct {
 	mux         *http.ServeMux
 
 	mu            sync.Mutex
-	inServeMap    int
 	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        wgkey.Key
 	privKey       wgkey.Private
@@ -510,22 +509,7 @@ func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
 	return peersToUpdate
 }
 
-func (s *Server) incrInServeMap(delta int) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.inServeMap += delta
-}
-
-// InServeMap returns the number of clients currently in a MapRequest HTTP handler.
-func (s *Server) InServeMap() int {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.inServeMap
-}
-
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
-	s.incrInServeMap(1)
-	defer s.incrInServeMap(-1)
 	ctx := r.Context()
 
 	req := new(tailcfg.MapRequest)

tstest/integration/vms/distros.go

@@ -2,33 +2,24 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux
+
 package vms
 
-import (
-	_ "embed"
-	"log"
-
-	"github.com/tailscale/hujson"
-)
-
-// go:generate go run ./gen
-
 type Distro struct {
-	Name           string // amazon-linux
-	URL            string // URL to a qcow2 image
-	SHA256Sum      string // hex-encoded sha256 sum of contents of URL
-	MemoryMegs     int    // VM memory in megabytes
-	PackageManager string // yum/apt/dnf/zypper
-	InitSystem     string // systemd/openrc
+	name           string // amazon-linux
+	url            string // URL to a qcow2 image
+	sha256sum      string // hex-encoded sha256 sum of contents of URL
+	mem            int    // VM memory in megabytes
+	packageManager string // yum/apt/dnf/zypper
+	initSystem     string // systemd/openrc
 }
 
 func (d *Distro) InstallPre() string {
-	switch d.PackageManager {
+	switch d.packageManager {
 	case "yum":
 		return ` - [ yum, update, gnupg2 ]
- - [ yum, "-y", install, iptables ]
- - [ sh, "-c", "printf '\n\nUseDNS no\n\n' | tee -a /etc/ssh/sshd_config" ]
- - [ systemctl, restart, "sshd.service" ]`
+ - [ yum, "-y", install, iptables ]`
 	case "zypper":
 		return ` - [ zypper, in, "-y", iptables ]`
 
@@ -47,15 +38,58 @@ func (d *Distro) InstallPre() string {
 	return ""
 }
 
-//go:embed distros.hujson
-var distroData string
+var distros = []Distro{
+	// NOTE(Xe): If you run into issues getting the autoconfig to work, run
+	// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
+	// client with a command like this:
+	//
+	//    $ vncviewer :0
+	//
+	// On NixOS you can get away with something like this:
+	//
+	//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
+	//
+	// Login as root with the password root. Then look in
+	// /var/log/cloud-init-output.log for what you messed up.
 
-var Distros []Distro = func() []Distro {
-	var result []Distro
-	err := hujson.Unmarshal([]byte(distroData), &result)
-	if err != nil {
-		log.Fatalf("error decoding distros: %v", err)
-	}
+	// NOTE(Xe): These images are not official images created by the Alpine Linux
+	// cloud team because the cloud team hasn't created any official images yet.
+	// These images were created under the guidance of the cloud team and contain
+	// few notable differences from what they would end up shipping. The Alpine
+	// Linux cloud team probably won't have official images up until a year or so
+	// after this comment is written (2021-06-11), but overall they will be
+	// compatible with these images. These images were created using the setup in
+	// this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
+	// these links.
+	{"alpine-3-13-5", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", 256, "apk", "openrc"},
+	{"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk", "openrc"},
 
-	return result
-}()
+	// NOTE(Xe): All of the following images are official images straight from each
+	// distribution's official documentation.
+	{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum", "systemd"},
+	{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman", "systemd"},
+	{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum", "systemd"},
+	{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf", "systemd"},
+	{"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt", "systemd"},
+	{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt", "systemd"},
+	{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf", "systemd"},
+	{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper", "systemd"},
+	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"},
+	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"},
+	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"},
+	{"oracle-linux-7", "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2", "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb", 512, "yum", "systemd"},
+	{"oracle-linux-8", "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2", "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b", 768, "dnf", "systemd"},
+	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"},
+	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"},
+	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
+	{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"},
+	{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"},
+
+	// NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
+	// used here is actually the URL of the NixOS channel being built from and the
+	// shasum is meaningless. This `channel:name` syntax is documented at [1].
+	//
+	// [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
+	{"nixos-21-05", "channel:nixos-21.05", "lolfakesha", 512, "nix", "systemd"},
+	{"nixos-unstable", "channel:nixos-unstable", "lolfakesha", 512, "nix", "systemd"},
+}

tstest/integration/vms/distros.hujson

@@ -1,208 +0,0 @@
-// NOTE(Xe): If you run into issues getting the autoconfig to work, run
-// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
-// client with a command like this:
-//
-//    $ vncviewer :0
-//
-// On NixOS you can get away with something like this:
-//
-//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
-//
-// Login as root with the password root. Then look in
-// /var/log/cloud-init-output.log for what you messed up.
-[
-	  // NOTE(Xe): These images are not official images created by the Alpine Linux
-	  // cloud team because the cloud team hasn't created any official images yet.
-	  // These images were created under the guidance of the cloud team and contain
-	  // few notable differences from what they would end up shipping. The Alpine
-	  // Linux cloud team probably won't have official images up until a year or so
-	  // after this comment is written (2021-06-11), but overall they will be
-	  // compatible with these images. These images were created using the setup in
-	  // this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
-	  // these links.
-    {
-        "Name": "alpine-3-13-5",
-        "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2",
-        "SHA256Sum": "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287",
-        "MemoryMegs": 256,
-        "PackageManager": "apk",
-        "InitSystem": "openrc"
-    },
-    {
-        "Name": "alpine-edge",
-        "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2",
-        "SHA256Sum": "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0",
-        "MemoryMegs": 256,
-        "PackageManager": "apk",
-        "InitSystem": "openrc"
-    },
-
-	  // NOTE(Xe): All of the following images are official images straight from each
-	  // distribution's official documentation.
-    {
-        "Name": "amazon-linux",
-        "URL": "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2",
-        "SHA256Sum": "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b",
-        "MemoryMegs": 512,
-        "PackageManager": "yum",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "arch",
-        "URL": "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2",
-        "SHA256Sum": "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27",
-        "MemoryMegs": 512,
-        "PackageManager": "pacman",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "centos-7",
-        "URL": "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c",
-        "SHA256Sum": "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87",
-        "MemoryMegs": 512,
-        "PackageManager": "yum",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "centos-8",
-        "URL": "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2",
-        "SHA256Sum": "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0",
-        "MemoryMegs": 768,
-        "PackageManager": "dnf",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "debian-9",
-        "URL": "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2",
-        "SHA256Sum": "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d",
-        "MemoryMegs": 512,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "debian-10",
-        "URL": "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2",
-        "SHA256Sum": "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0",
-        "MemoryMegs": 768,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "fedora-34",
-        "URL": "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2",
-        "SHA256Sum": "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea",
-        "MemoryMegs": 768,
-        "PackageManager": "dnf",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "opensuse-leap-15-1",
-        "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2",
-        "SHA256Sum": "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02",
-        "MemoryMegs": 512,
-        "PackageManager": "zypper",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "opensuse-leap-15-2",
-        "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2",
-        "SHA256Sum": "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f",
-        "MemoryMegs": 512,
-        "PackageManager": "zypper",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "opensuse-leap-15-3",
-        "URL": "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2",
-        "SHA256Sum": "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5",
-        "MemoryMegs": 512,
-        "PackageManager": "zypper",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "opensuse-tumbleweed",
-        "URL": "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2",
-        "SHA256Sum": "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f",
-        "MemoryMegs": 512,
-        "PackageManager": "zypper",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "oracle-linux-7",
-        "URL": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2",
-        "SHA256Sum": "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb",
-        "MemoryMegs": 512,
-        "PackageManager": "yum",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "oracle-linux-8",
-        "URL": "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2",
-        "SHA256Sum": "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b",
-        "MemoryMegs": 768,
-        "PackageManager": "dnf",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "ubuntu-16-04",
-        "URL": "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img",
-        "SHA256Sum": "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b",
-        "MemoryMegs": 512,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "ubuntu-18-04",
-        "URL": "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img",
-        "SHA256Sum": "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022",
-        "MemoryMegs": 512,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "ubuntu-20-04",
-        "URL": "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img",
-        "SHA256Sum": "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb",
-        "MemoryMegs": 512,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "ubuntu-20-10",
-        "URL": "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img",
-        "SHA256Sum": "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef",
-        "MemoryMegs": 512,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "ubuntu-21-04",
-        "URL": "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img",
-        "SHA256Sum": "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b",
-        "MemoryMegs": 512,
-        "PackageManager": "apt",
-        "InitSystem": "systemd"
-    },
-
-	  // NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
-	  // used here is actually the URL of the NixOS channel being built from and the
-	  // shasum is meaningless. This `channel:name` syntax is documented at [1].
-	  //
-	  // [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
-    {
-        "Name": "nixos-21-05",
-        "URL": "channel:nixos-21.05",
-        "SHA256Sum": "lolfakesha",
-        "MemoryMegs": 512,
-        "PackageManager": "nix",
-        "InitSystem": "systemd"
-    },
-    {
-        "Name": "nixos-unstable",
-        "URL": "channel:nixos-unstable",
-        "SHA256Sum": "lolfakesha",
-        "MemoryMegs": 512,
-        "PackageManager": "nix",
-        "InitSystem": "systemd"
-    }
-]

tstest/integration/vms/distros_test.go

@@ -1,17 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux
-
-package vms
-
-import (
-	"testing"
-)
-
-func TestDistrosGotLoaded(t *testing.T) {
-	if len(Distros) == 0 {
-		t.Fatal("no distros were loaded")
-	}
-}

tstest/integration/vms/gen/test_codegen.go

@@ -1,55 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// build ignore
-
-package main
-
-import (
-	_ "embed"
-	"fmt"
-	"log"
-	"os"
-	"time"
-
-	"github.com/dave/jennifer/jen"
-	"github.com/iancoleman/strcase"
-	"tailscale.com/tstest/integration/vms"
-)
-
-func main() {
-	f := jen.NewFile("vms")
-	f.Comment("Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.")
-
-	ptr := jen.Op("*")
-
-	for i, d := range vms.Distros {
-		f.Func().
-			Id("TestRun" + strcase.ToCamel(d.Name)).
-			Params(jen.Id("t").Add(ptr).Qual("testing", "T")).
-			BlockFunc(func(g *jen.Group) {
-				g.Id("t").Dot("Parallel").Call()
-				g.Id("setupTests").Call(jen.Id("t"))
-				g.Id("testOneDistribution").Call(jen.Id("t"), jen.Lit(i), jen.Id("Distros").Index(jen.Lit(i)))
-			})
-	}
-
-	os.Remove("top_level_test.go")
-	fout, err := os.Create("top_level_test.go")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer fout.Close()
-
-	fmt.Fprintf(fout, "// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.\n", time.Now().Year())
-	fout.WriteString("// Use of this source code is governed by a BSD-style\n")
-	fout.WriteString("// license that can be found in the LICENSE file.\n")
-	fout.WriteString("\n")
-	fout.WriteString("// +build linux\n\n")
-
-	err = f.Render(fout)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

tstest/integration/vms/nixos_test.go

@@ -167,13 +167,9 @@ func copyUnit(t *testing.T, bins *integration.Binaries) {
 }
 
 func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
-	if d.Name == "nixos-unstable" {
-		t.Skip("https://github.com/NixOS/nixpkgs/issues/131098")
-	}
-
 	copyUnit(t, h.bins)
 	dir := t.TempDir()
-	fname := filepath.Join(dir, d.Name+".nix")
+	fname := filepath.Join(dir, d.name+".nix")
 	fout, err := os.Create(fname)
 	if err != nil {
 		t.Fatal(err)
@@ -200,10 +196,10 @@ func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
 	os.MkdirAll(outpath, 0755)
 
 	t.Cleanup(func() {
-		os.RemoveAll(filepath.Join(outpath, d.Name)) // makes the disk image a candidate for GC
+		os.RemoveAll(filepath.Join(outpath, d.name)) // makes the disk image a candidate for GC
 	})
 
-	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.Name), "-c", fname)
+	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.name), "-c", fname)
 	if *verboseNixOutput {
 		cmd.Stdout = logger.FuncWriter(t.Logf)
 		cmd.Stderr = logger.FuncWriter(t.Logf)
@@ -218,16 +214,16 @@ func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string {
 		cmd.Stderr = fout
 		defer fout.Close()
 	}
-	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.URL)
+	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.url)
 	cmd.Dir = outpath
 	t.Logf("running %s %#v", "nixos-generate", cmd.Args)
 	if err := cmd.Run(); err != nil {
-		t.Fatalf("error while making NixOS image for %s: %v", d.Name, err)
+		t.Fatalf("error while making NixOS image for %s: %v", d.name, err)
 	}
 
 	if !*verboseNixOutput {
 		t.Log("done")
 	}
 
-	return filepath.Join(outpath, d.Name, "nixos.qcow2")
+	return filepath.Join(outpath, d.name, "nixos.qcow2")
 }

tstest/integration/vms/opensuse_leap_15_1_test.go

@@ -41,7 +41,7 @@ type openSUSELeap151MetaDataMeta struct {
 }
 
 func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
-	if d.Name != "opensuse-leap-15-1" {
+	if d.name != "opensuse-leap-15-1" {
 		return false
 	}
 
@@ -54,14 +54,14 @@ func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
 
 	metadata, err := json.Marshal(openSUSELeap151MetaData{
 		Zone:        "nova",
-		Hostname:    d.Name,
+		Hostname:    d.name,
 		LaunchIndex: "0",
 		Meta: openSUSELeap151MetaDataMeta{
 			Role:      "server",
 			DSMode:    "local",
 			Essential: "false",
 		},
-		Name: d.Name,
+		Name: d.name,
 		UUID: uuid.New().String(),
 	})
 	if err != nil {

tstest/integration/vms/top_level_test.go

@@ -1,121 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux
-
-package vms
-
-import "testing"
-
-// Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.
-func TestRunAlpine3135(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 0, Distros[0])
-}
-func TestRunAlpineEdge(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 1, Distros[1])
-}
-func TestRunAmazonLinux(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 2, Distros[2])
-}
-func TestRunArch(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 3, Distros[3])
-}
-func TestRunCentos7(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 4, Distros[4])
-}
-func TestRunCentos8(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 5, Distros[5])
-}
-func TestRunDebian9(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 6, Distros[6])
-}
-func TestRunDebian10(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 7, Distros[7])
-}
-func TestRunFedora34(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 8, Distros[8])
-}
-func TestRunOpensuseLeap151(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 9, Distros[9])
-}
-func TestRunOpensuseLeap152(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 10, Distros[10])
-}
-func TestRunOpensuseLeap153(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 11, Distros[11])
-}
-func TestRunOpensuseTumbleweed(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 12, Distros[12])
-}
-func TestRunOracleLinux7(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 13, Distros[13])
-}
-func TestRunOracleLinux8(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 14, Distros[14])
-}
-func TestRunUbuntu1604(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 15, Distros[15])
-}
-func TestRunUbuntu1804(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 16, Distros[16])
-}
-func TestRunUbuntu2004(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 17, Distros[17])
-}
-func TestRunUbuntu2010(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 18, Distros[18])
-}
-func TestRunUbuntu2104(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 19, Distros[19])
-}
-func TestRunNixos2105(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 20, Distros[20])
-}
-func TestRunNixosUnstable(t *testing.T) {
-	t.Parallel()
-	setupTests(t)
-	testOneDistribution(t, 21, Distros[21])
-}

tstest/integration/vms/vm_setup_test.go

@@ -53,17 +53,17 @@ func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir stri
 	mkLayeredQcow(t, tdir, d, h.fetchDistro(t, d))
 	mkSeed(t, d, sshKey, hostURL, tdir, port)
 
-	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.Name+".qcow2"))
+	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
 
 	args := []string{
 		"-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off",
 		"-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port),
 		"-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25",
-		"-m", fmt.Sprint(d.MemoryMegs),
+		"-m", fmt.Sprint(d.mem),
 		"-boot", "c",
 		"-drive", driveArg,
-		"-cdrom", filepath.Join(tdir, d.Name, "seed", "seed.iso"),
-		"-smbios", "type=1,serial=ds=nocloud;h=" + d.Name,
+		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
+		"-smbios", "type=1,serial=ds=nocloud;h=" + d.name,
 	}
 
 	if *useVNC {
@@ -101,7 +101,7 @@ func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir stri
 	t.Cleanup(func() {
 		err := cmd.Process.Kill()
 		if err != nil {
-			t.Errorf("can't kill %s (%d): %v", d.Name, cmd.Process.Pid, err)
+			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
 		}
 
 		cmd.Wait()
@@ -139,15 +139,15 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
 		d.PartSize = 64 * 1024 * 1024 // 64MB per part
 	})
 
-	t.Logf("fetching s3://%s/%s", bucketName, d.SHA256Sum)
+	t.Logf("fetching s3://%s/%s", bucketName, d.sha256sum)
 
 	_, err = dler.Download(fout, &s3.GetObjectInput{
 		Bucket: aws.String(bucketName),
-		Key:    aws.String(d.SHA256Sum),
+		Key:    aws.String(d.sha256sum),
 	})
 	if err != nil {
 		fout.Close()
-		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.SHA256Sum, err)
+		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.sha256sum, err)
 	}
 
 	err = fout.Close()
@@ -169,17 +169,17 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	}
 	cdir = filepath.Join(cdir, "tailscale", "vm-test")
 
-	if strings.HasPrefix(resultDistro.Name, "nixos") {
+	if strings.HasPrefix(resultDistro.name, "nixos") {
 		return h.makeNixOSImage(t, resultDistro, cdir)
 	}
 
-	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.SHA256Sum)
+	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
 
 	_, err = os.Stat(qcowPath)
 	if err == nil {
 		hash := checkCachedImageHash(t, resultDistro, cdir)
-		if hash != resultDistro.SHA256Sum {
-			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.Name, qcowPath, resultDistro.SHA256Sum)
+		if hash != resultDistro.sha256sum {
+			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.name, qcowPath, resultDistro.sha256sum)
 			err = errors.New("some fake non-nil error to force a redownload")
 
 			if err := os.Remove(qcowPath); err != nil {
@@ -189,26 +189,26 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 	}
 
 	if err != nil {
-		t.Logf("downloading distro image %s to %s", resultDistro.URL, qcowPath)
+		t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
 		fout, err := os.Create(qcowPath)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		if !fetchFromS3(t, fout, resultDistro) {
-			resp, err := http.Get(resultDistro.URL)
+			resp, err := http.Get(resultDistro.url)
 			if err != nil {
-				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.Name, resultDistro.URL, err)
+				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
 			}
 
 			if resp.StatusCode != http.StatusOK {
 				resp.Body.Close()
-				t.Fatalf("%s replied %s", resultDistro.URL, resp.Status)
+				t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
 			}
 
 			_, err = io.Copy(fout, resp.Body)
 			if err != nil {
-				t.Fatalf("download of %s failed: %v", resultDistro.URL, err)
+				t.Fatalf("download of %s failed: %v", resultDistro.url, err)
 			}
 
 			resp.Body.Close()
@@ -219,8 +219,8 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 
 			hash := checkCachedImageHash(t, resultDistro, cdir)
 
-			if hash != resultDistro.SHA256Sum {
-				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.SHA256Sum, hash)
+			if hash != resultDistro.sha256sum {
+				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.sha256sum, hash)
 			}
 		}
 	}
@@ -231,7 +231,7 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string {
 func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) {
 	t.Helper()
 
-	qcowPath := filepath.Join(cacheDir, "qcow2", d.SHA256Sum)
+	qcowPath := filepath.Join(cacheDir, "qcow2", d.sha256sum)
 
 	fin, err := os.Open(qcowPath)
 	if err != nil {
@@ -244,8 +244,8 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri
 	}
 	hash := hex.EncodeToString(hasher.Sum(nil))
 
-	if hash != d.SHA256Sum {
-		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.SHA256Sum)
+	if hash != d.sha256sum {
+		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.sha256sum)
 	}
 
 	gotHash = hash
@@ -255,7 +255,7 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri
 
 func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
 	bins := h.bins
-	if strings.HasPrefix(d.Name, "nixos") {
+	if strings.HasPrefix(d.name, "nixos") {
 		return
 	}
 
@@ -275,7 +275,7 @@ func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) {
 	// TODO(Xe): revisit this assumption before it breaks the test.
 	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
 
-	switch d.InitSystem {
+	switch d.initSystem {
 	case "openrc":
 		mkdir(t, cli, "/etc/init.d")
 		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.openrc", "/etc/init.d/tailscaled")

tstest/integration/vms/vms_test.go

@@ -18,7 +18,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
 	"testing"
 	"text/template"
 	"time"
@@ -58,14 +57,14 @@ func TestDownloadImages(t *testing.T) {
 
 	bins := integration.BuildTestBinaries(t)
 
-	for _, d := range Distros {
+	for _, d := range distros {
 		distro := d
-		t.Run(distro.Name, func(t *testing.T) {
-			if !distroRex.Unwrap().MatchString(distro.Name) {
-				t.Skipf("distro name %q doesn't match regex: %s", distro.Name, distroRex)
+		t.Run(distro.name, func(t *testing.T) {
+			if !distroRex.Unwrap().MatchString(distro.name) {
+				t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex)
 			}
 
-			if strings.HasPrefix(distro.Name, "nixos") {
+			if strings.HasPrefix(distro.name, "nixos") {
 				t.Skip("NixOS is built on the fly, no need to download it")
 			}
 
@@ -99,7 +98,7 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro, qcowBase string) {
 	run(t, tdir, "qemu-img", "create",
 		"-f", "qcow2",
 		"-o", "backing_file="+qcowBase,
-		filepath.Join(tdir, d.Name+".qcow2"),
+		filepath.Join(tdir, d.name+".qcow2"),
 	)
 }
 
@@ -113,7 +112,7 @@ var (
 func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 	t.Helper()
 
-	dir := filepath.Join(tdir, d.Name, "seed")
+	dir := filepath.Join(tdir, d.name, "seed")
 	os.MkdirAll(dir, 0700)
 
 	// make meta-data
@@ -128,7 +127,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			Hostname string
 		}{
 			ID:       "31337",
-			Hostname: d.Name,
+			Hostname: d.name,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -157,7 +156,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 		}{
 			SSHKey:     strings.TrimSpace(sshKey),
 			HostURL:    hostURL,
-			Hostname:   d.Name,
+			Hostname:   d.name,
 			Port:       port,
 			InstallPre: d.InstallPre(),
 			Password:   securePassword,
@@ -221,11 +220,10 @@ func getProbablyFreePortNumber() (int, error) {
 	return portNum, nil
 }
 
-func setupTests(t *testing.T) {
-	ramsem.once.Do(func() {
-		ramsem.sem = semaphore.NewWeighted(int64(*vmRamLimit))
-	})
-
+// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
+// tailscale on it and then ensures that it connects to the network
+// successfully.
+func TestVMIntegrationEndToEnd(t *testing.T) {
 	if !*runVMTests {
 		t.Skip("not running integration tests (need --run-vm-tests)")
 	}
@@ -241,53 +239,56 @@ func setupTests(t *testing.T) {
 		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test --v --timeout=60m --run-vm-tests'")
 		t.Fatalf("missing dependency: %v", err)
 	}
-}
 
-var ramsem struct {
-	once sync.Once
-	sem  *semaphore.Weighted
-}
+	ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
+	rex := distroRex.Unwrap()
 
-func testOneDistribution(t *testing.T, n int, distro Distro) {
-	setupTests(t)
-
-	if distroRex.Unwrap().MatchString(distro.Name) {
-		t.Logf("%s matches %s", distro.Name, distroRex.Unwrap())
-	} else {
-		t.Skip("regex not matched")
-	}
-
-	ctx, done := context.WithCancel(context.Background())
-	t.Cleanup(done)
-
-	h := newHarness(t)
-	dir := t.TempDir()
-
-	err := ramsem.sem.Acquire(ctx, int64(distro.MemoryMegs))
-	if err != nil {
-		t.Fatalf("can't acquire ram semaphore: %v", err)
-	}
-	t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) })
-
-	h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir)
-	var ipm ipMapping
-
-	t.Run("wait-for-start", func(t *testing.T) {
-		waiter := time.NewTicker(time.Second)
-		defer waiter.Stop()
-		var ok bool
-		for {
-			<-waiter.C
-			h.ipMu.Lock()
-			if ipm, ok = h.ipMap[distro.Name]; ok {
-				h.ipMu.Unlock()
-				break
+	t.Run("do", func(t *testing.T) {
+		for n, distro := range distros {
+			n, distro := n, distro
+			if rex.MatchString(distro.name) {
+				t.Logf("%s matches %s", distro.name, rex)
+			} else {
+				continue
 			}
-			h.ipMu.Unlock()
+
+			t.Run(distro.name, func(t *testing.T) {
+				ctx, done := context.WithCancel(context.Background())
+				t.Cleanup(done)
+
+				t.Parallel()
+
+				h := newHarness(t)
+				dir := t.TempDir()
+
+				err := ramsem.Acquire(ctx, int64(distro.mem))
+				if err != nil {
+					t.Fatalf("can't acquire ram semaphore: %v", err)
+				}
+				defer ramsem.Release(int64(distro.mem))
+
+				h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir)
+				var ipm ipMapping
+
+				t.Run("wait-for-start", func(t *testing.T) {
+					waiter := time.NewTicker(time.Second)
+					defer waiter.Stop()
+					var ok bool
+					for {
+						<-waiter.C
+						h.ipMu.Lock()
+						if ipm, ok = h.ipMap[distro.name]; ok {
+							h.ipMu.Unlock()
+							break
+						}
+						h.ipMu.Unlock()
+					}
+				})
+
+				h.testDistro(t, distro, ipm)
+			})
 		}
 	})
-
-	h.testDistro(t, distro, ipm)
 }
 
 func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
@@ -338,7 +339,7 @@ func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
 			&expect.BExp{R: `(\#)`},
 		}
 
-		switch d.InitSystem {
+		switch d.initSystem {
 		case "openrc":
 			// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
 			// of service readiness. If this sleep is removed then tailscale will not be

tstime/mono/mono.go

@@ -1,121 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package mono provides fast monotonic time.
-// On most platforms, mono.Now is about 2x faster than time.Now.
-// However, time.Now is really fast, and nicer to use.
-//
-// For almost all purposes, you should use time.Now.
-//
-// Package mono exists because we get the current time multiple
-// times per network packet, at which point it makes a
-// measurable difference.
-package mono
-
-import (
-	"fmt"
-	"sync/atomic"
-	"time"
-	_ "unsafe" // for go:linkname
-)
-
-// Time is the number of nanoseconds elapsed since an unspecified reference start time.
-type Time int64
-
-// Now returns the current monotonic time.
-func Now() Time {
-	// On a newly started machine, the monotonic clock might be very near zero.
-	// Thus mono.Time(0).Before(mono.Now.Add(-time.Minute)) might yield true.
-	// The corresponding package time expression never does, if the wall clock is correct.
-	// Preserve this correspondence by increasing the "base" monotonic clock by a fair amount.
-	const baseOffset int64 = 1 << 55 // approximately 10,000 hours in nanoseconds
-	return Time(now() + baseOffset)
-}
-
-// Since returns the time elapsed since t.
-func Since(t Time) time.Duration {
-	return time.Duration(Now() - t)
-}
-
-// Sub returns t-n, the duration from n to t.
-func (t Time) Sub(n Time) time.Duration {
-	return time.Duration(t - n)
-}
-
-// Add returns t+d.
-func (t Time) Add(d time.Duration) Time {
-	return t + Time(d)
-}
-
-// After reports t > n, whether t is after n.
-func (t Time) After(n Time) bool {
-	return t > n
-}
-
-// After reports t < n, whether t is before n.
-func (t Time) Before(n Time) bool {
-	return t < n
-}
-
-// IsZero reports whether t == 0.
-func (t Time) IsZero() bool {
-	return t == 0
-}
-
-// StoreAtomic does an atomic store *t = new.
-func (t *Time) StoreAtomic(new Time) {
-	atomic.StoreInt64((*int64)(t), int64(new))
-}
-
-// LoadAtomic does an atomic load *t.
-func (t *Time) LoadAtomic() Time {
-	return Time(atomic.LoadInt64((*int64)(t)))
-}
-
-//go:linkname now runtime.nanotime1
-func now() int64
-
-// baseWall and baseMono are a pair of almost-identical times used to correlate a Time with a wall time.
-var (
-	baseWall time.Time
-	baseMono Time
-)
-
-func init() {
-	baseWall = time.Now()
-	baseMono = Now()
-}
-
-// String prints t, including an estimated equivalent wall clock.
-// This is best-effort only, for rough debugging purposes only.
-// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
-// Even in the best of circumstances, it may vary by a few milliseconds.
-func (t Time) String() string {
-	return fmt.Sprintf("mono.Time(ns=%d, estimated wall=%v)", int64(t), baseWall.Add(t.Sub(baseMono)).Truncate(0))
-}
-
-// MarshalJSON formats t for JSON as if it were a time.Time.
-// We format Time this way for backwards-compatibility.
-// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
-// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
-// Even in the best of circumstances, it may vary by a few milliseconds.
-func (t Time) MarshalJSON() ([]byte, error) {
-	var tt time.Time
-	if !t.IsZero() {
-		tt = baseWall.Add(t.Sub(baseMono)).Truncate(0)
-	}
-	return tt.MarshalJSON()
-}
-
-// UnmarshalJSON sets t according to data.
-// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
-func (t *Time) UnmarshalJSON(data []byte) error {
-	var tt time.Time
-	err := tt.UnmarshalJSON(data)
-	if err != nil {
-		return err
-	}
-	*t = Now().Add(-time.Since(tt))
-	return nil
-}

tstime/mono/mono_test.go

@@ -1,30 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package mono
-
-import (
-	"testing"
-	"time"
-)
-
-func TestNow(t *testing.T) {
-	start := Now()
-	time.Sleep(100 * time.Millisecond)
-	if elapsed := Since(start); elapsed < 100*time.Millisecond {
-		t.Errorf("short sleep: %v elapsed, want min %v", elapsed, 100*time.Millisecond)
-	}
-}
-
-func BenchmarkMonoNow(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		Now()
-	}
-}
-
-func BenchmarkTimeNow(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		time.Now()
-	}
-}

tstime/rate/rate.go

@@ -1,89 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This is a modified, simplified version of code from golang.org/x/time/rate.
-
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package rate provides a rate limiter.
-package rate
-
-import (
-	"sync"
-	"time"
-
-	"tailscale.com/tstime/mono"
-)
-
-// Limit defines the maximum frequency of some events.
-// Limit is represented as number of events per second.
-// A zero Limit is invalid.
-type Limit float64
-
-// Every converts a minimum time interval between events to a Limit.
-func Every(interval time.Duration) Limit {
-	if interval <= 0 {
-		panic("invalid interval")
-	}
-	return 1 / Limit(interval.Seconds())
-}
-
-// A Limiter controls how frequently events are allowed to happen.
-// It implements a "token bucket" of size b, initially full and refilled
-// at rate r tokens per second.
-// Informally, in any large enough time interval, the Limiter limits the
-// rate to r tokens per second, with a maximum burst size of b events.
-// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
-// Use NewLimiter to create non-zero Limiters.
-type Limiter struct {
-	limit  Limit
-	burst  float64
-	mu     sync.Mutex // protects following fields
-	tokens float64    // number of tokens currently in bucket
-	last   mono.Time  // the last time the limiter's tokens field was updated
-}
-
-// NewLimiter returns a new Limiter that allows events up to rate r and permits
-// bursts of at most b tokens.
-func NewLimiter(r Limit, b int) *Limiter {
-	if b < 1 {
-		panic("bad burst, must be at least 1")
-	}
-	return &Limiter{limit: r, burst: float64(b)}
-}
-
-// AllowN reports whether an event may happen now.
-func (lim *Limiter) Allow() bool {
-	return lim.allow(mono.Now())
-}
-
-func (lim *Limiter) allow(now mono.Time) bool {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-
-	// If time has moved backwards, look around awkwardly and pretend nothing happened.
-	if now.Before(lim.last) {
-		lim.last = now
-	}
-
-	// Calculate the new number of tokens available due to the passage of time.
-	elapsed := now.Sub(lim.last)
-	tokens := lim.tokens + float64(lim.limit)*elapsed.Seconds()
-	if tokens > lim.burst {
-		tokens = lim.burst
-	}
-
-	// Consume a token.
-	tokens--
-
-	// Update state.
-	ok := tokens >= 0
-	if ok {
-		lim.last = now
-		lim.tokens = tokens
-	}
-	return ok
-}

tstime/rate/rate_test.go

@@ -1,246 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This is a modified, simplified version of code from golang.org/x/time/rate.
-
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build go1.7
-// +build go1.7
-
-package rate
-
-import (
-	"context"
-	"math"
-	"runtime"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"tailscale.com/tstime/mono"
-)
-
-func closeEnough(a, b Limit) bool {
-	return (math.Abs(float64(a)/float64(b)) - 1.0) < 1e-9
-}
-
-func TestEvery(t *testing.T) {
-	cases := []struct {
-		interval time.Duration
-		lim      Limit
-	}{
-		{1 * time.Nanosecond, Limit(1e9)},
-		{1 * time.Microsecond, Limit(1e6)},
-		{1 * time.Millisecond, Limit(1e3)},
-		{10 * time.Millisecond, Limit(100)},
-		{100 * time.Millisecond, Limit(10)},
-		{1 * time.Second, Limit(1)},
-		{2 * time.Second, Limit(0.5)},
-		{time.Duration(2.5 * float64(time.Second)), Limit(0.4)},
-		{4 * time.Second, Limit(0.25)},
-		{10 * time.Second, Limit(0.1)},
-		{time.Duration(math.MaxInt64), Limit(1e9 / float64(math.MaxInt64))},
-	}
-	for _, tc := range cases {
-		lim := Every(tc.interval)
-		if !closeEnough(lim, tc.lim) {
-			t.Errorf("Every(%v) = %v want %v", tc.interval, lim, tc.lim)
-		}
-	}
-}
-
-const (
-	d = 100 * time.Millisecond
-)
-
-var (
-	t0 = mono.Now()
-	t1 = t0.Add(time.Duration(1) * d)
-	t2 = t0.Add(time.Duration(2) * d)
-	t3 = t0.Add(time.Duration(3) * d)
-	t4 = t0.Add(time.Duration(4) * d)
-	t5 = t0.Add(time.Duration(5) * d)
-	t9 = t0.Add(time.Duration(9) * d)
-)
-
-type allow struct {
-	t  mono.Time
-	ok bool
-}
-
-func run(t *testing.T, lim *Limiter, allows []allow) {
-	t.Helper()
-	for i, allow := range allows {
-		ok := lim.allow(allow.t)
-		if ok != allow.ok {
-			t.Errorf("step %d: lim.AllowN(%v) = %v want %v",
-				i, allow.t, ok, allow.ok)
-		}
-	}
-}
-
-func TestLimiterBurst1(t *testing.T) {
-	run(t, NewLimiter(10, 1), []allow{
-		{t0, true},
-		{t0, false},
-		{t0, false},
-		{t1, true},
-		{t1, false},
-		{t1, false},
-		{t2, true},
-		{t2, false},
-	})
-}
-
-func TestLimiterJumpBackwards(t *testing.T) {
-	run(t, NewLimiter(10, 3), []allow{
-		{t1, true}, // start at t1
-		{t0, true}, // jump back to t0, two tokens remain
-		{t0, true},
-		{t0, false},
-		{t0, false},
-		{t1, true}, // got a token
-		{t1, false},
-		{t1, false},
-		{t2, true}, // got another token
-		{t2, false},
-		{t2, false},
-	})
-}
-
-// Ensure that tokensFromDuration doesn't produce
-// rounding errors by truncating nanoseconds.
-// See golang.org/issues/34861.
-func TestLimiter_noTruncationErrors(t *testing.T) {
-	if !NewLimiter(0.7692307692307693, 1).Allow() {
-		t.Fatal("expected true")
-	}
-}
-
-func TestSimultaneousRequests(t *testing.T) {
-	const (
-		limit       = 1
-		burst       = 5
-		numRequests = 15
-	)
-	var (
-		wg    sync.WaitGroup
-		numOK = uint32(0)
-	)
-
-	// Very slow replenishing bucket.
-	lim := NewLimiter(limit, burst)
-
-	// Tries to take a token, atomically updates the counter and decreases the wait
-	// group counter.
-	f := func() {
-		defer wg.Done()
-		if ok := lim.Allow(); ok {
-			atomic.AddUint32(&numOK, 1)
-		}
-	}
-
-	wg.Add(numRequests)
-	for i := 0; i < numRequests; i++ {
-		go f()
-	}
-	wg.Wait()
-	if numOK != burst {
-		t.Errorf("numOK = %d, want %d", numOK, burst)
-	}
-}
-
-func TestLongRunningQPS(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping in short mode")
-	}
-	if runtime.GOOS == "openbsd" {
-		t.Skip("low resolution time.Sleep invalidates test (golang.org/issue/14183)")
-		return
-	}
-
-	// The test runs for a few seconds executing many requests and then checks
-	// that overall number of requests is reasonable.
-	const (
-		limit = 100
-		burst = 100
-	)
-	var numOK = int32(0)
-
-	lim := NewLimiter(limit, burst)
-
-	var wg sync.WaitGroup
-	f := func() {
-		if ok := lim.Allow(); ok {
-			atomic.AddInt32(&numOK, 1)
-		}
-		wg.Done()
-	}
-
-	start := time.Now()
-	end := start.Add(5 * time.Second)
-	for time.Now().Before(end) {
-		wg.Add(1)
-		go f()
-
-		// This will still offer ~500 requests per second, but won't consume
-		// outrageous amount of CPU.
-		time.Sleep(2 * time.Millisecond)
-	}
-	wg.Wait()
-	elapsed := time.Since(start)
-	ideal := burst + (limit * float64(elapsed) / float64(time.Second))
-
-	// We should never get more requests than allowed.
-	if want := int32(ideal + 1); numOK > want {
-		t.Errorf("numOK = %d, want %d (ideal %f)", numOK, want, ideal)
-	}
-	// We should get very close to the number of requests allowed.
-	if want := int32(0.999 * ideal); numOK < want {
-		t.Errorf("numOK = %d, want %d (ideal %f)", numOK, want, ideal)
-	}
-}
-
-type request struct {
-	t   time.Time
-	n   int
-	act time.Time
-	ok  bool
-}
-
-// dFromDuration converts a duration to a multiple of the global constant d
-func dFromDuration(dur time.Duration) int {
-	// Adding a millisecond to be swallowed by the integer division
-	// because we don't care about small inaccuracies
-	return int((dur + time.Millisecond) / d)
-}
-
-// dSince returns multiples of d since t0
-func dSince(t mono.Time) int {
-	return dFromDuration(t.Sub(t0))
-}
-
-type wait struct {
-	name   string
-	ctx    context.Context
-	n      int
-	delay  int // in multiples of d
-	nilErr bool
-}
-
-func BenchmarkAllowN(b *testing.B) {
-	lim := NewLimiter(Every(1*time.Second), 1)
-	now := mono.Now()
-	b.ReportAllocs()
-	b.ResetTimer()
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			lim.allow(now)
-		}
-	})
-}

tsweb/tsweb.go

@@ -409,7 +409,7 @@ func VarzHandler(w http.ResponseWriter, r *http.Request) {
 		case expvar.Func:
 			val := v()
 			switch val.(type) {
-			case float64, int64, int:
+			case int64, int:
 				fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
 			default:
 				fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)

util/deephash/deephash.go

@@ -2,11 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package deephash hashes a Go value recursively, in a predictable order,
-// without looping. The hash is only valid within the lifetime of a program.
-// Users should not store the hash on disk or send it over the network.
-// The hash is sufficiently strong and unique such that
-// Hash(x) == Hash(y) is an appropriate replacement for x == y.
+// Package deephash hashes a Go value recursively, in a predictable
+// order, without looping.
 //
 // This package, like most of the tailscale.com Go module, should be
 // considered Tailscale-internal; we make no API promises.
@@ -21,9 +18,8 @@ import (
 	"hash"
 	"math"
 	"reflect"
+	"strconv"
 	"sync"
-	"time"
-	"unsafe"
 )
 
 const scratchSize = 128
@@ -31,78 +27,85 @@ const scratchSize = 128
 // hasher is reusable state for hashing a value.
 // Get one via hasherPool.
 type hasher struct {
-	h          hash.Hash
-	bw         *bufio.Writer
-	scratch    [scratchSize]byte
-	visitStack visitStack
+	h       hash.Hash
+	bw      *bufio.Writer
+	scratch [scratchSize]byte
+	visited map[uintptr]bool
 }
 
-func (h *hasher) reset() {
-	if h.h == nil {
-		h.h = sha256.New()
-	}
-	if h.bw == nil {
-		h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
+// newHasher initializes a new hasher, for use by hasherPool.
+func newHasher() *hasher {
+	h := &hasher{
+		h:       sha256.New(),
+		visited: map[uintptr]bool{},
 	}
+	h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
+	return h
+}
+
+// setBufioWriter switches the bufio writer to w after flushing
+// any output to the old one. It then also returns the old one, so
+// the caller can switch back to it.
+func (h *hasher) setBufioWriter(w *bufio.Writer) (old *bufio.Writer) {
+	old = h.bw
+	old.Flush()
+	h.bw = w
+	return old
+}
+
+// Hash returns the raw SHA-256 (not hex) of v.
+func (h *hasher) Hash(v interface{}) (hash [sha256.Size]byte) {
 	h.bw.Flush()
 	h.h.Reset()
-}
-
-// Sum is an opaque checksum type that is comparable.
-type Sum struct {
-	sum [sha256.Size]byte
-}
-
-func (s1 *Sum) xor(s2 Sum) {
-	for i := 0; i < sha256.Size; i++ {
-		s1.sum[i] ^= s2.sum[i]
-	}
-}
-
-func (s Sum) String() string {
-	return hex.EncodeToString(s.sum[:])
-}
-
-var (
-	once sync.Once
-	seed uint64
-)
-
-func (h *hasher) sum() (s Sum) {
+	h.print(reflect.ValueOf(v))
 	h.bw.Flush()
 	// Sum into scratch & copy out, as hash.Hash is an interface
 	// so the slice necessarily escapes, and there's no sha256
 	// concrete type exported and we don't want the 'hash' result
 	// parameter to escape to the heap:
-	copy(s.sum[:], h.h.Sum(h.scratch[:0]))
-	return s
+	h.h.Sum(h.scratch[:0])
+	copy(hash[:], h.scratch[:])
+	return
 }
 
 var hasherPool = &sync.Pool{
-	New: func() interface{} { return new(hasher) },
+	New: func() interface{} { return newHasher() },
 }
 
-// Hash returns the hash of v.
-func Hash(v interface{}) (s Sum) {
+// Hash returns the raw SHA-256 hash of v.
+func Hash(v interface{}) [sha256.Size]byte {
 	h := hasherPool.Get().(*hasher)
 	defer hasherPool.Put(h)
-	h.reset()
-	once.Do(func() {
-		seed = uint64(time.Now().UnixNano())
-	})
-	h.hashUint64(seed)
-	h.hashValue(reflect.ValueOf(v))
-	return h.sum()
+	for k := range h.visited {
+		delete(h.visited, k)
+	}
+	return h.Hash(v)
 }
 
-// Update sets last to the hash of v and reports whether its value changed.
-func Update(last *Sum, v ...interface{}) (changed bool) {
+// UpdateHash sets last to the hex-encoded hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
 	sum := Hash(v)
-	if sum == *last {
+	if sha256EqualHex(sum, *last) {
 		// unchanged.
 		return false
 	}
-	*last = sum
+	*last = hex.EncodeToString(sum[:])
+	return true
+}
+
+// sha256EqualHex reports whether hx is the hex encoding of sum.
+func sha256EqualHex(sum [sha256.Size]byte, hx string) bool {
+	if len(hx) != len(sum)*2 {
+		return false
+	}
+	const hextable = "0123456789abcdef"
+	j := 0
+	for _, v := range sum {
+		if hx[j] != hextable[v>>4] || hx[j+1] != hextable[v&0x0f] {
+			return false
+		}
+		j += 2
+	}
 	return true
 }
 
@@ -112,78 +115,64 @@ type appenderTo interface {
 	AppendTo([]byte) []byte
 }
 
-func (h *hasher) hashUint8(i uint8) {
-	h.bw.WriteByte(i)
+func (h *hasher) uint(i uint64) {
+	binary.BigEndian.PutUint64(h.scratch[:8], i)
+	h.bw.Write(h.scratch[:8])
 }
-func (h *hasher) hashUint16(i uint16) {
-	binary.LittleEndian.PutUint16(h.scratch[:2], i)
-	h.bw.Write(h.scratch[:2])
-}
-func (h *hasher) hashUint32(i uint32) {
-	binary.LittleEndian.PutUint32(h.scratch[:4], i)
-	h.bw.Write(h.scratch[:4])
-}
-func (h *hasher) hashUint64(i uint64) {
-	binary.LittleEndian.PutUint64(h.scratch[:8], i)
+
+func (h *hasher) int(i int) {
+	binary.BigEndian.PutUint64(h.scratch[:8], uint64(i))
 	h.bw.Write(h.scratch[:8])
 }
 
 var uint8Type = reflect.TypeOf(byte(0))
 
-func (h *hasher) hashValue(v reflect.Value) {
+// print hashes v into w.
+// It reports whether it was able to do so without hitting a cycle.
+func (h *hasher) print(v reflect.Value) (acyclic bool) {
 	if !v.IsValid() {
-		return
+		return true
 	}
 
 	w := h.bw
+	visited := h.visited
 
 	if v.CanInterface() {
 		// Use AppendTo methods, if available and cheap.
 		if v.CanAddr() && v.Type().Implements(appenderToType) {
 			a := v.Addr().Interface().(appenderTo)
-			size := h.scratch[:8]
-			record := a.AppendTo(size)
-			binary.LittleEndian.PutUint64(record, uint64(len(record)-len(size)))
-			w.Write(record)
-			return
+			scratch := a.AppendTo(h.scratch[:0])
+			w.Write(scratch)
+			return true
 		}
 	}
 
-	// TODO(dsnet): Avoid cycle detection for types that cannot have cycles.
-
 	// Generic handling.
 	switch v.Kind() {
 	default:
 		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
 	case reflect.Ptr:
-		if v.IsNil() {
-			h.hashUint8(0) // indicates nil
-			return
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return false
 		}
-
-		// Check for cycle.
-		ptr := pointerOf(v)
-		if idx, ok := h.visitStack.seen(ptr); ok {
-			h.hashUint8(2) // indicates cycle
-			h.hashUint64(uint64(idx))
-			return
-		}
-		h.visitStack.push(ptr)
-		defer h.visitStack.pop(ptr)
-
-		h.hashUint8(1) // indicates visiting a pointer
-		h.hashValue(v.Elem())
+		visited[ptr] = true
+		return h.print(v.Elem())
 	case reflect.Struct:
+		acyclic = true
 		w.WriteString("struct")
-		h.hashUint64(uint64(v.NumField()))
+		h.int(v.NumField())
 		for i, n := 0, v.NumField(); i < n; i++ {
-			h.hashUint64(uint64(i))
-			h.hashValue(v.Field(i))
+			h.int(i)
+			if !h.print(v.Field(i)) {
+				acyclic = false
+			}
 		}
+		return acyclic
 	case reflect.Slice, reflect.Array:
 		vLen := v.Len()
 		if v.Kind() == reflect.Slice {
-			h.hashUint64(uint64(vLen))
+			h.int(vLen)
 		}
 		if v.Type().Elem() == uint8Type && v.CanInterface() {
 			if vLen > 0 && vLen <= scratchSize {
@@ -193,167 +182,164 @@ func (h *hasher) hashValue(v reflect.Value) {
 				// to allocate, so it's not a win.
 				n := reflect.Copy(reflect.ValueOf(&h.scratch).Elem(), v)
 				w.Write(h.scratch[:n])
-				return
+				return true
 			}
 			fmt.Fprintf(w, "%s", v.Interface())
-			return
+			return true
 		}
+		acyclic = true
 		for i := 0; i < vLen; i++ {
-			// TODO(dsnet): Perform cycle detection for slices,
-			// which is functionally a list of pointers.
-			// See https://github.com/google/go-cmp/blob/402949e8139bb890c71a707b6faf6dd05c92f4e5/cmp/compare.go#L438-L450
-			h.hashUint64(uint64(i))
-			h.hashValue(v.Index(i))
+			h.int(i)
+			if !h.print(v.Index(i)) {
+				acyclic = false
+			}
 		}
+		return acyclic
 	case reflect.Interface:
-		if v.IsNil() {
-			h.hashUint8(0) // indicates nil
-			return
-		}
-		v = v.Elem()
-
-		h.hashUint8(1) // indicates visiting interface value
-		h.hashType(v.Type())
-		h.hashValue(v)
+		return h.print(v.Elem())
 	case reflect.Map:
-		// Check for cycle.
-		ptr := pointerOf(v)
-		if idx, ok := h.visitStack.seen(ptr); ok {
-			h.hashUint8(2) // indicates cycle
-			h.hashUint64(uint64(idx))
-			return
+		// TODO(bradfitz): ideally we'd avoid these map
+		// operations to detect cycles if we knew from the map
+		// element type that there no way to form a cycle,
+		// which is the common case. Notably, we don't care
+		// about hashing the same map+contents twice in
+		// different parts of the tree. In fact, we should
+		// ideally. (And this prevents it) We should only stop
+		// hashing when there's a cycle.  What we should
+		// probably do is make sure we enumerate the data
+		// structure tree is a fixed order and then give each
+		// pointer an increasing number, and when we hit a
+		// dup, rather than emitting nothing, we should emit a
+		// "value #12" reference. Which implies that all things
+		// emit to the bufio.Writer should be type-tagged so
+		// we can distinguish loop references without risk of
+		// collisions.
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return false
 		}
-		h.visitStack.push(ptr)
-		defer h.visitStack.pop(ptr)
+		visited[ptr] = true
 
-		h.hashUint8(1) // indicates visiting a map
-		h.hashMap(v)
-	case reflect.String:
-		s := v.String()
-		h.hashUint64(uint64(len(s)))
-		w.WriteString(s)
-	case reflect.Bool:
-		if v.Bool() {
-			h.hashUint8(1)
-		} else {
-			h.hashUint8(0)
+		if h.hashMapAcyclic(v) {
+			return true
 		}
-	case reflect.Int8:
-		h.hashUint8(uint8(v.Int()))
-	case reflect.Int16:
-		h.hashUint16(uint16(v.Int()))
-	case reflect.Int32:
-		h.hashUint32(uint32(v.Int()))
-	case reflect.Int64, reflect.Int:
-		h.hashUint64(uint64(v.Int()))
-	case reflect.Uint8:
-		h.hashUint8(uint8(v.Uint()))
-	case reflect.Uint16:
-		h.hashUint16(uint16(v.Uint()))
-	case reflect.Uint32:
-		h.hashUint32(uint32(v.Uint()))
-	case reflect.Uint64, reflect.Uint, reflect.Uintptr:
-		h.hashUint64(uint64(v.Uint()))
-	case reflect.Float32:
-		h.hashUint32(math.Float32bits(float32(v.Float())))
-	case reflect.Float64:
-		h.hashUint64(math.Float64bits(float64(v.Float())))
-	case reflect.Complex64:
-		h.hashUint32(math.Float32bits(real(complex64(v.Complex()))))
-		h.hashUint32(math.Float32bits(imag(complex64(v.Complex()))))
-	case reflect.Complex128:
-		h.hashUint64(math.Float64bits(real(complex128(v.Complex()))))
-		h.hashUint64(math.Float64bits(imag(complex128(v.Complex()))))
+		return h.hashMapFallback(v)
+	case reflect.String:
+		h.int(v.Len())
+		w.WriteString(v.String())
+	case reflect.Bool:
+		w.Write(strconv.AppendBool(h.scratch[:0], v.Bool()))
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		w.Write(strconv.AppendInt(h.scratch[:0], v.Int(), 10))
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		h.uint(v.Uint())
+	case reflect.Float32, reflect.Float64:
+		w.Write(strconv.AppendUint(h.scratch[:0], math.Float64bits(v.Float()), 10))
+	case reflect.Complex64, reflect.Complex128:
+		fmt.Fprintf(w, "%v", v.Complex())
 	}
+	return true
 }
 
 type mapHasher struct {
-	h    hasher
-	val  valueCache      // re-usable values for map iteration
-	iter reflect.MapIter // re-usable map iterator
+	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
+	ebuf [sha256.Size]byte // scratch buffer
+	s256 hash.Hash         // sha256 hash.Hash
+	bw   *bufio.Writer     // to hasher into ebuf
+	val  valueCache        // re-usable values for map iteration
+	iter *reflect.MapIter  // re-usable map iterator
+}
+
+func (mh *mapHasher) Reset() {
+	for i := range mh.xbuf {
+		mh.xbuf[i] = 0
+	}
+}
+
+func (mh *mapHasher) startEntry() {
+	for i := range mh.ebuf {
+		mh.ebuf[i] = 0
+	}
+	mh.bw.Flush()
+	mh.s256.Reset()
+}
+
+func (mh *mapHasher) endEntry() {
+	mh.bw.Flush()
+	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
+		mh.xbuf[i] ^= b
+	}
 }
 
 var mapHasherPool = &sync.Pool{
-	New: func() interface{} { return new(mapHasher) },
+	New: func() interface{} {
+		mh := new(mapHasher)
+		mh.s256 = sha256.New()
+		mh.bw = bufio.NewWriter(mh.s256)
+		mh.val = make(valueCache)
+		mh.iter = new(reflect.MapIter)
+		return mh
+	},
 }
 
 type valueCache map[reflect.Type]reflect.Value
 
-func (c *valueCache) get(t reflect.Type) reflect.Value {
-	v, ok := (*c)[t]
+func (c valueCache) get(t reflect.Type) reflect.Value {
+	v, ok := c[t]
 	if !ok {
 		v = reflect.New(t).Elem()
-		if *c == nil {
-			*c = make(valueCache)
-		}
-		(*c)[t] = v
+		c[t] = v
 	}
 	return v
 }
 
-// hashMap hashes a map in a sort-free manner.
-// It relies on a map being a functionally an unordered set of KV entries.
-// So long as we hash each KV entry together, we can XOR all
-// of the individual hashes to produce a unique hash for the entire map.
-func (h *hasher) hashMap(v reflect.Value) {
+// hashMapAcyclic is the faster sort-free version of map hashing. If
+// it detects a cycle it returns false and guarantees that nothing was
+// written to w.
+func (h *hasher) hashMapAcyclic(v reflect.Value) (acyclic bool) {
 	mh := mapHasherPool.Get().(*mapHasher)
 	defer mapHasherPool.Put(mh)
-	iter := mapIter(&mh.iter, v)
-	defer mapIter(&mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+	mh.Reset()
+	iter := mapIter(mh.iter, v)
+	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+
+	// Temporarily switch to the map hasher's bufio.Writer.
+	oldw := h.setBufioWriter(mh.bw)
+	defer h.setBufioWriter(oldw)
 
-	var sum Sum
 	k := mh.val.get(v.Type().Key())
 	e := mh.val.get(v.Type().Elem())
-	mh.h.visitStack = h.visitStack // always use the parent's visit stack to avoid cycles
 	for iter.Next() {
 		key := iterKey(iter, k)
 		val := iterVal(iter, e)
-		mh.h.reset()
-		mh.h.hashValue(key)
-		mh.h.hashValue(val)
-		sum.xor(mh.h.sum())
+		mh.startEntry()
+		if !h.print(key) {
+			return false
+		}
+		if !h.print(val) {
+			return false
+		}
+		mh.endEntry()
 	}
-	h.bw.Write(append(h.scratch[:0], sum.sum[:]...)) // append into scratch to avoid heap allocation
+	oldw.Write(mh.xbuf[:])
+	return true
 }
 
-// visitStack is a stack of pointers visited.
-// Pointers are pushed onto the stack when visited, and popped when leaving.
-// The integer value is the depth at which the pointer was visited.
-// The length of this stack should be zero after every hashing operation.
-type visitStack map[pointer]int
-
-func (v visitStack) seen(p pointer) (int, bool) {
-	idx, ok := v[p]
-	return idx, ok
-}
-
-func (v *visitStack) push(p pointer) {
-	if *v == nil {
-		*v = make(map[pointer]int)
+func (h *hasher) hashMapFallback(v reflect.Value) (acyclic bool) {
+	acyclic = true
+	sm := newSortedMap(v)
+	w := h.bw
+	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+	for i, k := range sm.Key {
+		if !h.print(k) {
+			acyclic = false
+		}
+		w.WriteString(": ")
+		if !h.print(sm.Value[i]) {
+			acyclic = false
+		}
+		w.WriteString("\n")
 	}
-	(*v)[p] = len(*v)
-}
-
-func (v visitStack) pop(p pointer) {
-	delete(v, p)
-}
-
-// pointer is a thin wrapper over unsafe.Pointer.
-// We only rely on comparability of pointers; we cannot rely on uintptr since
-// that would break if Go ever switched to a moving GC.
-type pointer struct{ p unsafe.Pointer }
-
-func pointerOf(v reflect.Value) pointer {
-	return pointer{unsafe.Pointer(v.Pointer())}
-}
-
-// hashType hashes a reflect.Type.
-// The hash is only consistent within the lifetime of a program.
-func (h *hasher) hashType(t reflect.Type) {
-	// This approach relies on reflect.Type always being backed by a unique
-	// *reflect.rtype pointer. A safer approach is to use a global sync.Map
-	// that maps reflect.Type to some arbitrary and unique index.
-	// While safer, it requires global state with memory that can never be GC'd.
-	rtypeAddr := reflect.ValueOf(t).Pointer() // address of *reflect.rtype
-	h.hashUint64(uint64(rtypeAddr))
+	w.WriteString("}\n")
+	return acyclic
 }

util/deephash/deephash_test.go

@@ -5,11 +5,11 @@
 package deephash
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
-	"math"
 	"reflect"
 	"testing"
 
@@ -23,95 +23,6 @@ import (
 	"tailscale.com/wgengine/wgcfg"
 )
 
-type appendBytes []byte
-
-func (p appendBytes) AppendTo(b []byte) []byte {
-	return append(b, p...)
-}
-
-func TestHash(t *testing.T) {
-	type tuple [2]interface{}
-	type iface struct{ X interface{} }
-	type scalars struct {
-		I8   int8
-		I16  int16
-		I32  int32
-		I64  int64
-		I    int
-		U8   uint8
-		U16  uint16
-		U32  uint32
-		U64  uint64
-		U    uint
-		UP   uintptr
-		F32  float32
-		F64  float64
-		C64  complex64
-		C128 complex128
-	}
-	type MyBool bool
-	type MyHeader tar.Header
-	tests := []struct {
-		in     tuple
-		wantEq bool
-	}{
-		{in: tuple{false, true}, wantEq: false},
-		{in: tuple{true, true}, wantEq: true},
-		{in: tuple{false, false}, wantEq: true},
-		{
-			in: tuple{
-				scalars{-8, -16, -32, -64, -1234, 8, 16, 32, 64, 1234, 5678, 32.32, 64.64, 32 + 32i, 64 + 64i},
-				scalars{-8, -16, -32, -64, -1234, 8, 16, 32, 64, 1234, 5678, 32.32, 64.64, 32 + 32i, 64 + 64i},
-			},
-			wantEq: true,
-		},
-		{in: tuple{scalars{I8: math.MinInt8}, scalars{I8: math.MinInt8 / 2}}, wantEq: false},
-		{in: tuple{scalars{I16: math.MinInt16}, scalars{I16: math.MinInt16 / 2}}, wantEq: false},
-		{in: tuple{scalars{I32: math.MinInt32}, scalars{I32: math.MinInt32 / 2}}, wantEq: false},
-		{in: tuple{scalars{I64: math.MinInt64}, scalars{I64: math.MinInt64 / 2}}, wantEq: false},
-		{in: tuple{scalars{I: -1234}, scalars{I: -1234 / 2}}, wantEq: false},
-		{in: tuple{scalars{U8: math.MaxUint8}, scalars{U8: math.MaxUint8 / 2}}, wantEq: false},
-		{in: tuple{scalars{U16: math.MaxUint16}, scalars{U16: math.MaxUint16 / 2}}, wantEq: false},
-		{in: tuple{scalars{U32: math.MaxUint32}, scalars{U32: math.MaxUint32 / 2}}, wantEq: false},
-		{in: tuple{scalars{U64: math.MaxUint64}, scalars{U64: math.MaxUint64 / 2}}, wantEq: false},
-		{in: tuple{scalars{U: 1234}, scalars{U: 1234 / 2}}, wantEq: false},
-		{in: tuple{scalars{UP: 5678}, scalars{UP: 5678 / 2}}, wantEq: false},
-		{in: tuple{scalars{F32: 32.32}, scalars{F32: math.Nextafter32(32.32, 0)}}, wantEq: false},
-		{in: tuple{scalars{F64: 64.64}, scalars{F64: math.Nextafter(64.64, 0)}}, wantEq: false},
-		{in: tuple{scalars{F32: float32(math.NaN())}, scalars{F32: float32(math.NaN())}}, wantEq: true},
-		{in: tuple{scalars{F64: float64(math.NaN())}, scalars{F64: float64(math.NaN())}}, wantEq: true},
-		{in: tuple{scalars{C64: 32 + 32i}, scalars{C64: complex(math.Nextafter32(32, 0), 32)}}, wantEq: false},
-		{in: tuple{scalars{C128: 64 + 64i}, scalars{C128: complex(math.Nextafter(64, 0), 64)}}, wantEq: false},
-		{in: tuple{[]appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}, []appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}}, wantEq: true},
-		{in: tuple{[]appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}, []appendBytes{{0, 0, 0, 0, 0, 0, 0, 1}, {}}}, wantEq: false},
-		{in: tuple{iface{MyBool(true)}, iface{MyBool(true)}}, wantEq: true},
-		{in: tuple{iface{true}, iface{MyBool(true)}}, wantEq: false},
-		{in: tuple{iface{MyHeader{}}, iface{MyHeader{}}}, wantEq: true},
-		{in: tuple{iface{MyHeader{}}, iface{tar.Header{}}}, wantEq: false},
-		{in: tuple{iface{&MyHeader{}}, iface{&MyHeader{}}}, wantEq: true},
-		{in: tuple{iface{&MyHeader{}}, iface{&tar.Header{}}}, wantEq: false},
-		{in: tuple{iface{[]map[string]MyBool{}}, iface{[]map[string]MyBool{}}}, wantEq: true},
-		{in: tuple{iface{[]map[string]bool{}}, iface{[]map[string]MyBool{}}}, wantEq: false},
-		{
-			in: func() tuple {
-				i1 := 1
-				i2 := 2
-				v1 := [3]*int{&i1, &i2, &i1}
-				v2 := [3]*int{&i1, &i2, &i2}
-				return tuple{v1, v2}
-			}(),
-			wantEq: false,
-		},
-	}
-
-	for _, tt := range tests {
-		gotEq := Hash(tt.in[0]) == Hash(tt.in[1])
-		if gotEq != tt.wantEq {
-			t.Errorf("(Hash(%v) == Hash(%v)) = %v, want %v", tt.in[0], tt.in[1], gotEq, tt.wantEq)
-		}
-	}
-}
-
 func TestDeepHash(t *testing.T) {
 	// v contains the types of values we care about for our current callers.
 	// Mostly we're just testing that we don't panic on handled types.
@@ -247,8 +158,13 @@ func TestHashMapAcyclic(t *testing.T) {
 		v := reflect.ValueOf(m)
 		buf.Reset()
 		bw.Reset(&buf)
-		h := &hasher{bw: bw}
-		h.hashMap(v)
+		h := &hasher{
+			bw:      bw,
+			visited: map[uintptr]bool{},
+		}
+		if !h.hashMapAcyclic(v) {
+			t.Fatal("returned false")
+		}
 		if got[string(buf.Bytes())] {
 			continue
 		}
@@ -263,14 +179,17 @@ func TestPrintArray(t *testing.T) {
 	type T struct {
 		X [32]byte
 	}
-	x := T{X: [32]byte{1: 1, 31: 31}}
+	x := &T{X: [32]byte{1: 1, 31: 31}}
 	var got bytes.Buffer
 	bw := bufio.NewWriter(&got)
-	h := &hasher{bw: bw}
-	h.hashValue(reflect.ValueOf(x))
+	h := &hasher{
+		bw:      bw,
+		visited: map[uintptr]bool{},
+	}
+	h.print(reflect.ValueOf(x))
 	bw.Flush()
 	const want = "struct" +
-		"\x01\x00\x00\x00\x00\x00\x00\x00" + // 1 field
+		"\x00\x00\x00\x00\x00\x00\x00\x01" + // 1 field
 		"\x00\x00\x00\x00\x00\x00\x00\x00" + // 0th field
 		// the 32 bytes:
 		"\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f"
@@ -290,12 +209,17 @@ func BenchmarkHashMapAcyclic(b *testing.B) {
 	bw := bufio.NewWriter(&buf)
 	v := reflect.ValueOf(m)
 
-	h := &hasher{bw: bw}
+	h := &hasher{
+		bw:      bw,
+		visited: map[uintptr]bool{},
+	}
 
 	for i := 0; i < b.N; i++ {
 		buf.Reset()
 		bw.Reset(&buf)
-		h.hashMap(v)
+		if !h.hashMapAcyclic(v) {
+			b.Fatal("returned false")
+		}
 	}
 }
 
@@ -309,7 +233,7 @@ func BenchmarkTailcfgNode(b *testing.B) {
 }
 
 func TestExhaustive(t *testing.T) {
-	seen := make(map[Sum]bool)
+	seen := make(map[[sha256.Size]byte]bool)
 	for i := 0; i < 100000; i++ {
 		s := Hash(i)
 		if seen[s] {
@@ -319,6 +243,19 @@ func TestExhaustive(t *testing.T) {
 	}
 }
 
+func TestSHA256EqualHex(t *testing.T) {
+	for i := 0; i < 1000; i++ {
+		sum := Hash(i)
+		hx := hex.EncodeToString(sum[:])
+		if !sha256EqualHex(sum, hx) {
+			t.Fatal("didn't match, should've")
+		}
+		if sha256EqualHex(sum, hx[:len(hx)-1]) {
+			t.Fatal("matched on wrong length")
+		}
+	}
+}
+
 // verify this doesn't loop forever, as it used to (Issue 2340)
 func TestMapCyclicFallback(t *testing.T) {
 	type T struct {

util/deephash/fmtsort.go

@@ -0,0 +1,224 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// and
+
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
+
+package deephash
+
+import (
+	"reflect"
+	"sort"
+)
+
+// Note: Throughout this package we avoid calling reflect.Value.Interface as
+// it is not always legal to do so and it's easier to avoid the issue than to face it.
+
+// sortedMap represents a map's keys and values. The keys and values are
+// aligned in index order: Value[i] is the value in the map corresponding to Key[i].
+type sortedMap struct {
+	Key   []reflect.Value
+	Value []reflect.Value
+}
+
+func (o *sortedMap) Len() int           { return len(o.Key) }
+func (o *sortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 }
+func (o *sortedMap) Swap(i, j int) {
+	o.Key[i], o.Key[j] = o.Key[j], o.Key[i]
+	o.Value[i], o.Value[j] = o.Value[j], o.Value[i]
+}
+
+// Sort accepts a map and returns a sortedMap that has the same keys and
+// values but in a stable sorted order according to the keys, modulo issues
+// raised by unorderable key values such as NaNs.
+//
+// The ordering rules are more general than with Go's < operator:
+//
+//  - when applicable, nil compares low
+//  - ints, floats, and strings order by <
+//  - NaN compares less than non-NaN floats
+//  - bool compares false before true
+//  - complex compares real, then imag
+//  - pointers compare by machine address
+//  - channel values compare by machine address
+//  - structs compare each field in turn
+//  - arrays compare each element in turn.
+//    Otherwise identical arrays compare by length.
+//  - interface values compare first by reflect.Type describing the concrete type
+//    and then by concrete value as described in the previous rules.
+//
+func newSortedMap(mapValue reflect.Value) *sortedMap {
+	if mapValue.Type().Kind() != reflect.Map {
+		return nil
+	}
+	// Note: this code is arranged to not panic even in the presence
+	// of a concurrent map update. The runtime is responsible for
+	// yelling loudly if that happens. See issue 33275.
+	n := mapValue.Len()
+	key := make([]reflect.Value, 0, n)
+	value := make([]reflect.Value, 0, n)
+	iter := mapValue.MapRange()
+	for iter.Next() {
+		key = append(key, iter.Key())
+		value = append(value, iter.Value())
+	}
+	sorted := &sortedMap{
+		Key:   key,
+		Value: value,
+	}
+	sort.Stable(sorted)
+	return sorted
+}
+
+// compare compares two values of the same type. It returns -1, 0, 1
+// according to whether a > b (1), a == b (0), or a < b (-1).
+// If the types differ, it returns -1.
+// See the comment on Sort for the comparison rules.
+func compare(aVal, bVal reflect.Value) int {
+	aType, bType := aVal.Type(), bVal.Type()
+	if aType != bType {
+		return -1 // No good answer possible, but don't return 0: they're not equal.
+	}
+	switch aVal.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		a, b := aVal.Int(), bVal.Int()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		a, b := aVal.Uint(), bVal.Uint()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.String:
+		a, b := aVal.String(), bVal.String()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Float32, reflect.Float64:
+		return floatCompare(aVal.Float(), bVal.Float())
+	case reflect.Complex64, reflect.Complex128:
+		a, b := aVal.Complex(), bVal.Complex()
+		if c := floatCompare(real(a), real(b)); c != 0 {
+			return c
+		}
+		return floatCompare(imag(a), imag(b))
+	case reflect.Bool:
+		a, b := aVal.Bool(), bVal.Bool()
+		switch {
+		case a == b:
+			return 0
+		case a:
+			return 1
+		default:
+			return -1
+		}
+	case reflect.Ptr:
+		a, b := aVal.Pointer(), bVal.Pointer()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Chan:
+		if c, ok := nilCompare(aVal, bVal); ok {
+			return c
+		}
+		ap, bp := aVal.Pointer(), bVal.Pointer()
+		switch {
+		case ap < bp:
+			return -1
+		case ap > bp:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Struct:
+		for i := 0; i < aVal.NumField(); i++ {
+			if c := compare(aVal.Field(i), bVal.Field(i)); c != 0 {
+				return c
+			}
+		}
+		return 0
+	case reflect.Array:
+		for i := 0; i < aVal.Len(); i++ {
+			if c := compare(aVal.Index(i), bVal.Index(i)); c != 0 {
+				return c
+			}
+		}
+		return 0
+	case reflect.Interface:
+		if c, ok := nilCompare(aVal, bVal); ok {
+			return c
+		}
+		c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type()))
+		if c != 0 {
+			return c
+		}
+		return compare(aVal.Elem(), bVal.Elem())
+	default:
+		// Certain types cannot appear as keys (maps, funcs, slices), but be explicit.
+		panic("bad type in compare: " + aType.String())
+	}
+}
+
+// nilCompare checks whether either value is nil. If not, the boolean is false.
+// If either value is nil, the boolean is true and the integer is the comparison
+// value. The comparison is defined to be 0 if both are nil, otherwise the one
+// nil value compares low. Both arguments must represent a chan, func,
+// interface, map, pointer, or slice.
+func nilCompare(aVal, bVal reflect.Value) (int, bool) {
+	if aVal.IsNil() {
+		if bVal.IsNil() {
+			return 0, true
+		}
+		return -1, true
+	}
+	if bVal.IsNil() {
+		return 1, true
+	}
+	return 0, false
+}
+
+// floatCompare compares two floating-point values. NaNs compare low.
+func floatCompare(a, b float64) int {
+	switch {
+	case isNaN(a):
+		return -1 // No good answer if b is a NaN so don't bother checking.
+	case isNaN(b):
+		return 1
+	case a < b:
+		return -1
+	case a > b:
+		return 1
+	}
+	return 0
+}
+
+func isNaN(a float64) bool {
+	return a != a
+}

version/version.go

@@ -8,7 +8,7 @@ package version
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-var Long = "date.20210727"
+var Long = "date.20210603"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.

wgengine/filter/filter.go

@@ -11,10 +11,10 @@ import (
 	"sync"
 	"time"
 
+	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/packet"
-	"tailscale.com/tstime/rate"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 )

wgengine/filter/filter_test.go

@@ -13,11 +13,11 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
-	"tailscale.com/tstime/rate"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 )

wgengine/magicsock/legacy.go

@@ -23,7 +23,6 @@ import (
 	"golang.zx2c4.com/wireguard/tai64n"
 	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
@@ -349,12 +348,12 @@ type addrSet struct {
 	ipPorts []netaddr.IPPort
 
 	// clock, if non-nil, is used in tests instead of time.Now.
-	clock func() mono.Time
+	clock func() time.Time
 	Logf  logger.Logf // must not be nil
 
 	mu sync.Mutex // guards following fields
 
-	lastSend mono.Time
+	lastSend time.Time
 
 	// roamAddr is non-nil if/when we receive a correctly signed
 	// WireGuard packet from an unexpected address. If so, we
@@ -370,10 +369,10 @@ type addrSet struct {
 	curAddr int
 
 	// stopSpray is the time after which we stop spraying packets.
-	stopSpray mono.Time
+	stopSpray time.Time
 
 	// lastSpray is the last time we sprayed a packet.
-	lastSpray mono.Time
+	lastSpray time.Time
 
 	// loggedLogPriMask is a bit field of that tracks whether
 	// we've already logged about receiving a packet from a low
@@ -396,11 +395,11 @@ func (as *addrSet) derpID() int {
 	return 0
 }
 
-func (as *addrSet) timeNow() mono.Time {
+func (as *addrSet) timeNow() time.Time {
 	if as.clock != nil {
 		return as.clock()
 	}
-	return mono.Now()
+	return time.Now()
 }
 
 var noAddr, _ = netaddr.FromStdAddr(net.ParseIP("127.127.127.127"), 127, "")

wgengine/magicsock/magicsock.go

@@ -47,7 +47,6 @@ import (
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -84,8 +83,6 @@ var (
 	// on mobile devices, lowers the shutdown interval, and logs more
 	// verbosely about idle measurements.
 	debugReSTUNStopOnIdle, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_RESTUN_STOP_ON_IDLE"))
-	// debugAlwaysDERP disables the use of UDP, forcing all peer communication over DERP.
-	debugAlwaysDERP, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_ALWAYS_USE_DERP"))
 )
 
 // useDerpRoute reports whether magicsock should enable the DERP
@@ -815,20 +812,20 @@ func (c *Conn) SetNetInfoCallback(fn func(*tailcfg.NetInfo)) {
 	}
 }
 
-// LastRecvActivityOfDisco describes the time we last got traffic from
+// LastRecvActivityOfDisco returns the time we last got traffic from
 // this endpoint (updated every ~10 seconds).
-func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) string {
+func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) time.Time {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	de, ok := c.endpointOfDisco[dk]
 	if !ok {
-		return "never"
+		return time.Time{}
 	}
-	saw := de.lastRecv.LoadAtomic()
-	if saw == 0 {
-		return "never"
+	unix := atomic.LoadInt64(&de.lastRecvUnixAtomic)
+	if unix == 0 {
+		return time.Time{}
 	}
-	return mono.Since(saw).Round(time.Second).String()
+	return time.Unix(unix, 0)
 }
 
 // Ping handles a "tailscale ping" CLI query.
@@ -2668,12 +2665,6 @@ func (c *Conn) bindSocket(rucPtr **RebindingUDPConn, network string, curPortFate
 	ruc.mu.Lock()
 	defer ruc.mu.Unlock()
 
-	if debugAlwaysDERP {
-		c.logf("disabled %v per TS_DEBUG_ALWAYS_USE_DERP", network)
-		ruc.pconn = newBlockForeverConn()
-		return nil
-	}
-
 	// Build a list of preferred ports.
 	// Best is the port that the user requested.
 	// Second best is the port that is currently in use.
@@ -3127,7 +3118,7 @@ func ippDebugString(ua netaddr.IPPort) string {
 // advertise a DiscoKey and participate in active discovery.
 type discoEndpoint struct {
 	// atomically accessed; declared first for alignment reasons
-	lastRecv              mono.Time
+	lastRecvUnixAtomic    int64
 	numStopAndResetAtomic int64
 
 	// These fields are initialized once and never modified.
@@ -3146,13 +3137,13 @@ type discoEndpoint struct {
 	mu sync.Mutex // Lock ordering: Conn.mu, then discoEndpoint.mu
 
 	heartBeatTimer *time.Timer    // nil when idle
-	lastSend       mono.Time      // last time there was outgoing packets sent to this peer (from wireguard-go)
-	lastFullPing   mono.Time      // last time we pinged all endpoints
+	lastSend       time.Time      // last time there was outgoing packets sent to this peer (from wireguard-go)
+	lastFullPing   time.Time      // last time we pinged all endpoints
 	derpAddr       netaddr.IPPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients)
 
 	bestAddr           addrLatency // best non-DERP path; zero if none
-	bestAddrAt         mono.Time   // time best address re-confirmed
-	trustBestAddrUntil mono.Time   // time when bestAddr expires
+	bestAddrAt         time.Time   // time best address re-confirmed
+	trustBestAddrUntil time.Time   // time when bestAddr expires
 	sentPing           map[stun.TxID]sentPing
 	endpointState      map[netaddr.IPPort]*endpointState
 	isCallMeMaybeEP    map[netaddr.IPPort]bool
@@ -3219,7 +3210,7 @@ type endpointState struct {
 	// all fields guarded by discoEndpoint.mu
 
 	// lastPing is the last (outgoing) ping time.
-	lastPing mono.Time
+	lastPing time.Time
 
 	// lastGotPing, if non-zero, means that this was an endpoint
 	// that we learned about at runtime (from an incoming ping)
@@ -3267,14 +3258,14 @@ const pongHistoryCount = 64
 
 type pongReply struct {
 	latency time.Duration
-	pongAt  mono.Time      // when we received the pong
+	pongAt  time.Time      // when we received the pong
 	from    netaddr.IPPort // the pong's src (usually same as endpoint map key)
 	pongSrc netaddr.IPPort // what they reported they heard
 }
 
 type sentPing struct {
 	to      netaddr.IPPort
-	at      mono.Time
+	at      time.Time
 	timer   *time.Timer // timeout timer
 	purpose discoPingPurpose
 }
@@ -3294,10 +3285,10 @@ func (de *discoEndpoint) initFakeUDPAddr() {
 // endpoint and reports whether it's been at least 10 seconds since the last
 // receive activity (including having never received from this peer before).
 func (de *discoEndpoint) isFirstRecvActivityInAwhile() bool {
-	now := mono.Now()
-	elapsed := now.Sub(de.lastRecv.LoadAtomic())
-	if elapsed > 10*time.Second {
-		de.lastRecv.StoreAtomic(now)
+	now := time.Now().Unix()
+	old := atomic.LoadInt64(&de.lastRecvUnixAtomic)
+	if old <= now-10 {
+		atomic.StoreInt64(&de.lastRecvUnixAtomic, now)
 		return true
 	}
 	return false
@@ -3322,7 +3313,7 @@ func (de *discoEndpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr)
 // addr may be non-zero.
 //
 // de.mu must be held.
-func (de *discoEndpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netaddr.IPPort) {
+func (de *discoEndpoint) addrForSendLocked(now time.Time) (udpAddr, derpAddr netaddr.IPPort) {
 	udpAddr = de.bestAddr.IPPort
 	if udpAddr.IsZero() || now.After(de.trustBestAddrUntil) {
 		// We had a bestAddr but it expired so send both to it
@@ -3345,13 +3336,13 @@ func (de *discoEndpoint) heartbeat() {
 		return
 	}
 
-	if mono.Since(de.lastSend) > sessionActiveTimeout {
+	if time.Since(de.lastSend) > sessionActiveTimeout {
 		// Session's idle. Stop heartbeating.
 		de.c.logf("[v1] magicsock: disco: ending heartbeats for idle session to %v (%v)", de.publicKey.ShortString(), de.discoShort)
 		return
 	}
 
-	now := mono.Now()
+	now := time.Now()
 	udpAddr, _ := de.addrForSendLocked(now)
 	if !udpAddr.IsZero() {
 		// We have a preferred path. Ping that every 2 seconds.
@@ -3369,7 +3360,7 @@ func (de *discoEndpoint) heartbeat() {
 // a better path.
 //
 // de.mu must be held.
-func (de *discoEndpoint) wantFullPingLocked(now mono.Time) bool {
+func (de *discoEndpoint) wantFullPingLocked(now time.Time) bool {
 	if de.bestAddr.IsZero() || de.lastFullPing.IsZero() {
 		return true
 	}
@@ -3386,7 +3377,7 @@ func (de *discoEndpoint) wantFullPingLocked(now mono.Time) bool {
 }
 
 func (de *discoEndpoint) noteActiveLocked() {
-	de.lastSend = mono.Now()
+	de.lastSend = time.Now()
 	if de.heartBeatTimer == nil {
 		de.heartBeatTimer = time.AfterFunc(heartbeatInterval, de.heartbeat)
 	}
@@ -3400,7 +3391,7 @@ func (de *discoEndpoint) cliPing(res *ipnstate.PingResult, cb func(*ipnstate.Pin
 
 	de.pendingCLIPings = append(de.pendingCLIPings, pendingCLIPing{res, cb})
 
-	now := mono.Now()
+	now := time.Now()
 	udpAddr, derpAddr := de.addrForSendLocked(now)
 	if !derpAddr.IsZero() {
 		de.startPingLocked(derpAddr, now, pingCLI)
@@ -3420,7 +3411,7 @@ func (de *discoEndpoint) cliPing(res *ipnstate.PingResult, cb func(*ipnstate.Pin
 }
 
 func (de *discoEndpoint) send(b []byte) error {
-	now := mono.Now()
+	now := time.Now()
 
 	de.mu.Lock()
 	udpAddr, derpAddr := de.addrForSendLocked(now)
@@ -3453,7 +3444,7 @@ func (de *discoEndpoint) pingTimeout(txid stun.TxID) {
 	if !ok {
 		return
 	}
-	if debugDisco || de.bestAddr.IsZero() || mono.Now().After(de.trustBestAddrUntil) {
+	if debugDisco || de.bestAddr.IsZero() || time.Now().After(de.trustBestAddrUntil) {
 		de.c.logf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort)
 	}
 	de.removeSentPingLocked(txid, sp)
@@ -3490,7 +3481,7 @@ func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLev
 // discoPingPurpose is the reason why a discovery ping message was sent.
 type discoPingPurpose int
 
-//go:generate go run tailscale.com/cmd/addlicense -year 2020 -file discopingpurpose_string.go go run golang.org/x/tools/cmd/stringer -type=discoPingPurpose -trimprefix=ping
+//go:generate stringer -type=discoPingPurpose -trimprefix=ping
 const (
 	// pingDiscovery means that purpose of a ping was to see if a
 	// path was valid.
@@ -3505,7 +3496,7 @@ const (
 	pingCLI
 )
 
-func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now mono.Time, purpose discoPingPurpose) {
+func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time, purpose discoPingPurpose) {
 	if purpose != pingCLI {
 		st, ok := de.endpointState[ep]
 		if !ok {
@@ -3531,7 +3522,7 @@ func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now mono.Time, purpo
 	go de.sendDiscoPing(ep, txid, logLevel)
 }
 
-func (de *discoEndpoint) sendPingsLocked(now mono.Time, sendCallMeMaybe bool) {
+func (de *discoEndpoint) sendPingsLocked(now time.Time, sendCallMeMaybe bool) {
 	de.lastFullPing = now
 	var sentAny bool
 	for ep, st := range de.endpointState {
@@ -3653,7 +3644,7 @@ func (de *discoEndpoint) noteConnectivityChange() {
 	de.mu.Lock()
 	defer de.mu.Unlock()
 
-	de.trustBestAddrUntil = 0
+	de.trustBestAddrUntil = time.Time{}
 }
 
 // handlePongConnLocked handles a Pong message (a reply to an earlier ping).
@@ -3671,7 +3662,7 @@ func (de *discoEndpoint) handlePongConnLocked(m *disco.Pong, src netaddr.IPPort)
 	}
 	de.removeSentPingLocked(m.TxID, sp)
 
-	now := mono.Now()
+	now := time.Now()
 	latency := now.Sub(sp.at)
 
 	if !isDerp {
@@ -3823,9 +3814,9 @@ func (de *discoEndpoint) handleCallMeMaybe(m *disco.CallMeMaybe) {
 	// Zero out all the lastPing times to force sendPingsLocked to send new ones,
 	// even if it's been less than 5 seconds ago.
 	for _, st := range de.endpointState {
-		st.lastPing = 0
+		st.lastPing = time.Time{}
 	}
-	de.sendPingsLocked(mono.Now(), false)
+	de.sendPingsLocked(time.Now(), false)
 }
 
 func (de *discoEndpoint) populatePeerStatus(ps *ipnstate.PeerStatus) {
@@ -3838,7 +3829,7 @@ func (de *discoEndpoint) populatePeerStatus(ps *ipnstate.PeerStatus) {
 
 	ps.LastWrite = de.lastSend
 
-	now := mono.Now()
+	now := time.Now()
 	if udpAddr, derpAddr := de.addrForSendLocked(now); !udpAddr.IsZero() && derpAddr.IsZero() {
 		ps.CurAddr = udpAddr.String()
 	}
@@ -3856,13 +3847,13 @@ func (de *discoEndpoint) stopAndReset() {
 
 	// Zero these fields so if the user re-starts the network, the discovery
 	// state isn't a mix of before & after two sessions.
-	de.lastSend = 0
-	de.lastFullPing = 0
+	de.lastSend = time.Time{}
+	de.lastFullPing = time.Time{}
 	de.bestAddr = addrLatency{}
-	de.bestAddrAt = 0
-	de.trustBestAddrUntil = 0
+	de.bestAddrAt = time.Time{}
+	de.trustBestAddrUntil = time.Time{}
 	for _, es := range de.endpointState {
-		es.lastPing = 0
+		es.lastPing = time.Time{}
 	}
 
 	for txid, sp := range de.sentPing {

wgengine/magicsock/magicsock_test.go

@@ -38,7 +38,6 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/natlab"
-	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -1147,13 +1146,13 @@ func TestAddrSet(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			faket := mono.Time(0)
+			faket := time.Unix(0, 0)
 			var logBuf bytes.Buffer
 			tt.as.Logf = func(format string, args ...interface{}) {
 				fmt.Fprintf(&logBuf, format, args...)
 				t.Logf(format, args...)
 			}
-			tt.as.clock = func() mono.Time { return faket }
+			tt.as.clock = func() time.Time { return faket }
 			for i, st := range tt.steps {
 				faket = faket.Add(st.advance)
 
@@ -1230,8 +1229,8 @@ func TestDiscoStringLogRace(t *testing.T) {
 func Test32bitAlignment(t *testing.T) {
 	var de discoEndpoint
 
-	if off := unsafe.Offsetof(de.lastRecv); off%8 != 0 {
-		t.Fatalf("discoEndpoint.lastRecv is not 8-byte aligned")
+	if off := unsafe.Offsetof(de.lastRecvUnixAtomic); off%8 != 0 {
+		t.Fatalf("discoEndpoint.lastRecvUnixAtomic is not 8-byte aligned")
 	}
 
 	if !de.isFirstRecvActivityInAwhile() { // verify this doesn't panic on 32-bit

wgengine/monitor/monitor.go

@@ -62,7 +62,6 @@ type Mon struct {
 
 	mu         sync.Mutex // guards all following fields
 	cbs        map[*callbackHandle]ChangeFunc
-	ruleDelCB  map[*callbackHandle]RuleDeleteCallback
 	ifState    *interfaces.State
 	gwValid    bool       // whether gw and gwSelfIP are valid
 	gw         netaddr.IP // our gateway's IP
@@ -149,30 +148,6 @@ func (m *Mon) RegisterChangeCallback(callback ChangeFunc) (unregister func()) {
 	}
 }
 
-// RuleDeleteCallback is a callback when a Linux IP policy routing
-// rule is deleted. The table is the table number (52, 253, 354) and
-// priority is the priority order number (for Tailscale rules
-// currently: 5210, 5230, 5250, 5270)
-type RuleDeleteCallback func(table uint8, priority uint32)
-
-// RegisterRuleDeleteCallback adds callback to the set of parties to be
-// notified (in their own goroutine) when a Linux ip rule is deleted.
-// To remove this callback, call unregister (or close the monitor).
-func (m *Mon) RegisterRuleDeleteCallback(callback RuleDeleteCallback) (unregister func()) {
-	handle := new(callbackHandle)
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.ruleDelCB == nil {
-		m.ruleDelCB = map[*callbackHandle]RuleDeleteCallback{}
-	}
-	m.ruleDelCB[handle] = callback
-	return func() {
-		m.mu.Lock()
-		defer m.mu.Unlock()
-		delete(m.ruleDelCB, handle)
-	}
-}
-
 // Start starts the monitor.
 // A monitor can only be started & closed once.
 func (m *Mon) Start() {
@@ -267,10 +242,6 @@ func (m *Mon) pump() {
 			time.Sleep(time.Second)
 			continue
 		}
-		if rdm, ok := msg.(ipRuleDeletedMessage); ok {
-			m.notifyRuleDeleted(rdm)
-			continue
-		}
 		if msg.ignore() {
 			continue
 		}
@@ -278,14 +249,6 @@ func (m *Mon) pump() {
 	}
 }
 
-func (m *Mon) notifyRuleDeleted(rdm ipRuleDeletedMessage) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	for _, cb := range m.ruleDelCB {
-		go cb(rdm.table, rdm.priority)
-	}
-}
-
 // debounce calls the callback function with a delay between events
 // and exits when a stop is issued.
 func (m *Mon) debounce() {
@@ -375,10 +338,3 @@ func (m *Mon) checkWallTimeAdvanceLocked() {
 	}
 	m.lastWall = now
 }
-
-type ipRuleDeletedMessage struct {
-	table    uint8
-	priority uint32
-}
-
-func (ipRuleDeletedMessage) ignore() bool { return true }

wgengine/monitor/monitor_linux.go

@@ -42,9 +42,7 @@ func newOSMon(logf logger.Logf, m *Mon) (osMon, error) {
 		// address as well to cover things like DHCP deciding to give
 		// us a new address upon renewal - routing wouldn't change,
 		// but all reachability would.
-		Groups: unix.RTMGRP_IPV4_IFADDR | unix.RTMGRP_IPV6_IFADDR |
-			unix.RTMGRP_IPV4_ROUTE | unix.RTMGRP_IPV6_ROUTE |
-			unix.RTMGRP_IPV4_RULE, // no IPV6_RULE in x/sys/unix
+		Groups: unix.RTMGRP_IPV4_IFADDR | unix.RTMGRP_IPV6_IFADDR | unix.RTMGRP_IPV4_ROUTE | unix.RTMGRP_IPV6_ROUTE,
 	})
 	if err != nil {
 		// Google Cloud Run does not implement NETLINK_ROUTE RTMGRP support
@@ -119,25 +117,6 @@ func (c *nlConn) Receive() (message, error) {
 			Dst:     dst,
 			Gateway: gw,
 		}, nil
-	case unix.RTM_NEWRULE:
-		// Probably ourselves adding it.
-		return ignoreMessage{}, nil
-	case unix.RTM_DELRULE:
-		// For https://github.com/tailscale/tailscale/issues/1591 where
-		// systemd-networkd deletes our rules.
-		var rmsg rtnetlink.RouteMessage
-		err := rmsg.UnmarshalBinary(msg.Data)
-		if err != nil {
-			c.logf("ip rule deleted; failed to parse netlink message: %v", err)
-		} else {
-			c.logf("ip rule deleted: %+v", rmsg)
-			// On `ip -4 rule del pref 5210 table main`, logs:
-			// monitor: ip rule deleted: {Family:2 DstLength:0 SrcLength:0 Tos:0 Table:254 Protocol:0 Scope:0 Type:1 Flags:0 Attributes:{Dst:<nil> Src:<nil> Gateway:<nil> OutIface:0 Priority:5210 Table:254 Mark:4294967295 Expires:<nil> Metrics:<nil> Multipath:[]}}
-		}
-		return ipRuleDeletedMessage{
-			table:    rmsg.Table,
-			priority: rmsg.Attributes.Priority,
-		}, nil
 	default:
 		c.logf("unhandled netlink msg type %+v, %q", msg.Header, msg.Data)
 		return unspecifiedMessage{}, nil

wgengine/netstack/netstack.go

@@ -136,25 +136,8 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 	return ns, nil
 }
 
-// wrapProtoHandler returns protocol handler h wrapped in a version
-// that dynamically reconfigures ns's subnet addresses as needed for
-// outbound traffic.
-func (ns *Impl) wrapProtoHandler(h func(stack.TransportEndpointID, *stack.PacketBuffer) bool) func(stack.TransportEndpointID, *stack.PacketBuffer) bool {
-	return func(tei stack.TransportEndpointID, pb *stack.PacketBuffer) bool {
-		addr := tei.LocalAddress
-		ip, ok := netaddr.FromStdIP(net.IP(addr))
-		if !ok {
-			ns.logf("netstack: could not parse local address for incoming connection")
-			return false
-		}
-		if !ns.isLocalIP(ip) {
-			ns.addSubnetAddress(ip)
-		}
-		return h(tei, pb)
-	}
-}
-
 // Start sets up all the handlers so netstack can start working. Implements
+
 // wgengine.FakeImpl.
 func (ns *Impl) Start() error {
 	ns.e.AddNetworkMapCallback(ns.updateIPs)
@@ -163,8 +146,25 @@ func (ns *Impl) Start() error {
 	const maxInFlightConnectionAttempts = 16
 	tcpFwd := tcp.NewForwarder(ns.ipstack, tcpReceiveBufferSize, maxInFlightConnectionAttempts, ns.acceptTCP)
 	udpFwd := udp.NewForwarder(ns.ipstack, ns.acceptUDP)
-	ns.ipstack.SetTransportProtocolHandler(tcp.ProtocolNumber, ns.wrapProtoHandler(tcpFwd.HandlePacket))
-	ns.ipstack.SetTransportProtocolHandler(udp.ProtocolNumber, ns.wrapProtoHandler(udpFwd.HandlePacket))
+	ns.ipstack.SetTransportProtocolHandler(tcp.ProtocolNumber, func(tei stack.TransportEndpointID, pb *stack.PacketBuffer) bool {
+		addr := tei.LocalAddress
+		var pn tcpip.NetworkProtocolNumber
+		if addr.To4() != "" {
+			pn = ipv4.ProtocolNumber
+		} else {
+			pn = ipv6.ProtocolNumber
+		}
+		ip, ok := netaddr.FromStdIP(net.IP(addr))
+		if !ok {
+			ns.logf("netstack: could not parse local address %s for incoming TCP connection", ip)
+			return false
+		}
+		if !ns.isLocalIP(ip) {
+			ns.addSubnetAddress(pn, ip)
+		}
+		return tcpFwd.HandlePacket(tei, pb)
+	})
+	ns.ipstack.SetTransportProtocolHandler(udp.ProtocolNumber, udpFwd.HandlePacket)
 	go ns.injectOutbound()
 	ns.tundev.PostFilterIn = ns.injectInbound
 	return nil
@@ -215,19 +215,13 @@ func (ns *Impl) updateDNS(nm *netmap.NetworkMap) {
 	ns.dns = DNSMapFromNetworkMap(nm)
 }
 
-func (ns *Impl) addSubnetAddress(ip netaddr.IP) {
+func (ns *Impl) addSubnetAddress(pn tcpip.NetworkProtocolNumber, ip netaddr.IP) {
 	ns.mu.Lock()
 	ns.connsOpenBySubnetIP[ip]++
 	needAdd := ns.connsOpenBySubnetIP[ip] == 1
 	ns.mu.Unlock()
 	// Only register address into netstack for first concurrent connection.
 	if needAdd {
-		var pn tcpip.NetworkProtocolNumber
-		if ip.Is4() {
-			pn = ipv4.ProtocolNumber
-		} else if ip.Is6() {
-			pn = ipv6.ProtocolNumber
-		}
 		ns.ipstack.AddAddress(nicID, pn, tcpip.Address(ip.IPAddr().IP))
 	}
 }
@@ -550,9 +544,9 @@ func (ns *Impl) forwardTCP(client *gonet.TCPConn, wq *waiter.Queue, dialAddr tcp
 }
 
 func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {
-	sess := r.ID()
+	reqDetails := r.ID()
 	if debugNetstack {
-		ns.logf("[v2] UDP ForwarderRequest: %v", stringifyTEI(sess))
+		ns.logf("[v2] UDP ForwarderRequest: %v", stringifyTEI(reqDetails))
 	}
 	var wq waiter.Queue
 	ep, err := r.CreateEndpoint(&wq)
@@ -560,50 +554,30 @@ func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {
 		ns.logf("acceptUDP: could not create endpoint: %v", err)
 		return
 	}
-	dstAddr, ok := ipPortOfNetstackAddr(sess.LocalAddress, sess.LocalPort)
-	if !ok {
+	localAddr, err := ep.GetLocalAddress()
+	if err != nil {
 		return
 	}
-	srcAddr, ok := ipPortOfNetstackAddr(sess.RemoteAddress, sess.RemotePort)
-	if !ok {
+	remoteAddr, err := ep.GetRemoteAddress()
+	if err != nil {
 		return
 	}
-
 	c := gonet.NewUDPConn(ns.ipstack, &wq, ep)
-	go ns.forwardUDP(c, &wq, srcAddr, dstAddr)
+	go ns.forwardUDP(c, &wq, localAddr, remoteAddr)
 }
 
-// forwardUDP proxies between client (with addr clientAddr) and dstAddr.
-//
-// dstAddr may be either a local Tailscale IP, in which we case we proxy to
-// 127.0.0.1, or any other IP (from an advertised subnet), in which case we
-// proxy to it directly.
-func (ns *Impl) forwardUDP(client *gonet.UDPConn, wq *waiter.Queue, clientAddr, dstAddr netaddr.IPPort) {
-	port, srcPort := dstAddr.Port(), clientAddr.Port()
+func (ns *Impl) forwardUDP(client *gonet.UDPConn, wq *waiter.Queue, clientLocalAddr, clientRemoteAddr tcpip.FullAddress) {
+	port := clientLocalAddr.Port
 	ns.logf("[v2] netstack: forwarding incoming UDP connection on port %v", port)
-
-	var backendListenAddr *net.UDPAddr
-	var backendRemoteAddr *net.UDPAddr
-	isLocal := ns.isLocalIP(dstAddr.IP())
-	if isLocal {
-		backendRemoteAddr = &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(port)}
-		backendListenAddr = &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(srcPort)}
-	} else {
-		backendRemoteAddr = dstAddr.UDPAddr()
-		if dstAddr.IP().Is4() {
-			backendListenAddr = &net.UDPAddr{IP: net.ParseIP("0.0.0.0"), Port: int(srcPort)}
-		} else {
-			backendListenAddr = &net.UDPAddr{IP: net.ParseIP("::"), Port: int(srcPort)}
-		}
-	}
-
-	backendConn, err := net.ListenUDP("udp", backendListenAddr)
+	backendListenAddr := &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(clientRemoteAddr.Port)}
+	backendRemoteAddr := &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(port)}
+	backendConn, err := net.ListenUDP("udp4", backendListenAddr)
 	if err != nil {
-		ns.logf("netstack: could not bind local port %v: %v, trying again with random port", backendListenAddr.Port, err)
+		ns.logf("netstack: could not bind local port %v: %v, trying again with random port", clientRemoteAddr.Port, err)
 		backendListenAddr.Port = 0
-		backendConn, err = net.ListenUDP("udp", backendListenAddr)
+		backendConn, err = net.ListenUDP("udp4", backendListenAddr)
 		if err != nil {
-			ns.logf("netstack: could not create UDP socket, preventing forwarding to %v: %v", dstAddr, err)
+			ns.logf("netstack: could not connect to local UDP server on port %v: %v", port, err)
 			return
 		}
 	}
@@ -612,47 +586,28 @@ func (ns *Impl) forwardUDP(client *gonet.UDPConn, wq *waiter.Queue, clientAddr,
 	if !ok {
 		ns.logf("could not get backend local IP:port from %v:%v", backendLocalAddr.IP, backendLocalAddr.Port)
 	}
-	if isLocal {
-		ns.e.RegisterIPPortIdentity(backendLocalIPPort, dstAddr.IP())
-	}
+	clientRemoteIP, _ := netaddr.FromStdIP(net.ParseIP(clientRemoteAddr.Addr.String()))
+	ns.e.RegisterIPPortIdentity(backendLocalIPPort, clientRemoteIP)
 	ctx, cancel := context.WithCancel(context.Background())
-
-	idleTimeout := 2 * time.Minute
-	if port == 53 {
-		// Make DNS packet copies time out much sooner.
-		//
-		// TODO(bradfitz): make DNS queries over UDP forwarding even
-		// cheaper by adding an additional idleTimeout post-DNS-reply.
-		// For instance, after the DNS response goes back out, then only
-		// wait a few seconds (or zero, really)
-		idleTimeout = 30 * time.Second
-	}
-	timer := time.AfterFunc(idleTimeout, func() {
-		if isLocal {
-			ns.e.UnregisterIPPortIdentity(backendLocalIPPort)
-		}
-		ns.logf("netstack: UDP session between %s and %s timed out", backendListenAddr, backendRemoteAddr)
+	timer := time.AfterFunc(2*time.Minute, func() {
+		ns.e.UnregisterIPPortIdentity(backendLocalIPPort)
+		ns.logf("netstack: UDP session between %s and %s timed out", clientRemoteAddr, backendRemoteAddr)
 		cancel()
 		client.Close()
 		backendConn.Close()
 	})
 	extend := func() {
-		timer.Reset(idleTimeout)
+		timer.Reset(2 * time.Minute)
 	}
-	startPacketCopy(ctx, cancel, client, clientAddr.UDPAddr(), backendConn, ns.logf, extend)
+	startPacketCopy(ctx, cancel, client, &net.UDPAddr{
+		IP:   net.ParseIP(clientRemoteAddr.Addr.String()),
+		Port: int(clientRemoteAddr.Port),
+	}, backendConn, ns.logf, extend)
 	startPacketCopy(ctx, cancel, backendConn, backendRemoteAddr, client, ns.logf, extend)
-	if isLocal {
-		// Wait for the copies to be done before decrementing the
-		// subnet address count to potentially remove the route.
-		<-ctx.Done()
-		ns.removeSubnetAddress(dstAddr.IP())
-	}
+
 }
 
 func startPacketCopy(ctx context.Context, cancel context.CancelFunc, dst net.PacketConn, dstAddr net.Addr, src net.PacketConn, logf logger.Logf, extend func()) {
-	if debugNetstack {
-		logf("[v2] netstack: startPacketCopy to %v (%T) from %T", dstAddr, dst, src)
-	}
 	go func() {
 		defer cancel() // tear down the other direction's copy
 		pkt := make([]byte, mtu)
@@ -689,7 +644,3 @@ func stringifyTEI(tei stack.TransportEndpointID) string {
 	remoteHostPort := net.JoinHostPort(tei.RemoteAddress.String(), strconv.Itoa(int(tei.RemotePort)))
 	return fmt.Sprintf("%s -> %s", remoteHostPort, localHostPort)
 }
-
-func ipPortOfNetstackAddr(a tcpip.Address, port uint16) (ipp netaddr.IPPort, ok bool) {
-	return netaddr.FromStdAddr(net.IP(a), int(port), "") // TODO(bradfitz): can do without allocs
-}

wgengine/pendopen.go

@@ -231,7 +231,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	e.logf("open-conn-track: timeout opening %v to node %v; online=%v, lastRecv=%v",
 		flow, n.Key.ShortString(),
 		online,
-		e.magicConn.LastRecvActivityOfDisco(n.DiscoKey))
+		durFmt(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
 }
 
 func durFmt(t time.Time) string {

wgengine/router/router.go

@@ -11,7 +11,6 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
-	"tailscale.com/wgengine/monitor"
 )
 
 // Router is responsible for managing the system network stack.
@@ -32,12 +31,9 @@ type Router interface {
 
 // New returns a new Router for the current platform, using the
 // provided tun device.
-//
-// If linkMon is nil, it's not used. It's currently (2021-07-20) only
-// used on Linux in some situations.
-func New(logf logger.Logf, tundev tun.Device, linkMon *monitor.Mon) (Router, error) {
+func New(logf logger.Logf, tundev tun.Device) (Router, error) {
 	logf = logger.WithPrefix(logf, "router: ")
-	return newUserspaceRouter(logf, tundev, linkMon)
+	return newUserspaceRouter(logf, tundev)
 }
 
 // Cleanup restores the system network configuration to its original state