cmd/allsrc: WIP tool to print out all a program's source

Change-Id: Ie8ed3ad744af5b5b7772cb9b4516a9b8e2f2866d
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/linux.yml

@@ -28,6 +28,15 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
+    - name: Get QEMU
+      run: |
+        # The qemu in Ubuntu 20.04 (Focal) is too old; we need 5.x something
+        # to run Go binaries. 5.2.0 (Debian bullseye) empirically works, and
+        # use this PPA which brings in a modern qemu.
+        sudo add-apt-repository -y ppa:jacob/virtualisation
+        sudo apt-get -y update
+        sudo apt-get -y install qemu-user
+
     - name: Run tests on linux
       run: go test -bench=. -benchtime=1x ./...
 

Dockerfile

@@ -48,8 +48,9 @@ ARG VERSION_SHORT=""
 ENV VERSION_SHORT=$VERSION_SHORT
 ARG VERSION_GIT_HASH=""
 ENV VERSION_GIT_HASH=$VERSION_GIT_HASH
+ARG TARGETARCH
 
-RUN go install -tags=xversion -ldflags="\
+RUN GOARCH=$TARGETARCH go install -tags=xversion -ldflags="\
       -X tailscale.com/version.Long=$VERSION_LONG \
       -X tailscale.com/version.Short=$VERSION_SHORT \
       -X tailscale.com/version.GitCommit=$VERSION_GIT_HASH" \

Makefile

@@ -1,3 +1,5 @@
+IMAGE_REPO ?= tailscale/tailscale
+
 usage:
 	echo "See Makefile"
 
@@ -21,6 +23,10 @@ build386:
 buildlinuxarm:
 	GOOS=linux GOARCH=arm go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled
 
+
+buildmultiarchimage:
+	docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t ${IMAGE_REPO}:latest --push -f Dockerfile .
+
 check: staticcheck vet depaware buildwindows build386 buildlinuxarm
 
 staticcheck:

client/tailscale/tailscale.go

@@ -56,7 +56,7 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 			return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(TailscaledSocket, 41112)
+	return safesocket.Connect(TailscaledSocket, safesocket.WindowsLocalPort)
 }
 
 var (
@@ -196,6 +196,12 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return get200(ctx, "/localapi/v0/goroutines")
 }
 
+// DaemonMetrics returns the Tailscale daemon's metrics in
+// the Prometheus text exposition format.
+func DaemonMetrics(ctx context.Context) ([]byte, error) {
+	return get200(ctx, "/localapi/v0/metrics")
+}
+
 // Profile returns a pprof profile of the Tailscale daemon.
 func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string

cmd/allsrc/allsrc.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+
+	"golang.org/x/tools/go/packages"
+)
+
+var cfg = &packages.Config{
+	Mode: (0 |
+		packages.NeedName |
+		packages.NeedFiles |
+		packages.NeedCompiledGoFiles |
+		packages.NeedImports |
+		packages.NeedDeps |
+		packages.NeedModule |
+		packages.NeedTypes |
+		packages.NeedSyntax |
+		0),
+}
+
+func main() {
+	flag.Parse()
+
+	var w walker
+	w.walk("tailscale.com/cmd/tailscaled")
+}
+
+type walker struct {
+	done map[string]bool
+}
+
+func (w *walker) walk(mainPkg string) {
+	pkgs, err := packages.Load(cfg, mainPkg)
+	if err != nil {
+		log.Fatalf("packages.Load: %v", err)
+	}
+	for _, pkg := range pkgs {
+		w.walkPackage(pkg)
+	}
+}
+
+func (w *walker) walkPackage(pkg *packages.Package) {
+	if w.done[pkg.PkgPath] {
+		return
+	}
+	if w.done == nil {
+		w.done = map[string]bool{}
+	}
+	w.done[pkg.PkgPath] = true
+
+	fmt.Printf("\n### PACKAGE %v\n", pkg.PkgPath)
+
+	if len(pkg.Errors) > 0 {
+		log.Fatalf("errors reading %q: %q", pkg.PkgPath, pkg.Errors)
+	}
+
+	var imports []*packages.Package
+	for _, p := range pkg.Imports {
+		imports = append(imports, p)
+	}
+	sort.Slice(imports, func(i, j int) bool {
+		return imports[i].PkgPath < imports[j].PkgPath
+	})
+	for _, f := range pkg.GoFiles {
+		fmt.Printf("file.go %q\n", f)
+	}
+	for _, f := range pkg.OtherFiles {
+		fmt.Printf("file.other %q\n", f)
+	}
+	for _, p := range imports {
+		fmt.Printf("import %q => %q\n", pkg.PkgPath, p.PkgPath)
+	}
+	fmt.Printf("Fset: %p\n", pkg.Fset)
+	fmt.Printf("Syntax: %v\n", len(pkg.Syntax))
+	fmt.Printf("Modules: %+v\n", pkg.Module)
+
+	for _, p := range imports {
+		w.walkPackage(p)
+	}
+}

cmd/derper/derper.go

@@ -141,7 +141,7 @@ func main() {
 
 	cfg := loadConfig()
 
-	serveTLS := tsweb.IsProd443(*addr)
+	serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual"
 
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)

cmd/tailscale/cli/cli.go

@@ -191,7 +191,7 @@ var rootArgs struct {
 var gotSignal syncs.AtomicBool
 
 func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	c, err := safesocket.Connect(rootArgs.socket, safesocket.WindowsLocalPort)
 	if err != nil {
 		if runtime.GOOS != "windows" && rootArgs.socket == "" {
 			fatalf("--socket cannot be empty")

cmd/tailscale/cli/debug.go

@@ -5,6 +5,8 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -14,7 +16,9 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -24,39 +28,76 @@ import (
 )
 
 var debugCmd = &ffcli.Command{
-	Name: "debug",
-	Exec: runDebug,
+	Name:     "debug",
+	Exec:     runDebug,
+	LongHelp: `"tailscale debug" contains misc debug facilities; it is not a stable interface.`,
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("debug")
-		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
-		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
-		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
-		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
-		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
-		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
-		fs.BoolVar(&debugArgs.env, "env", false, "dump environment")
-		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file; - for stdout")
 		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout")
 		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs
 	})(),
+	Subcommands: []*ffcli.Command{
+		{
+			Name:      "derp-map",
+			Exec:      runDERPMap,
+			ShortHelp: "print DERP map",
+		},
+		{
+			Name:      "daemon-goroutines",
+			Exec:      runDaemonGoroutines,
+			ShortHelp: "print tailscaled's goroutines",
+		},
+		{
+			Name:      "metrics",
+			Exec:      runDaemonMetrics,
+			ShortHelp: "print tailscaled's metrics",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("metrics")
+				fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values")
+				return fs
+			})(),
+		},
+		{
+			Name:      "env",
+			Exec:      runEnv,
+			ShortHelp: "print cmd/tailscale environment",
+		},
+		{
+			Name:      "local-creds",
+			Exec:      runLocalCreds,
+			ShortHelp: "print how to access Tailscale local API",
+		},
+		{
+			Name:      "prefs",
+			Exec:      runPrefs,
+			ShortHelp: "print prefs",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("prefs")
+				fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output")
+				return fs
+			})(),
+		},
+		{
+			Name:      "watch-ipn",
+			Exec:      runWatchIPN,
+			ShortHelp: "subscribe to IPN message bus",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("watch-ipn")
+				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
+				return fs
+			})(),
+		},
+	},
 }
 
 var debugArgs struct {
-	env        bool
-	localCreds bool
-	goroutines bool
-	ipn        bool
-	netMap     bool
-	derpMap    bool
-	file       string
-	prefs      bool
-	pretty     bool
-	cpuSec     int
-	cpuFile    string
-	memFile    string
+	file    string
+	cpuSec  int
+	cpuFile string
+	memFile string
 }
 
 func writeProfile(dst string, v []byte) error {
@@ -81,26 +122,9 @@ func runDebug(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("unknown arguments")
 	}
-	if debugArgs.env {
-		for _, e := range os.Environ() {
-			outln(e)
-		}
-		return nil
-	}
-	if debugArgs.localCreds {
-		port, token, err := safesocket.LocalTCPPortAndToken()
-		if err == nil {
-			printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
-			return nil
-		}
-		if runtime.GOOS == "windows" {
-			printf("curl http://localhost:41112/localapi/v0/status\n")
-			return nil
-		}
-		printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
-		return nil
-	}
+	var usedFlag bool
 	if out := debugArgs.cpuFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec)
 		if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil {
 			return err
@@ -112,6 +136,7 @@ func runDebug(ctx context.Context, args []string) error {
 		}
 	}
 	if out := debugArgs.memFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing memory profile ...")
 		if v, err := tailscale.Profile(ctx, "heap", 0); err != nil {
 			return err
@@ -122,55 +147,8 @@ func runDebug(ctx context.Context, args []string) error {
 			log.Printf("Memory profile written to %s", outName(out))
 		}
 	}
-	if debugArgs.prefs {
-		prefs, err := tailscale.GetPrefs(ctx)
-		if err != nil {
-			return err
-		}
-		if debugArgs.pretty {
-			outln(prefs.Pretty())
-		} else {
-			j, _ := json.MarshalIndent(prefs, "", "\t")
-			outln(string(j))
-		}
-		return nil
-	}
-	if debugArgs.goroutines {
-		goroutines, err := tailscale.Goroutines(ctx)
-		if err != nil {
-			return err
-		}
-		Stdout.Write(goroutines)
-		return nil
-	}
-	if debugArgs.derpMap {
-		dm, err := tailscale.CurrentDERPMap(ctx)
-		if err != nil {
-			return fmt.Errorf(
-				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
-			)
-		}
-		enc := json.NewEncoder(Stdout)
-		enc.SetIndent("", "\t")
-		enc.Encode(dm)
-		return nil
-	}
-	if debugArgs.ipn {
-		c, bc, ctx, cancel := connect(ctx)
-		defer cancel()
-
-		bc.SetNotifyCallback(func(n ipn.Notify) {
-			if !debugArgs.netMap {
-				n.NetMap = nil
-			}
-			j, _ := json.MarshalIndent(n, "", "\t")
-			printf("%s\n", j)
-		})
-		bc.RequestEngineStatus()
-		pump(ctx, bc, c)
-		return errors.New("exit")
-	}
 	if debugArgs.file != "" {
+		usedFlag = true // TODO(bradfitz): add "file" subcommand
 		if debugArgs.file == "get" {
 			wfs, err := tailscale.WaitingFiles(ctx)
 			if err != nil {
@@ -193,5 +171,148 @@ func runDebug(ctx context.Context, args []string) error {
 		io.Copy(Stdout, rc)
 		return nil
 	}
+	if usedFlag {
+		// TODO(bradfitz): delete this path when all debug flags are migrated
+		// to subcommands.
+		return nil
+	}
+	return errors.New("see 'tailscale debug --help")
+}
+
+func runLocalCreds(ctx context.Context, args []string) error {
+	port, token, err := safesocket.LocalTCPPortAndToken()
+	if err == nil {
+		printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
+		return nil
+	}
+	if runtime.GOOS == "windows" {
+		printf("curl http://localhost:%v/localapi/v0/status\n", safesocket.WindowsLocalPort)
+		return nil
+	}
+	printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
 	return nil
 }
+
+var prefsArgs struct {
+	pretty bool
+}
+
+func runPrefs(ctx context.Context, args []string) error {
+	prefs, err := tailscale.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+	if prefsArgs.pretty {
+		outln(prefs.Pretty())
+	} else {
+		j, _ := json.MarshalIndent(prefs, "", "\t")
+		outln(string(j))
+	}
+	return nil
+}
+
+var watchIPNArgs struct {
+	netmap bool
+}
+
+func runWatchIPN(ctx context.Context, args []string) error {
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if !watchIPNArgs.netmap {
+			n.NetMap = nil
+		}
+		j, _ := json.MarshalIndent(n, "", "\t")
+		printf("%s\n", j)
+	})
+	bc.RequestEngineStatus()
+	pump(ctx, bc, c)
+	return errors.New("exit")
+}
+
+func runDERPMap(ctx context.Context, args []string) error {
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+		)
+	}
+	enc := json.NewEncoder(Stdout)
+	enc.SetIndent("", "\t")
+	enc.Encode(dm)
+	return nil
+}
+
+func runEnv(ctx context.Context, args []string) error {
+	for _, e := range os.Environ() {
+		outln(e)
+	}
+	return nil
+}
+
+func runDaemonGoroutines(ctx context.Context, args []string) error {
+	goroutines, err := tailscale.Goroutines(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(goroutines)
+	return nil
+}
+
+var metricsArgs struct {
+	watch bool
+}
+
+func runDaemonMetrics(ctx context.Context, args []string) error {
+	last := map[string]int64{}
+	for {
+		out, err := tailscale.DaemonMetrics(ctx)
+		if err != nil {
+			return err
+		}
+		if !metricsArgs.watch {
+			Stdout.Write(out)
+			return nil
+		}
+		bs := bufio.NewScanner(bytes.NewReader(out))
+		type change struct {
+			name     string
+			from, to int64
+		}
+		var changes []change
+		var maxNameLen int
+		for bs.Scan() {
+			line := bytes.TrimSpace(bs.Bytes())
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			f := strings.Fields(string(line))
+			if len(f) != 2 {
+				continue
+			}
+			name := f[0]
+			n, _ := strconv.ParseInt(f[1], 10, 64)
+			prev, ok := last[name]
+			if ok && prev == n {
+				continue
+			}
+			last[name] = n
+			if !ok {
+				continue
+			}
+			changes = append(changes, change{name, prev, n})
+			if len(name) > maxNameLen {
+				maxNameLen = len(name)
+			}
+		}
+		if len(changes) > 0 {
+			format := fmt.Sprintf("%%-%ds %%+5d => %%v\n", maxNameLen)
+			for _, c := range changes {
+				fmt.Fprintf(Stdout, format, c.name, c.to-c.from, c.to)
+			}
+			io.WriteString(Stdout, "\n")
+		}
+		time.Sleep(time.Second)
+	}
+}

cmd/tailscale/depaware.txt

@@ -72,6 +72,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/structs                                  from tailscale.com/ipn+
+        tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli

cmd/tailscaled/depaware.txt

@@ -59,7 +59,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-        github.com/go-multierror/multierror                          from tailscale.com/cmd/tailscaled+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
@@ -92,27 +91,27 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/wgengine/router
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink                               from tailscale.com/wgengine/router
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/vishvananda/netlink
-   L    github.com/vishvananda/netns                                 from github.com/vishvananda/netlink+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+   W 💣 golang.zx2c4.com/wintun                                      from golang.zx2c4.com/wireguard/tun
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
      💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
      💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
-   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+   W 💣 golang.zx2c4.com/wireguard/ipc/namedpipe                     from golang.zx2c4.com/wireguard/ipc
         golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
         golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device
      💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
-   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/cmd/tailscaled+
         inet.af/netaddr                                              from inet.af/wf+
         inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
@@ -176,14 +175,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/log/filelogger                                 from tailscale.com/ipn/ipnserver
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled
-        tailscale.com/logtail                                        from tailscale.com/logpolicy
+        tailscale.com/logtail                                        from tailscale.com/logpolicy+
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
      💣 tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/net/dns/resolver                               from tailscale.com/net/dns+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
-        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
      💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock
@@ -222,12 +221,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/clientmetric                              from tailscale.com/ipn/localapi+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/dns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy

cmd/tailscaled/tailscaled.go

@@ -20,6 +20,7 @@ import (
 	"net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
@@ -27,17 +28,20 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/socks5/tssocks"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -78,6 +82,7 @@ var args struct {
 	debug          string
 	port           uint16
 	statepath      string
+	statedir       string
 	socketpath     string
 	birdSocketPath string
 	verbose        int
@@ -114,7 +119,8 @@ func main() {
 	flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM")
+	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "absolute path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM. If empty and --statedir is provided, the default is <statedir>/tailscaled.state")
+	flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -202,6 +208,16 @@ func trySynologyMigration(p string) error {
 	return nil
 }
 
+func statePathOrDefault() string {
+	if args.statepath != "" {
+		return args.statepath
+	}
+	if args.statedir != "" {
+		return filepath.Join(args.statedir, "tailscaled.state")
+	}
+	return ""
+}
+
 func ipnServerOpts() (o ipnserver.Options) {
 	// Allow changing the OS-specific IPN behavior for tests
 	// so we can e.g. test Windows-specific behaviors on Linux.
@@ -210,9 +226,15 @@ func ipnServerOpts() (o ipnserver.Options) {
 		goos = runtime.GOOS
 	}
 
-	o.Port = 41112
-	o.StatePath = args.statepath
-	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+	o.VarRoot = args.statedir
+
+	// If an absolute --state is provided but not --statedir, try to derive
+	// a state directory.
+	if o.VarRoot == "" && filepath.IsAbs(args.statepath) {
+		if dir := filepath.Dir(args.statepath); strings.EqualFold(filepath.Base(dir), "tailscale") {
+			o.VarRoot = dir
+		}
+	}
 
 	switch goos {
 	default:
@@ -227,7 +249,7 @@ func ipnServerOpts() (o ipnserver.Options) {
 func run() error {
 	var err error
 
-	pol := logpolicy.New("tailnode.log.tailscale.io")
+	pol := logpolicy.New(logtail.CollectionNode)
 	pol.SetVerbosityLevel(args.verbose)
 	defer func() {
 		// Finish uploading logs after closing everything else.
@@ -261,10 +283,10 @@ func run() error {
 		return nil
 	}
 
-	if args.statepath == "" {
-		log.Fatalf("--state is required")
+	if args.statepath == "" && args.statedir == "" {
+		log.Fatalf("--statedir (or at least --state) is required")
 	}
-	if err := trySynologyMigration(args.statepath); err != nil {
+	if err := trySynologyMigration(statePathOrDefault()); err != nil {
 		log.Printf("error in synology migration: %v", err)
 	}
 
@@ -289,10 +311,14 @@ func run() error {
 		return err
 	}
 
-	var ns *netstack.Impl
-	if useNetstack || wrapNetstack {
-		onlySubnets := wrapNetstack && !useNetstack
-		ns = mustStartNetstack(logf, e, onlySubnets)
+	ns, err := newNetstack(logf, e)
+	if err != nil {
+		return fmt.Errorf("newNetstack: %w", err)
+	}
+	ns.ProcessLocalIPs = useNetstack
+	ns.ProcessSubnets = useNetstack || wrapNetstack
+	if err := ns.Start(); err != nil {
+		log.Fatalf("failed to start netstack: %v", err)
 	}
 
 	if socksListener != nil || httpProxyListener != nil {
@@ -332,8 +358,27 @@ func run() error {
 	}()
 
 	opts := ipnServerOpts()
-	opts.DebugMux = debugMux
-	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+	srv, err := ipnserver.New(logf, pol.PublicID.String(), store, e, nil, opts)
+	if err != nil {
+		logf("ipnserver.New: %v", err)
+		return err
+	}
+
+	if debugMux != nil {
+		debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus)
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = srv.Run(ctx, ln)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
 		logf("ipnserver.Run: %v", err)
@@ -357,7 +402,7 @@ func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, us
 		logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err)
 		errs = append(errs, err)
 	}
-	return nil, false, multierror.New(errs)
+	return nil, false, multierr.New(errs...)
 }
 
 var wrapNetstack = shouldWrapNetstack()
@@ -435,6 +480,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 
 func newDebugMux() *http.ServeMux {
 	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/metrics", servePrometheusMetrics)
 	mux.HandleFunc("/debug/pprof/", pprof.Index)
 	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
@@ -443,6 +489,11 @@ func newDebugMux() *http.ServeMux {
 	return mux
 }
 
+func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 func runDebugServer(mux *http.ServeMux, addr string) {
 	srv := &http.Server{
 		Addr:    addr,
@@ -453,19 +504,12 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 	}
 }
 
-func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) *netstack.Impl {
+func newNetstack(logf logger.Logf, e wgengine.Engine) (*netstack.Impl, error) {
 	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
 	if !ok {
-		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+		return nil, fmt.Errorf("%T is not a wgengine.InternalsGetter", e)
 	}
-	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
-	if err != nil {
-		log.Fatalf("netstack.Create: %v", err)
-	}
-	if err := ns.Start(); err != nil {
-		log.Fatalf("failed to start netstack: %v", err)
-	}
-	return ns
+	return netstack.Create(logf, tunDev, e, magicConn)
 }
 
 func mustStartTCPListener(name, addr string) net.Listener {

cmd/tailscaled/tailscaled_windows.go

@@ -33,6 +33,7 @@ import (
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
@@ -74,7 +75,11 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	go func() {
 		defer close(doneCh)
 		args := []string{"/subproc", service.Policy.PublicID.String()}
-		ipnserver.BabysitProc(ctx, args, log.Printf)
+		// Make a logger without a date prefix, as filelogger
+		// and logtail both already add their own. All we really want
+		// from the log package is the automatic newline.
+		logger := log.New(os.Stderr, "", 0)
+		ipnserver.BabysitProc(ctx, args, logger.Printf)
 	}()
 
 	changes <- svc.Status{State: svc.Running, Accepts: svcAccepts}
@@ -202,9 +207,14 @@ func startIPNServer(ctx context.Context, logid string) error {
 			dev.Close()
 			return nil, fmt.Errorf("engine: %w", err)
 		}
-		onlySubnets := true
-		if wrapNetstack {
-			mustStartNetstack(logf, eng, onlySubnets)
+		ns, err := newNetstack(logf, eng)
+		if err != nil {
+			return nil, fmt.Errorf("newNetstack: %w", err)
+		}
+		ns.ProcessLocalIPs = false
+		ns.ProcessSubnets = wrapNetstack
+		if err := ns.Start(); err != nil {
+			return nil, fmt.Errorf("failed to start netstack: %w", err)
 		}
 		return wgengine.NewWatchdog(eng), nil
 	}
@@ -266,7 +276,18 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = ipnserver.Run(ctx, logf, ln, store, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

control/controlclient/client.go

@@ -79,3 +79,9 @@ type Client interface {
 	// requesting a DNS record be created or updated.
 	SetDNS(context.Context, *tailcfg.SetDNSRequest) error
 }
+
+// UserVisibleError is an error that should be shown to users.
+type UserVisibleError string
+
+func (e UserVisibleError) Error() string            { return string(e) }
+func (e UserVisibleError) UserVisibleError() string { return string(e) }

control/controlclient/direct.go

@@ -46,6 +46,7 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/systemd"
 	"tailscale.com/wgengine/monitor"
 )
@@ -60,7 +61,7 @@ type Direct struct {
 	keepAlive              bool
 	logf                   logger.Logf
 	linkMon                *monitor.Mon // or nil
-	discoPubKey            tailcfg.DiscoKey
+	discoPubKey            key.DiscoPublic
 	getMachinePrivKey      func() (key.MachinePrivate, error)
 	debugFlags             []string
 	keepSharerAndUserSplit bool
@@ -88,7 +89,7 @@ type Options struct {
 	AuthKey              string                             // optional node auth key for auto registration
 	TimeNow              func() time.Time                   // time.Now implementation used by Client
 	Hostinfo             *tailcfg.Hostinfo                  // non-nil passes ownership, nil means to use default using os.Hostname, etc
-	DiscoPublicKey       tailcfg.DiscoKey
+	DiscoPublicKey       key.DiscoPublic
 	NewDecompressor      func() (Decompressor, error)
 	KeepAlive            bool
 	Logf                 logger.Logf
@@ -146,6 +147,13 @@ func NewDirect(opts Options) (*Direct, error) {
 	}
 
 	httpc := opts.HTTPTestClient
+	if httpc == nil && runtime.GOOS == "js" {
+		// In js/wasm, net/http.Transport (as of Go 1.18) will
+		// only use the browser's Fetch API if you're using
+		// the DefaultClient (or a client without dial hooks
+		// etc set).
+		httpc = http.DefaultClient
+	}
 	if httpc == nil {
 		dnsCache := &dnscache.Resolver{
 			Forward:          dnscache.Get().Forward, // use default cache's forwarder
@@ -284,8 +292,8 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -357,9 +365,9 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	now := time.Now().Round(time.Second)
 	request := tailcfg.RegisterRequest{
 		Version:    1,
-		OldNodeKey: tailcfg.NodeKeyFromNodePublic(oldNodeKey),
-		NodeKey:    tailcfg.NodeKeyFromNodePublic(tryingNewKey.Public()),
-		Hostinfo:   hostinfo,
+		OldNodeKey: oldNodeKey,
+		NodeKey:    tryingNewKey.Public(),
+		Hostinfo:   hi,
 		Followup:   opt.URL,
 		Timestamp:  &now,
 		Ephemeral:  (opt.Flags & LoginEphemeral) != 0,
@@ -431,7 +439,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
 
 	if resp.Error != "" {
-		return false, "", errors.New(resp.Error)
+		return false, "", UserVisibleError(resp.Error)
 	}
 	if resp.NodeKeyExpired {
 		if regen {
@@ -551,12 +559,21 @@ const pollTimeout = 120 * time.Second
 
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	metricMapRequests.Add(1)
+	metricMapRequestsActive.Add(1)
+	defer metricMapRequestsActive.Add(-1)
+	if maxPolls == -1 {
+		metricMapRequestsPoll.Add(1)
+	} else {
+		metricMapRequestsLite.Add(1)
+	}
+
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	localPort := c.localPort
 	var epStrs []string
 	var epTypes []tailcfg.EndpointType
@@ -595,18 +612,18 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	request := &tailcfg.MapRequest{
 		Version:       tailcfg.CurrentMapRequestVersion,
 		KeepAlive:     c.keepAlive,
-		NodeKey:       tailcfg.NodeKeyFromNodePublic(persist.PrivateNodeKey.Public()),
+		NodeKey:       persist.PrivateNodeKey.Public(),
 		DiscoKey:      c.discoPubKey,
 		Endpoints:     epStrs,
 		EndpointTypes: epTypes,
 		Stream:        allowStream,
-		Hostinfo:      hostinfo,
+		Hostinfo:      hi,
 		DebugFlags:    c.debugFlags,
 		OmitPeers:     cb == nil,
 	}
 	var extraDebugFlags []string
-	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
-		ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
+	if hi != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
+		ipForwardingBroken(hi.RoutableIPs, c.linkMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
 	if health.RouterHealth() != nil {
@@ -615,6 +632,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	if health.NetworkCategoryHealth() != nil {
 		extraDebugFlags = append(extraDebugFlags, "warn-network-category-unhealthy")
 	}
+	if hostinfo.DisabledEtcAptSource() {
+		extraDebugFlags = append(extraDebugFlags, "warn-etc-apt-source-disabled")
+	}
 	if len(extraDebugFlags) > 0 {
 		old := request.DebugFlags
 		request.DebugFlags = append(old[:len(old):len(old)], extraDebugFlags...)
@@ -737,11 +757,14 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return err
 		}
 
+		metricMapResponseMessages.Add(1)
+
 		if allowStream {
 			health.GotStreamedMapResponse()
 		}
 
 		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+			metricMapResponsePings.Add(1)
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -758,13 +781,23 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return ctx.Err()
 		}
 		if resp.KeepAlive {
+			metricMapResponseKeepAlives.Add(1)
 			continue
 		}
 
+		metricMapResponseMap.Add(1)
+		if i > 0 {
+			metricMapResponseMapDelta.Add(1)
+		}
+
 		hasDebug := resp.Debug != nil
 		// being conservative here, if Debug not present set to False
 		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
 		if hasDebug {
+			if code := resp.Debug.Exit; code != nil {
+				c.logf("exiting process with status %v per controlplane", *code)
+				os.Exit(*code)
+			}
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}
@@ -1167,7 +1200,13 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 
 // SetDNS sends the SetDNSRequest request to the control plane server,
 // requesting a DNS record be created or updated.
-func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) (err error) {
+	metricSetDNS.Add(1)
+	defer func() {
+		if err != nil {
+			metricSetDNSError.Add(1)
+		}
+	}()
 	c.mu.Lock()
 	serverKey := c.serverKey
 	c.mu.Unlock()
@@ -1267,3 +1306,20 @@ func postPingResult(now time.Time, logf logger.Logf, c *http.Client, pr *tailcfg
 	}
 	return nil
 }
+
+var (
+	metricMapRequestsActive = clientmetric.NewGauge("controlclient_map_requests_active")
+
+	metricMapRequests     = clientmetric.NewCounter("controlclient_map_requests")
+	metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite")
+	metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll")
+
+	metricMapResponseMessages   = clientmetric.NewCounter("controlclient_map_response_message") // any message type
+	metricMapResponsePings      = clientmetric.NewCounter("controlclient_map_response_ping")
+	metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive")
+	metricMapResponseMap        = clientmetric.NewCounter("controlclient_map_response_map")       // any non-keepalive map response
+	metricMapResponseMapDelta   = clientmetric.NewCounter("controlclient_map_response_map_delta") // 2nd+ non-keepalive map response
+
+	metricSetDNS      = clientmetric.NewCounter("controlclient_setdns")
+	metricSetDNSError = clientmetric.NewCounter("controlclient_setdns_error")
+)

control/controlclient/map.go

@@ -110,7 +110,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 	}
 
 	nm := &netmap.NetworkMap{
-		NodeKey:         tailcfg.NodeKeyFromNodePublic(ms.privateNodeKey.Public()),
+		NodeKey:         ms.privateNodeKey.Public(),
 		PrivateKey:      ms.privateNodeKey,
 		MachineKey:      ms.machinePubKey,
 		Peers:           resp.Peers,

control/noise/conn.go

@@ -0,0 +1,359 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package noise implements the base transport of the Tailscale 2021
+// control protocol.
+//
+// The base transport implements Noise IK, instantiated with
+// Curve25519, ChaCha20Poly1305 and BLAKE2s.
+package noise
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"tailscale.com/types/key"
+)
+
+const (
+	// maxMessageSize is the maximum size of a protocol frame on the
+	// wire, including header and payload.
+	maxMessageSize = 4096
+	// maxCiphertextSize is the maximum amount of ciphertext bytes
+	// that one protocol frame can carry, after framing.
+	maxCiphertextSize = maxMessageSize - 3
+	// maxPlaintextSize is the maximum amount of plaintext bytes that
+	// one protocol frame can carry, after encryption and framing.
+	maxPlaintextSize = maxCiphertextSize - chp.Overhead
+)
+
+// A Conn is a secured Noise connection. It implements the net.Conn
+// interface, with the unusual trait that any write error (including a
+// SetWriteDeadline induced i/o timeout) causes all future writes to
+// fail.
+type Conn struct {
+	conn          net.Conn
+	version       uint16
+	peer          key.MachinePublic
+	handshakeHash [blake2s.Size]byte
+	rx            rxState
+	tx            txState
+}
+
+// rxState is all the Conn state that Read uses.
+type rxState struct {
+	sync.Mutex
+	cipher    cipher.AEAD
+	nonce     nonce
+	buf       [maxMessageSize]byte
+	n         int    // number of valid bytes in buf
+	next      int    // offset of next undecrypted packet
+	plaintext []byte // slice into buf of decrypted bytes
+}
+
+// txState is all the Conn state that Write uses.
+type txState struct {
+	sync.Mutex
+	cipher cipher.AEAD
+	nonce  nonce
+	buf    [maxMessageSize]byte
+	err    error // records the first partial write error for all future calls
+}
+
+// ProtocolVersion returns the protocol version that was used to
+// establish this Conn.
+func (c *Conn) ProtocolVersion() int {
+	return int(c.version)
+}
+
+// HandshakeHash returns the Noise handshake hash for the connection,
+// which can be used to bind other messages to this connection
+// (i.e. to ensure that the message wasn't replayed from a different
+// connection).
+func (c *Conn) HandshakeHash() [blake2s.Size]byte {
+	return c.handshakeHash
+}
+
+// Peer returns the peer's long-term public key.
+func (c *Conn) Peer() key.MachinePublic {
+	return c.peer
+}
+
+// readNLocked reads into c.rx.buf until buf contains at least total
+// bytes. Returns a slice of the total bytes in rxBuf, or an
+// error if fewer than total bytes are available.
+func (c *Conn) readNLocked(total int) ([]byte, error) {
+	if total > maxMessageSize {
+		return nil, errReadTooBig{total}
+	}
+	for {
+		if total <= c.rx.n {
+			return c.rx.buf[:total], nil
+		}
+
+		n, err := c.conn.Read(c.rx.buf[c.rx.n:])
+		c.rx.n += n
+		if err != nil {
+			return nil, err
+		}
+	}
+}
+
+// decryptLocked decrypts msg (which is header+ciphertext) in-place
+// and sets c.rx.plaintext to the decrypted bytes.
+func (c *Conn) decryptLocked(msg []byte) (err error) {
+	if msgType := msg[0]; msgType != msgTypeRecord {
+		return fmt.Errorf("received message with unexpected type %d, want %d", msgType, msgTypeRecord)
+	}
+	// We don't check the length field here, because the caller
+	// already did in order to figure out how big the msg slice should
+	// be.
+	ciphertext := msg[headerLen:]
+
+	if !c.rx.nonce.Valid() {
+		return errCipherExhausted{}
+	}
+
+	c.rx.plaintext, err = c.rx.cipher.Open(ciphertext[:0], c.rx.nonce[:], ciphertext, nil)
+	c.rx.nonce.Increment()
+
+	if err != nil {
+		// Once a decryption has failed, our Conn is no longer
+		// synchronized with our peer. Nuke the cipher state to be
+		// safe, so that no further decryptions are attempted. Future
+		// read attempts will return net.ErrClosed.
+		c.rx.cipher = nil
+	}
+	return err
+}
+
+// encryptLocked encrypts plaintext into c.tx.buf (including the
+// packet header) and returns a slice of the ciphertext, or an error
+// if the cipher is exhausted (i.e. can no longer be used safely).
+func (c *Conn) encryptLocked(plaintext []byte) ([]byte, error) {
+	if !c.tx.nonce.Valid() {
+		// Received 2^64-1 messages on this cipher state. Connection
+		// is no longer usable.
+		return nil, errCipherExhausted{}
+	}
+
+	c.tx.buf[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(c.tx.buf[1:headerLen], uint16(len(plaintext)+chp.Overhead))
+	ret := c.tx.cipher.Seal(c.tx.buf[:headerLen], c.tx.nonce[:], plaintext, nil)
+	c.tx.nonce.Increment()
+
+	return ret, nil
+}
+
+// wholeMessageLocked returns a slice of one whole Noise transport
+// message from c.rx.buf, if one whole message is available, and
+// advances the read state to the next Noise message in the
+// buffer. Returns nil without advancing read state if there isn't one
+// whole message in c.rx.buf.
+func (c *Conn) wholeMessageLocked() []byte {
+	available := c.rx.n - c.rx.next
+	if available < headerLen {
+		return nil
+	}
+	bs := c.rx.buf[c.rx.next:c.rx.n]
+	totalSize := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	if len(bs) < totalSize {
+		return nil
+	}
+	c.rx.next += totalSize
+	return bs[:totalSize]
+}
+
+// decryptOneLocked decrypts one Noise transport message, reading from
+// c.conn as needed, and sets c.rx.plaintext to point to the decrypted
+// bytes. c.rx.plaintext is only valid if err == nil.
+func (c *Conn) decryptOneLocked() error {
+	c.rx.plaintext = nil
+
+	// Fast path: do we have one whole ciphertext frame buffered
+	// already?
+	if bs := c.wholeMessageLocked(); bs != nil {
+		return c.decryptLocked(bs)
+	}
+
+	if c.rx.next != 0 {
+		// To simplify the read logic, move the remainder of the
+		// buffered bytes back to the head of the buffer, so we can
+		// grow it without worrying about wraparound.
+		c.rx.n = copy(c.rx.buf[:], c.rx.buf[c.rx.next:c.rx.n])
+		c.rx.next = 0
+	}
+
+	bs, err := c.readNLocked(headerLen)
+	if err != nil {
+		return err
+	}
+	// The rest of the header (besides the length field) gets verified
+	// in decryptLocked, not here.
+	messageLen := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	bs, err = c.readNLocked(messageLen)
+	if err != nil {
+		return err
+	}
+
+	c.rx.next = len(bs)
+
+	return c.decryptLocked(bs)
+}
+
+// Read implements io.Reader.
+func (c *Conn) Read(bs []byte) (int, error) {
+	c.rx.Lock()
+	defer c.rx.Unlock()
+
+	if c.rx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+	// If no plaintext is buffered, decrypt incoming frames until we
+	// have some plaintext. Zero-byte Noise frames are allowed in this
+	// protocol, which is why we have to loop here rather than decrypt
+	// a single additional frame.
+	for len(c.rx.plaintext) == 0 {
+		if err := c.decryptOneLocked(); err != nil {
+			return 0, err
+		}
+	}
+	n := copy(bs, c.rx.plaintext)
+	c.rx.plaintext = c.rx.plaintext[n:]
+	return n, nil
+}
+
+// Write implements io.Writer.
+func (c *Conn) Write(bs []byte) (n int, err error) {
+	c.tx.Lock()
+	defer c.tx.Unlock()
+
+	if c.tx.err != nil {
+		return 0, c.tx.err
+	}
+	defer func() {
+		if err != nil {
+			// All write errors are fatal for this conn, so clear the
+			// cipher state whenever an error happens.
+			c.tx.cipher = nil
+		}
+		if c.tx.err == nil {
+			// Only set c.tx.err if not nil so that we can return one
+			// error on the first failure, and a different one for
+			// subsequent calls. See the error handling around Write
+			// below for why.
+			c.tx.err = err
+		}
+	}()
+
+	if c.tx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+
+	var sent int
+	for len(bs) > 0 {
+		toSend := bs
+		if len(toSend) > maxPlaintextSize {
+			toSend = bs[:maxPlaintextSize]
+		}
+		bs = bs[len(toSend):]
+
+		ciphertext, err := c.encryptLocked(toSend)
+		if err != nil {
+			return 0, err
+		}
+
+		n, err := c.conn.Write(ciphertext)
+		sent += n
+		if err != nil {
+			// Return the raw error on the Write that actually
+			// failed. For future writes, return that error wrapped in
+			// a desync error.
+			c.tx.err = errPartialWrite{err}
+			return sent, err
+		}
+	}
+	return sent, nil
+}
+
+// Close implements io.Closer.
+func (c *Conn) Close() error {
+	closeErr := c.conn.Close() // unblocks any waiting reads or writes
+
+	// Remove references to live cipher state. Strictly speaking this
+	// is unnecessary, but we want to try and hand the active cipher
+	// state to the garbage collector promptly, to preserve perfect
+	// forward secrecy as much as we can.
+	c.rx.Lock()
+	c.rx.cipher = nil
+	c.rx.Unlock()
+	c.tx.Lock()
+	c.tx.cipher = nil
+	c.tx.Unlock()
+	return closeErr
+}
+
+func (c *Conn) LocalAddr() net.Addr                { return c.conn.LocalAddr() }
+func (c *Conn) RemoteAddr() net.Addr               { return c.conn.RemoteAddr() }
+func (c *Conn) SetDeadline(t time.Time) error      { return c.conn.SetDeadline(t) }
+func (c *Conn) SetReadDeadline(t time.Time) error  { return c.conn.SetReadDeadline(t) }
+func (c *Conn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) }
+
+// errCipherExhausted is the error returned when we run out of nonces
+// on a cipher.
+type errCipherExhausted struct{}
+
+func (errCipherExhausted) Error() string {
+	return "cipher exhausted, no more nonces available for current key"
+}
+func (errCipherExhausted) Timeout() bool   { return false }
+func (errCipherExhausted) Temporary() bool { return false }
+
+// errPartialWrite is the error returned when the cipher state has
+// become unusable due to a past partial write.
+type errPartialWrite struct {
+	err error
+}
+
+func (e errPartialWrite) Error() string {
+	return fmt.Sprintf("cipher state desynchronized due to partial write (%v)", e.err)
+}
+func (e errPartialWrite) Unwrap() error   { return e.err }
+func (e errPartialWrite) Temporary() bool { return false }
+func (e errPartialWrite) Timeout() bool   { return false }
+
+// errReadTooBig is the error returned when the peer sent an
+// unacceptably large Noise frame.
+type errReadTooBig struct {
+	requested int
+}
+
+func (e errReadTooBig) Error() string {
+	return fmt.Sprintf("requested read of %d bytes exceeds max allowed Noise frame size", e.requested)
+}
+func (e errReadTooBig) Temporary() bool {
+	// permanent error because this error only occurs when our peer
+	// sends us a frame so large we're unwilling to ever decode it.
+	return false
+}
+func (e errReadTooBig) Timeout() bool { return false }
+
+type nonce [chp.NonceSize]byte
+
+func (n *nonce) Valid() bool {
+	return binary.BigEndian.Uint32(n[:4]) == 0 && binary.BigEndian.Uint64(n[4:]) != invalidNonce
+}
+
+func (n *nonce) Increment() {
+	if !n.Valid() {
+		panic("increment of invalid nonce")
+	}
+	binary.BigEndian.PutUint64(n[4:], 1+binary.BigEndian.Uint64(n[4:]))
+}

control/noise/conn_test.go

@@ -0,0 +1,339 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"testing/iotest"
+
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/net/nettest"
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestMessageSize(t *testing.T) {
+	// This test is a regression guard against someone looking at
+	// maxCiphertextSize, going "huh, we could be more efficient if it
+	// were larger, and accidentally violating the Noise spec. Do not
+	// change this max value, it's a deliberate limitation of the
+	// cryptographic protocol we use (see Section 3 "Message Format"
+	// of the Noise spec).
+	const max = 65535
+	if maxCiphertextSize > max {
+		t.Fatalf("max ciphertext size is %d, which is larger than the maximum noise message size %d", maxCiphertextSize, max)
+	}
+}
+
+func TestConnBasic(t *testing.T) {
+	client, server := pair(t)
+
+	sb := sinkReads(server)
+
+	want := "test"
+	if _, err := io.WriteString(client, want); err != nil {
+		t.Fatalf("client write failed: %v", err)
+	}
+	client.Close()
+
+	if got := sb.String(4); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatal("client close wasn't seen by server")
+	}
+	if sb.Total() != 4 {
+		t.Fatalf("wrong amount of bytes received: got %d, want 4", sb.Total())
+	}
+}
+
+// bufferedWriteConn wraps a net.Conn and gives control over how
+// Writes get batched out.
+type bufferedWriteConn struct {
+	net.Conn
+	w           *bufio.Writer
+	manualFlush bool
+}
+
+func (c *bufferedWriteConn) Write(bs []byte) (int, error) {
+	n, err := c.w.Write(bs)
+	if err == nil && !c.manualFlush {
+		err = c.w.Flush()
+	}
+	return n, err
+}
+
+// TestFastPath exercises the Read codepath that can receive multiple
+// Noise frames at once and decode each in turn without making another
+// syscall.
+func TestFastPath(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	b := &bufferedWriteConn{s1, bufio.NewWriterSize(s1, 10000), false}
+	client, server := pairWithConns(t, b, s2)
+
+	b.manualFlush = true
+
+	sb := sinkReads(server)
+
+	const packets = 10
+	s := "test"
+	for i := 0; i < packets; i++ {
+		// Many separate writes, to force separate Noise frames that
+		// all get buffered up and then all sent as a single slice to
+		// the server.
+		if _, err := io.WriteString(client, s); err != nil {
+			t.Fatalf("client write1 failed: %v", err)
+		}
+	}
+	if err := b.w.Flush(); err != nil {
+		t.Fatalf("client flush failed: %v", err)
+	}
+	client.Close()
+
+	want := strings.Repeat(s, packets)
+	if got := sb.String(len(want)); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatalf("client close wasn't seen by server")
+	}
+}
+
+// Writes things larger than a single Noise frame, to check the
+// chunking on the encoder and decoder.
+func TestBigData(t *testing.T) {
+	client, server := pair(t)
+
+	serverReads := sinkReads(server)
+	clientReads := sinkReads(client)
+
+	const sz = 15 * 1024 // 15KiB
+	clientStr := strings.Repeat("abcde", sz/5)
+	serverStr := strings.Repeat("fghij", sz/5*2)
+
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+	if _, err := io.WriteString(server, serverStr); err != nil {
+		t.Fatalf("writing server>client: %v", err)
+	}
+
+	if serverGot := serverReads.String(sz); serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+	if clientGot := clientReads.String(2 * sz); clientGot != serverStr {
+		t.Error("client didn't receive what server sent")
+	}
+
+	getNonce := func(n [chp.NonceSize]byte) uint64 {
+		if binary.BigEndian.Uint32(n[:4]) != 0 {
+			panic("unexpected nonce")
+		}
+		return binary.BigEndian.Uint64(n[4:])
+	}
+
+	// Reach into the Conns and verify the cipher nonces advanced as
+	// expected.
+	if getNonce(client.tx.nonce) != getNonce(server.rx.nonce) {
+		t.Error("desynchronized client tx nonce")
+	}
+	if getNonce(server.tx.nonce) != getNonce(client.rx.nonce) {
+		t.Error("desynchronized server tx nonce")
+	}
+	if n := getNonce(client.tx.nonce); n != 4 {
+		t.Errorf("wrong client tx nonce, got %d want 4", n)
+	}
+	if n := getNonce(server.tx.nonce); n != 8 {
+		t.Errorf("wrong client tx nonce, got %d want 8", n)
+	}
+}
+
+// readerConn wraps a net.Conn and routes its Reads through a separate
+// io.Reader.
+type readerConn struct {
+	net.Conn
+	r io.Reader
+}
+
+func (c readerConn) Read(bs []byte) (int, error) { return c.r.Read(bs) }
+
+// Check that the receiver can handle not being able to read an entire
+// frame in a single syscall.
+func TestDataTrickle(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	client, server := pairWithConns(t, s1, readerConn{s2, iotest.OneByteReader(s2)})
+	serverReads := sinkReads(server)
+
+	const sz = 10000
+	clientStr := strings.Repeat("abcde", sz/5)
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+
+	serverGot := serverReads.String(sz)
+	if serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+}
+
+func TestConnStd(t *testing.T) {
+	// You can run this test manually, and noise.Conn should pass all
+	// of them except for TestConn/PastTimeout,
+	// TestConn/FutureTimeout, TestConn/ConcurrentMethods, because
+	// those tests assume that write errors are recoverable, and
+	// they're not on our Conn due to cipher security.
+	t.Skip("not all tests can pass on this Conn, see https://github.com/golang/go/issues/46977")
+	nettest.TestConn(t, func() (c1 net.Conn, c2 net.Conn, stop func(), err error) {
+		s1, s2 := tsnettest.NewConn("noise", 4096)
+		controlKey := key.NewMachine()
+		machineKey := key.NewMachine()
+		serverErr := make(chan error, 1)
+		go func() {
+			var err error
+			c2, err = Server(context.Background(), s2, controlKey)
+			serverErr <- err
+		}()
+		c1, err = Client(context.Background(), s1, machineKey, controlKey.Public())
+		if err != nil {
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting client: %w", err)
+		}
+		if err := <-serverErr; err != nil {
+			c1.Close()
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting server: %w", err)
+		}
+		return c1, c2, func() {
+			c1.Close()
+			c2.Close()
+		}, nil
+	})
+}
+
+// mkConns creates synthetic Noise Conns wrapping the given net.Conns.
+// This function is for testing just the Conn transport logic without
+// having to muck about with Noise handshakes.
+func mkConns(s1, s2 net.Conn) (*Conn, *Conn) {
+	var k1, k2 [chp.KeySize]byte
+	if _, err := rand.Read(k1[:]); err != nil {
+		panic(err)
+	}
+	if _, err := rand.Read(k2[:]); err != nil {
+		panic(err)
+	}
+
+	ret1 := &Conn{
+		conn: s1,
+		tx:   txState{cipher: newCHP(k1)},
+		rx:   rxState{cipher: newCHP(k2)},
+	}
+	ret2 := &Conn{
+		conn: s2,
+		tx:   txState{cipher: newCHP(k2)},
+		rx:   rxState{cipher: newCHP(k1)},
+	}
+
+	return ret1, ret2
+}
+
+type readSink struct {
+	r io.Reader
+
+	cond *sync.Cond
+	sync.Mutex
+	bs  bytes.Buffer
+	err error
+}
+
+func sinkReads(r io.Reader) *readSink {
+	ret := &readSink{
+		r: r,
+	}
+	ret.cond = sync.NewCond(&ret.Mutex)
+	go func() {
+		var buf [4096]byte
+		for {
+			n, err := r.Read(buf[:])
+			ret.Lock()
+			ret.bs.Write(buf[:n])
+			if err != nil {
+				ret.err = err
+			}
+			ret.cond.Broadcast()
+			ret.Unlock()
+			if err != nil {
+				return
+			}
+		}
+	}()
+	return ret
+}
+
+func (s *readSink) String(total int) string {
+	s.Lock()
+	defer s.Unlock()
+	for s.bs.Len() < total && s.err == nil {
+		s.cond.Wait()
+	}
+	if s.err != nil {
+		total = s.bs.Len()
+	}
+	return string(s.bs.Bytes()[:total])
+}
+
+func (s *readSink) Error() error {
+	s.Lock()
+	defer s.Unlock()
+	for s.err == nil {
+		s.cond.Wait()
+	}
+	return s.err
+}
+
+func (s *readSink) Total() int {
+	s.Lock()
+	defer s.Unlock()
+	return s.bs.Len()
+}
+
+func pairWithConns(t *testing.T, clientConn, serverConn net.Conn) (*Conn, *Conn) {
+	var (
+		controlKey = key.NewMachine()
+		machineKey = key.NewMachine()
+		server     *Conn
+		serverErr  = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, controlKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, machineKey, controlKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+	return client, server
+}
+
+func pair(t *testing.T) (*Conn, *Conn) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	return pairWithConns(t, s1, s2)
+}

control/noise/handshake.go

@@ -0,0 +1,443 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	"go4.org/mem"
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+	"tailscale.com/types/key"
+)
+
+const (
+	// protocolName is the name of the specific instantiation of Noise
+	// that the control protocol uses. This string's value is fixed by
+	// the Noise spec, and shouldn't be changed unless we're updating
+	// the control protocol to use a different Noise instance.
+	protocolName = "Noise_IK_25519_ChaChaPoly_BLAKE2s"
+	// protocolVersion is the version of the control protocol that
+	// Client will use when initiating a handshake.
+	protocolVersion uint16 = 1
+	// protocolVersionPrefix is the name portion of the protocol
+	// name+version string that gets mixed into the handshake as a
+	// prologue.
+	//
+	// This mixing verifies that both clients agree that they're
+	// executing the control protocol at a specific version that
+	// matches the advertised version in the cleartext packet header.
+	protocolVersionPrefix = "Tailscale Control Protocol v"
+	invalidNonce          = ^uint64(0)
+)
+
+func protocolVersionPrologue(version uint16) []byte {
+	ret := make([]byte, 0, len(protocolVersionPrefix)+5) // 5 bytes is enough to encode all possible version numbers.
+	ret = append(ret, protocolVersionPrefix...)
+	return strconv.AppendUint(ret, uint64(version), 10)
+}
+
+// Client initiates a control client handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process. Any preexisting Conn deadline is removed.
+func Client(ctx context.Context, conn net.Conn, machineKey key.MachinePrivate, controlKey key.MachinePublic) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	// prologue
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	s.MixHash(controlKey.UntypedBytes())
+
+	// -> e, es, s, ss
+	init := mkInitiationMessage()
+	machineEphemeral := key.NewMachine()
+	machineEphemeralPub := machineEphemeral.Public()
+	copy(init.EphemeralPub(), machineEphemeralPub.UntypedBytes())
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(machineEphemeral, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	machineKeyPub := machineKey.Public()
+	s.EncryptAndHash(cipher, init.MachinePub(), machineKeyPub.UntypedBytes())
+	cipher, err = s.MixDH(machineKey, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	s.EncryptAndHash(cipher, init.Tag(), nil) // empty message payload
+
+	if _, err := conn.Write(init[:]); err != nil {
+		return nil, fmt.Errorf("writing initiation: %w", err)
+	}
+
+	// Read in the payload and look for errors/protocol violations from the server.
+	var resp responseMessage
+	if _, err := io.ReadFull(conn, resp.Header()); err != nil {
+		return nil, fmt.Errorf("reading response header: %w", err)
+	}
+	if resp.Type() != msgTypeResponse {
+		if resp.Type() != msgTypeError {
+			return nil, fmt.Errorf("unexpected response message type %d", resp.Type())
+		}
+		msg := make([]byte, resp.Length())
+		if _, err := io.ReadFull(conn, msg); err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("server error: %q", msg)
+	}
+	if resp.Length() != len(resp.Payload()) {
+		return nil, fmt.Errorf("wrong length %d received for handshake response", resp.Length())
+	}
+	if _, err := io.ReadFull(conn, resp.Payload()); err != nil {
+		return nil, err
+	}
+
+	// <- e, ee, se
+	controlEphemeralPub := key.MachinePublicFromRaw32(mem.B(resp.EphemeralPub()))
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err = s.MixDH(machineEphemeral, controlEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(machineKey, controlEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, resp.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting payload: %w", err)
+	}
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          controlKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c1,
+		},
+		rx: rxState{
+			cipher: c2,
+		},
+	}
+	return c, nil
+}
+
+// Server initiates a control server handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process.
+func Server(ctx context.Context, conn net.Conn, controlKey key.MachinePrivate) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	// Deliberately does not support formatting, so that we don't echo
+	// attacker-controlled input back to them.
+	sendErr := func(msg string) error {
+		if len(msg) >= 1<<16 {
+			msg = msg[:1<<16]
+		}
+		var hdr [headerLen]byte
+		hdr[0] = msgTypeError
+		binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg)))
+		if _, err := conn.Write(hdr[:]); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		if _, err := io.WriteString(conn, msg); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		return fmt.Errorf("refused client handshake: %q", msg)
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	var init initiationMessage
+	if _, err := io.ReadFull(conn, init.Header()); err != nil {
+		return nil, err
+	}
+	if init.Version() != protocolVersion {
+		return nil, sendErr("unsupported protocol version")
+	}
+	if init.Type() != msgTypeInitiation {
+		return nil, sendErr("unexpected handshake message type")
+	}
+	if init.Length() != len(init.Payload()) {
+		return nil, sendErr("wrong handshake initiation length")
+	}
+	if _, err := io.ReadFull(conn, init.Payload()); err != nil {
+		return nil, err
+	}
+
+	// prologue. Can only do this once we at least think the client is
+	// handshaking using a supported version.
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	controlKeyPub := controlKey.Public()
+	s.MixHash(controlKeyPub.UntypedBytes())
+
+	// -> e, es, s, ss
+	machineEphemeralPub := key.MachinePublicFromRaw32(mem.B(init.EphemeralPub()))
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(controlKey, machineEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	var machineKeyBytes [32]byte
+	if err := s.DecryptAndHash(cipher, machineKeyBytes[:], init.MachinePub()); err != nil {
+		return nil, fmt.Errorf("decrypting machine key: %w", err)
+	}
+	machineKey := key.MachinePublicFromRaw32(mem.B(machineKeyBytes[:]))
+	cipher, err = s.MixDH(controlKey, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, init.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting initiation tag: %w", err)
+	}
+
+	// <- e, ee, se
+	resp := mkResponseMessage()
+	controlEphemeral := key.NewMachine()
+	controlEphemeralPub := controlEphemeral.Public()
+	copy(resp.EphemeralPub(), controlEphemeralPub.UntypedBytes())
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err := s.MixDH(controlEphemeral, machineEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(controlEphemeral, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	s.EncryptAndHash(cipher, resp.Tag(), nil) // empty message payload
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	if _, err := conn.Write(resp[:]); err != nil {
+		return nil, err
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          machineKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c2,
+		},
+		rx: rxState{
+			cipher: c1,
+		},
+	}
+	return c, nil
+}
+
+// symmetricState contains the state of an in-flight handshake.
+type symmetricState struct {
+	finished bool
+
+	h  [blake2s.Size]byte // hash of currently-processed handshake state
+	ck [blake2s.Size]byte // chaining key used to construct session keys at the end of the handshake
+}
+
+func (s *symmetricState) checkFinished() {
+	if s.finished {
+		panic("attempted to use symmetricState after Split was called")
+	}
+}
+
+// Initialize sets s to the initial handshake state, prior to
+// processing any handshake messages.
+func (s *symmetricState) Initialize() {
+	s.checkFinished()
+	s.h = blake2s.Sum256([]byte(protocolName))
+	s.ck = s.h
+}
+
+// MixHash updates s.h to be BLAKE2s(s.h || data), where || is
+// concatenation.
+func (s *symmetricState) MixHash(data []byte) {
+	s.checkFinished()
+	h := newBLAKE2s()
+	h.Write(s.h[:])
+	h.Write(data)
+	h.Sum(s.h[:0])
+}
+
+// MixDH updates s.ck with the result of X25519(priv, pub) and returns
+// a singleUseCHP that can be used to encrypt or decrypt handshake
+// data.
+//
+// MixDH corresponds to MixKey(X25519(...))) in the spec. Implementing
+// it as a single function allows for strongly-typed arguments that
+// reduce the risk of error in the caller (e.g. invoking X25519 with
+// two private keys, or two public keys), and thus producing the wrong
+// calculation.
+func (s *symmetricState) MixDH(priv key.MachinePrivate, pub key.MachinePublic) (*singleUseCHP, error) {
+	s.checkFinished()
+	keyData, err := curve25519.X25519(priv.UntypedBytes(), pub.UntypedBytes())
+	if err != nil {
+		return nil, fmt.Errorf("computing X25519: %w", err)
+	}
+
+	r := hkdf.New(newBLAKE2s, keyData, s.ck[:], nil)
+	if _, err := io.ReadFull(r, s.ck[:]); err != nil {
+		return nil, fmt.Errorf("extracting ck: %w", err)
+	}
+	var k [chp.KeySize]byte
+	if _, err := io.ReadFull(r, k[:]); err != nil {
+		return nil, fmt.Errorf("extracting k: %w", err)
+	}
+	return newSingleUseCHP(k), nil
+}
+
+// EncryptAndHash encrypts plaintext into ciphertext (which must be
+// the correct size to hold the encrypted plaintext) using cipher,
+// mixes the ciphertext into s.h, and returns the ciphertext.
+func (s *symmetricState) EncryptAndHash(cipher *singleUseCHP, ciphertext, plaintext []byte) {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		panic("ciphertext is wrong size for given plaintext")
+	}
+	ret := cipher.Seal(ciphertext[:0], plaintext, s.h[:])
+	s.MixHash(ret)
+}
+
+// DecryptAndHash decrypts the given ciphertext into plaintext (which
+// must be the correct size to hold the decrypted ciphertext) using
+// cipher. If decryption is successful, it mixes the ciphertext into
+// s.h.
+func (s *symmetricState) DecryptAndHash(cipher *singleUseCHP, plaintext, ciphertext []byte) error {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		return errors.New("plaintext is wrong size for given ciphertext")
+	}
+	if _, err := cipher.Open(plaintext[:0], ciphertext, s.h[:]); err != nil {
+		return err
+	}
+	s.MixHash(ciphertext)
+	return nil
+}
+
+// Split returns two ChaCha20Poly1305 ciphers with keys derived from
+// the current handshake state. Methods on s cannot be used again
+// after calling Split.
+func (s *symmetricState) Split() (c1, c2 cipher.AEAD, err error) {
+	s.finished = true
+
+	var k1, k2 [chp.KeySize]byte
+	r := hkdf.New(newBLAKE2s, nil, s.ck[:], nil)
+	if _, err := io.ReadFull(r, k1[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k1: %w", err)
+	}
+	if _, err := io.ReadFull(r, k2[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k2: %w", err)
+	}
+	c1, err = chp.New(k1[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c1: %w", err)
+	}
+	c2, err = chp.New(k2[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c2: %w", err)
+	}
+	return c1, c2, nil
+}
+
+// newBLAKE2s returns a hash.Hash implementing BLAKE2s, or panics on
+// error.
+func newBLAKE2s() hash.Hash {
+	h, err := blake2s.New256(nil)
+	if err != nil {
+		// Should never happen, errors only happen when using BLAKE2s
+		// in MAC mode with a key.
+		panic(err)
+	}
+	return h
+}
+
+// newCHP returns a cipher.AEAD implementing ChaCha20Poly1305, or
+// panics on error.
+func newCHP(key [chp.KeySize]byte) cipher.AEAD {
+	aead, err := chp.New(key[:])
+	if err != nil {
+		// Can only happen if we passed a key of the wrong length. The
+		// function signature prevents that.
+		panic(err)
+	}
+	return aead
+}
+
+// singleUseCHP is an instance of ChaCha20Poly1305 that can be used
+// only once, either for encrypting or decrypting, but not both. The
+// chosen operation is always executed with an all-zeros
+// nonce. Subsequent calls to either Seal or Open panic.
+type singleUseCHP struct {
+	c cipher.AEAD
+}
+
+func newSingleUseCHP(key [chp.KeySize]byte) *singleUseCHP {
+	return &singleUseCHP{newCHP(key)}
+}
+
+func (c *singleUseCHP) Seal(dst, plaintext, additionalData []byte) []byte {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Seal(dst, nonce[:], plaintext, additionalData)
+}
+
+func (c *singleUseCHP) Open(dst, ciphertext, additionalData []byte) ([]byte, error) {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Open(dst, nonce[:], ciphertext, additionalData)
+}

control/noise/handshake_test.go

@@ -0,0 +1,296 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestHandshake(t *testing.T) {
+	var (
+		clientConn, serverConn = tsnettest.NewConn("noise", 128000)
+		serverKey              = key.NewMachine()
+		clientKey              = key.NewMachine()
+		server                 *Conn
+		serverErr              = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, serverKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+
+	if client.HandshakeHash() != server.HandshakeHash() {
+		t.Fatal("client and server disagree on handshake hash")
+	}
+
+	if client.ProtocolVersion() != int(protocolVersion) {
+		t.Fatalf("client reporting wrong protocol version %d, want %d", client.ProtocolVersion(), protocolVersion)
+	}
+	if client.ProtocolVersion() != server.ProtocolVersion() {
+		t.Fatalf("peers disagree on protocol version, client=%d server=%d", client.ProtocolVersion(), server.ProtocolVersion())
+	}
+	if client.Peer() != serverKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+	if server.Peer() != clientKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+}
+
+// Check that handshaking repeatedly with the same long-term keys
+// result in different handshake hashes and wire traffic.
+func TestNoReuse(t *testing.T) {
+	var (
+		hashes           = map[[32]byte]bool{}
+		clientHandshakes = map[[96]byte]bool{}
+		serverHandshakes = map[[48]byte]bool{}
+		packets          = map[[32]byte]bool{}
+	)
+	for i := 0; i < 10; i++ {
+		var (
+			clientRaw, serverRaw = tsnettest.NewConn("noise", 128000)
+			clientBuf, serverBuf bytes.Buffer
+			clientConn           = &readerConn{clientRaw, io.TeeReader(clientRaw, &clientBuf)}
+			serverConn           = &readerConn{serverRaw, io.TeeReader(serverRaw, &serverBuf)}
+			serverKey            = key.NewMachine()
+			clientKey            = key.NewMachine()
+			server               *Conn
+			serverErr            = make(chan error, 1)
+		)
+		go func() {
+			var err error
+			server, err = Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client connection failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed: %v", err)
+		}
+
+		var clientHS [96]byte
+		copy(clientHS[:], serverBuf.Bytes())
+		if clientHandshakes[clientHS] {
+			t.Fatal("client handshake seen twice")
+		}
+		clientHandshakes[clientHS] = true
+
+		var serverHS [48]byte
+		copy(serverHS[:], clientBuf.Bytes())
+		if serverHandshakes[serverHS] {
+			t.Fatal("server handshake seen twice")
+		}
+		serverHandshakes[serverHS] = true
+
+		clientBuf.Reset()
+		serverBuf.Reset()
+		cb := sinkReads(client)
+		sb := sinkReads(server)
+
+		if hashes[client.HandshakeHash()] {
+			t.Fatalf("handshake hash %v seen twice", client.HandshakeHash())
+		}
+		hashes[client.HandshakeHash()] = true
+
+		// Sending 14 bytes turns into 32 bytes on the wire (+16 for
+		// the chacha20poly1305 overhead, +2 length header)
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if _, err := io.WriteString(server, strings.Repeat("b", 14)); err != nil {
+			t.Fatalf("server>client write failed: %v", err)
+		}
+
+		// Wait for the bytes to be read, so we know they've traveled end to end
+		cb.String(14)
+		sb.String(14)
+
+		var clientWire, serverWire [32]byte
+		copy(clientWire[:], clientBuf.Bytes())
+		copy(serverWire[:], serverBuf.Bytes())
+
+		if packets[clientWire] {
+			t.Fatalf("client wire traffic seen twice")
+		}
+		packets[clientWire] = true
+		if packets[serverWire] {
+			t.Fatalf("server wire traffic seen twice")
+		}
+		packets[serverWire] = true
+	}
+}
+
+// tamperReader wraps a reader and mutates the Nth byte.
+type tamperReader struct {
+	r     io.Reader
+	n     int
+	total int
+}
+
+func (r *tamperReader) Read(bs []byte) (int, error) {
+	n, err := r.r.Read(bs)
+	if off := r.n - r.total; off >= 0 && off < n {
+		bs[off] += 1
+	}
+	r.total += n
+	return n, err
+}
+
+func TestTampering(t *testing.T) {
+	// Tamper with every byte of the client initiation message.
+	for i := 0; i < 101; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			// If the server failed, we have to close the Conn to
+			// unblock the client.
+			if err != nil {
+				serverConn.Close()
+			}
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatalf("server connection succeeded despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the server response message.
+	for i := 0; i < 51; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed despite no tampering: %v", err)
+		}
+	}
+
+	// Tamper with every byte of the first server>client transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, 51 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			_, err = io.WriteString(server, strings.Repeat("a", 14))
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		// The client needs a timeout if the tampering is hitting the length header.
+		if i == 1 || i == 2 {
+			client.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+		}
+
+		var bs [100]byte
+		n, err := client.Read(bs[:])
+		if err == nil {
+			t.Fatal("read succeeded despite tampering")
+		}
+		if n != 0 {
+			t.Fatal("conn yielded some bytes despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the first client>server transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, 101 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			var bs [100]byte
+			// The server needs a timeout if the tampering is hitting the length header.
+			if i == 1 || i == 2 {
+				server.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+			}
+			n, err := server.Read(bs[:])
+			if n != 0 {
+				panic("server got bytes despite tampering")
+			} else {
+				serverErr <- err
+			}
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatal("server successfully received bytes despite tampering")
+		}
+	}
+}

control/noise/interop_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"net"
+	"testing"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+// Can a reference Noise IK client talk to our server?
+func TestInteropClient(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		serverErr   = make(chan error, 2)
+		serverBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		server, err := Server(context.Background(), s2, controlKey)
+		serverErr <- err
+		if err != nil {
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(server, buf[:len(c2s)])
+		serverBytes <- buf[:len(c2s)]
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		_, err = server.Write([]byte(s2c))
+		serverErr <- err
+	}()
+
+	gotS2C, err := noiseExplorerClient(s1, controlKey.Public(), machineKey, []byte(c2s))
+	if err != nil {
+		t.Fatalf("failed client interop: %v", err)
+	}
+	if string(gotS2C) != s2c {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotS2C), s2c)
+	}
+
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server handshake failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server read/write failed: %v", err)
+	}
+	if got := string(<-serverBytes); got != c2s {
+		t.Fatalf("server received %q, want %q", got, c2s)
+	}
+}
+
+// Can our client talk to a reference Noise IK server?
+func TestInteropServer(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		clientErr   = make(chan error, 2)
+		clientBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		client, err := Client(context.Background(), s1, machineKey, controlKey.Public())
+		clientErr <- err
+		if err != nil {
+			return
+		}
+		_, err = client.Write([]byte(c2s))
+		if err != nil {
+			clientErr <- err
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(client, buf[:len(s2c)])
+		clientBytes <- buf[:len(s2c)]
+		clientErr <- err
+	}()
+
+	gotC2S, err := noiseExplorerServer(s2, controlKey, machineKey.Public(), []byte(s2c))
+	if err != nil {
+		t.Fatalf("failed server interop: %v", err)
+	}
+	if string(gotC2S) != c2s {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotC2S), c2s)
+	}
+
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client handshake failed: %v", err)
+	}
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client read/write failed: %v", err)
+	}
+	if got := string(<-clientBytes); got != s2c {
+		t.Fatalf("client received %q, want %q", got, s2c)
+	}
+}
+
+// noiseExplorerClient uses the Noise Explorer implementation of Noise
+// IK to handshake as a Noise client on conn, transmit payload, and
+// read+return a payload from the peer.
+func noiseExplorerClient(conn net.Conn, controlKey key.MachinePublic, machineKey key.MachinePrivate, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], machineKey.UntypedBytes())
+	copy(mk.public_key[:], machineKey.Public().UntypedBytes())
+	var peerKey [32]byte
+	copy(peerKey[:], controlKey.UntypedBytes())
+	session := InitSession(true, protocolVersionPrologue(protocolVersion), mk, peerKey)
+
+	_, msg1 := SendMessage(&session, nil)
+	var hdr [initiationHeaderLen]byte
+	binary.BigEndian.PutUint16(hdr[:2], protocolVersion)
+	hdr[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(hdr[3:5], 96)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ns); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ciphertext); err != nil {
+		return nil, err
+	}
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:51]); err != nil {
+		return nil, err
+	}
+	// ignore the header for this test, we're only checking the noise
+	// implementation.
+	msg2 := messagebuffer{
+		ciphertext: buf[35:51],
+	}
+	copy(msg2.ne[:], buf[3:35])
+	_, p, valid := RecvMessage(&session, &msg2)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg3 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg3.ciphertext)))
+	if _, err := conn.Write(hdr[:3]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg3.ciphertext); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	// Ignore all of the header except the payload length
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg4 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg4)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	return p, nil
+}
+
+func noiseExplorerServer(conn net.Conn, controlKey key.MachinePrivate, wantMachineKey key.MachinePublic, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], controlKey.UntypedBytes())
+	copy(mk.public_key[:], controlKey.Public().UntypedBytes())
+	session := InitSession(false, protocolVersionPrologue(protocolVersion), mk, [32]byte{})
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:101]); err != nil {
+		return nil, err
+	}
+	// Ignore the header, we're just checking the noise implementation.
+	msg1 := messagebuffer{
+		ns:         buf[37:85],
+		ciphertext: buf[85:101],
+	}
+	copy(msg1.ne[:], buf[5:37])
+	_, p, valid := RecvMessage(&session, &msg1)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg2 := SendMessage(&session, nil)
+	var hdr [headerLen]byte
+	hdr[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(hdr[1:3], 48)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ciphertext[:]); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg3 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg3)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	_, msg4 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg4.ciphertext)))
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg4.ciphertext); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}

control/noise/messages.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import "encoding/binary"
+
+const (
+	// msgTypeInitiation frames carry a Noise IK handshake initiation message.
+	msgTypeInitiation = 1
+	// msgTypeResponse frames carry a Noise IK handshake response message.
+	msgTypeResponse = 2
+	// msgTypeError frames carry an unauthenticated human-readable
+	// error message.
+	//
+	// Errors reported in this message type must be treated as public
+	// hints only. They are not encrypted or authenticated, and so can
+	// be seen and tampered with on the wire.
+	msgTypeError = 3
+	// msgTypeRecord frames carry session data bytes.
+	msgTypeRecord = 4
+
+	// headerLen is the size of the header on all messages except msgTypeInitiation.
+	headerLen = 3
+	// initiationHeaderLen is the size of the header on all msgTypeInitiation messages.
+	initiationHeaderLen = 5
+)
+
+// initiationMessage is the protocol message sent from a client
+// machine to a control server.
+//
+// 2b: protocol version
+// 1b: message type (0x01)
+// 2b: payload length (96)
+// 5b: header (see headerLen for fields)
+// 32b: client ephemeral public key (cleartext)
+// 48b: client machine public key (encrypted)
+// 16b: message tag (authenticates the whole message)
+type initiationMessage [101]byte
+
+func mkInitiationMessage() initiationMessage {
+	var ret initiationMessage
+	binary.BigEndian.PutUint16(ret[:2], uint16(protocolVersion))
+	ret[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(ret[3:5], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *initiationMessage) Header() []byte  { return m[:initiationHeaderLen] }
+func (m *initiationMessage) Payload() []byte { return m[initiationHeaderLen:] }
+
+func (m *initiationMessage) Version() uint16 { return binary.BigEndian.Uint16(m[:2]) }
+func (m *initiationMessage) Type() byte      { return m[2] }
+func (m *initiationMessage) Length() int     { return int(binary.BigEndian.Uint16(m[3:5])) }
+
+func (m *initiationMessage) EphemeralPub() []byte {
+	return m[initiationHeaderLen : initiationHeaderLen+32]
+}
+func (m *initiationMessage) MachinePub() []byte {
+	return m[initiationHeaderLen+32 : initiationHeaderLen+32+48]
+}
+func (m *initiationMessage) Tag() []byte { return m[initiationHeaderLen+32+48:] }
+
+// responseMessage is the protocol message sent from a control server
+// to a client machine.
+//
+// 1b: message type (0x02)
+// 2b: payload length (48)
+// 32b: control ephemeral public key (cleartext)
+// 16b: message tag (authenticates the whole message)
+type responseMessage [51]byte
+
+func mkResponseMessage() responseMessage {
+	var ret responseMessage
+	ret[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(ret[1:], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *responseMessage) Header() []byte  { return m[:headerLen] }
+func (m *responseMessage) Payload() []byte { return m[headerLen:] }
+
+func (m *responseMessage) Type() byte  { return m[0] }
+func (m *responseMessage) Length() int { return int(binary.BigEndian.Uint16(m[1:3])) }
+
+func (m *responseMessage) EphemeralPub() []byte { return m[headerLen : headerLen+32] }
+func (m *responseMessage) Tag() []byte          { return m[headerLen+32:] }

control/noise/noiseexplorer_test.go

@@ -0,0 +1,475 @@
+// This file contains the implementation of Noise IK from
+// https://noiseexplorer.com/ . Unlike the rest of this repository,
+// this file is licensed under the terms of the GNU GPL v3. See
+// https://source.symbolic.software/noiseexplorer/noiseexplorer for
+// more information.
+//
+// This file is used here to verify that Tailscale's implementation of
+// Noise IK is interoperable with another implementation.
+//lint:file-ignore SA4006 not our code.
+
+/*
+IK:
+  <- s
+  ...
+  -> e, es, s, ss
+  <- e, ee, se
+  ->
+  <-
+*/
+
+// Implementation Version: 1.0.2
+
+/* ---------------------------------------------------------------- *
+ * PARAMETERS                                                       *
+ * ---------------------------------------------------------------- */
+
+package noise
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/binary"
+	"hash"
+	"io"
+	"math"
+
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+)
+
+/* ---------------------------------------------------------------- *
+ * TYPES                                                            *
+ * ---------------------------------------------------------------- */
+
+type keypair struct {
+	public_key  [32]byte
+	private_key [32]byte
+}
+
+type messagebuffer struct {
+	ne         [32]byte
+	ns         []byte
+	ciphertext []byte
+}
+
+type cipherstate struct {
+	k [32]byte
+	n uint32
+}
+
+type symmetricstate struct {
+	cs cipherstate
+	ck [32]byte
+	h  [32]byte
+}
+
+type handshakestate struct {
+	ss  symmetricstate
+	s   keypair
+	e   keypair
+	rs  [32]byte
+	re  [32]byte
+	psk [32]byte
+}
+
+type noisesession struct {
+	hs  handshakestate
+	h   [32]byte
+	cs1 cipherstate
+	cs2 cipherstate
+	mc  uint64
+	i   bool
+}
+
+/* ---------------------------------------------------------------- *
+ * CONSTANTS                                                        *
+ * ---------------------------------------------------------------- */
+
+var emptyKey = [32]byte{
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+}
+
+var minNonce = uint32(0)
+
+/* ---------------------------------------------------------------- *
+ * UTILITY FUNCTIONS                                                *
+ * ---------------------------------------------------------------- */
+
+func getPublicKey(kp *keypair) [32]byte {
+	return kp.public_key
+}
+
+func isEmptyKey(k [32]byte) bool {
+	return subtle.ConstantTimeCompare(k[:], emptyKey[:]) == 1
+}
+
+func validatePublicKey(k []byte) bool {
+	forbiddenCurveValues := [12][]byte{
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{224, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 0},
+		{95, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 87},
+		{236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{237, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{205, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 128},
+		{76, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 215},
+		{217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{218, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{219, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25},
+	}
+
+	for _, testValue := range forbiddenCurveValues {
+		if subtle.ConstantTimeCompare(k[:], testValue[:]) == 1 {
+			panic("Invalid public key")
+		}
+	}
+	return true
+}
+
+/* ---------------------------------------------------------------- *
+ * PRIMITIVES                                                       *
+ * ---------------------------------------------------------------- */
+
+func incrementNonce(n uint32) uint32 {
+	return n + 1
+}
+
+func dh(private_key [32]byte, public_key [32]byte) [32]byte {
+	var ss [32]byte
+	curve25519.ScalarMult(&ss, &private_key, &public_key)
+	return ss
+}
+
+func generateKeypair() keypair {
+	var public_key [32]byte
+	var private_key [32]byte
+	_, _ = rand.Read(private_key[:])
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	if validatePublicKey(public_key[:]) {
+		return keypair{public_key, private_key}
+	}
+	return generateKeypair()
+}
+
+func generatePublicKey(private_key [32]byte) [32]byte {
+	var public_key [32]byte
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	return public_key
+}
+
+func encrypt(k [32]byte, n uint32, ad []byte, plaintext []byte) []byte {
+	var nonce [12]byte
+	var ciphertext []byte
+	enc, _ := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	ciphertext = enc.Seal(nil, nonce[:], plaintext, ad)
+	return ciphertext
+}
+
+func decrypt(k [32]byte, n uint32, ad []byte, ciphertext []byte) (bool, []byte, []byte) {
+	var nonce [12]byte
+	var plaintext []byte
+	enc, err := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	plaintext, err = enc.Open(nil, nonce[:], ciphertext, ad)
+	return (err == nil), ad, plaintext
+}
+
+func getHash(a []byte, b []byte) [32]byte {
+	return blake2s.Sum256(append(a, b...))
+}
+
+func hashProtocolName(protocolName []byte) [32]byte {
+	var h [32]byte
+	if len(protocolName) <= 32 {
+		copy(h[:], protocolName)
+	} else {
+		h = getHash(protocolName, []byte{})
+	}
+	return h
+}
+
+func blake2HkdfInterface() hash.Hash {
+	h, _ := blake2s.New256([]byte{})
+	return h
+}
+
+func getHkdf(ck [32]byte, ikm []byte) ([32]byte, [32]byte, [32]byte) {
+	var k1 [32]byte
+	var k2 [32]byte
+	var k3 [32]byte
+	output := hkdf.New(blake2HkdfInterface, ikm[:], ck[:], []byte{})
+	io.ReadFull(output, k1[:])
+	io.ReadFull(output, k2[:])
+	io.ReadFull(output, k3[:])
+	return k1, k2, k3
+}
+
+/* ---------------------------------------------------------------- *
+ * STATE MANAGEMENT                                                 *
+ * ---------------------------------------------------------------- */
+
+/* CipherState */
+func initializeKey(k [32]byte) cipherstate {
+	return cipherstate{k, minNonce}
+}
+
+func hasKey(cs *cipherstate) bool {
+	return !isEmptyKey(cs.k)
+}
+
+func setNonce(cs *cipherstate, newNonce uint32) *cipherstate {
+	cs.n = newNonce
+	return cs
+}
+
+func encryptWithAd(cs *cipherstate, ad []byte, plaintext []byte) (*cipherstate, []byte) {
+	e := encrypt(cs.k, cs.n, ad, plaintext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, e
+}
+
+func decryptWithAd(cs *cipherstate, ad []byte, ciphertext []byte) (*cipherstate, []byte, bool) {
+	valid, ad, plaintext := decrypt(cs.k, cs.n, ad, ciphertext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, plaintext, valid
+}
+
+func reKey(cs *cipherstate) *cipherstate {
+	e := encrypt(cs.k, math.MaxUint32, []byte{}, emptyKey[:])
+	copy(cs.k[:], e)
+	return cs
+}
+
+/* SymmetricState */
+
+func initializeSymmetric(protocolName []byte) symmetricstate {
+	h := hashProtocolName(protocolName)
+	ck := h
+	cs := initializeKey(emptyKey)
+	return symmetricstate{cs, ck, h}
+}
+
+func mixKey(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	ck, tempK, _ := getHkdf(ss.ck, ikm[:])
+	ss.cs = initializeKey(tempK)
+	ss.ck = ck
+	return ss
+}
+
+func mixHash(ss *symmetricstate, data []byte) *symmetricstate {
+	ss.h = getHash(ss.h[:], data)
+	return ss
+}
+
+func mixKeyAndHash(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	var tempH [32]byte
+	var tempK [32]byte
+	ss.ck, tempH, tempK = getHkdf(ss.ck, ikm[:])
+	ss = mixHash(ss, tempH[:])
+	ss.cs = initializeKey(tempK)
+	return ss
+}
+
+func getHandshakeHash(ss *symmetricstate) [32]byte {
+	return ss.h
+}
+
+func encryptAndHash(ss *symmetricstate, plaintext []byte) (*symmetricstate, []byte) {
+	var ciphertext []byte
+	if hasKey(&ss.cs) {
+		_, ciphertext = encryptWithAd(&ss.cs, ss.h[:], plaintext)
+	} else {
+		ciphertext = plaintext
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, ciphertext
+}
+
+func decryptAndHash(ss *symmetricstate, ciphertext []byte) (*symmetricstate, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if hasKey(&ss.cs) {
+		_, plaintext, valid = decryptWithAd(&ss.cs, ss.h[:], ciphertext)
+	} else {
+		plaintext, valid = ciphertext, true
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, plaintext, valid
+}
+
+func split(ss *symmetricstate) (cipherstate, cipherstate) {
+	tempK1, tempK2, _ := getHkdf(ss.ck, []byte{})
+	cs1 := initializeKey(tempK1)
+	cs2 := initializeKey(tempK2)
+	return cs1, cs2
+}
+
+/* HandshakeState */
+
+func initializeInitiator(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, rs[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func initializeResponder(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, s.public_key[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func writeMessageA(hs *handshakestate, payload []byte) (*handshakestate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	spk := make([]byte, len(hs.s.public_key))
+	copy(spk[:], hs.s.public_key[:])
+	_, ns = encryptAndHash(&hs.ss, spk)
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return hs, messageBuffer
+}
+
+func writeMessageB(hs *handshakestate, payload []byte) ([32]byte, messagebuffer, cipherstate, cipherstate) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, messageBuffer, cs1, cs2
+}
+
+func writeMessageRegular(cs *cipherstate, payload []byte) (*cipherstate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	cs, ciphertext = encryptWithAd(cs, []byte{}, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return cs, messageBuffer
+}
+
+func readMessageA(hs *handshakestate, message *messagebuffer) (*handshakestate, []byte, bool) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, ns, valid1 := decryptAndHash(&hs.ss, message.ns)
+	if valid1 && len(ns) == 32 && validatePublicKey(message.ns[:]) {
+		copy(hs.rs[:], ns)
+	}
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	return hs, plaintext, (valid1 && valid2)
+}
+
+func readMessageB(hs *handshakestate, message *messagebuffer) ([32]byte, []byte, bool, cipherstate, cipherstate) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, plaintext, (valid1 && valid2), cs1, cs2
+}
+
+func readMessageRegular(cs *cipherstate, message *messagebuffer) (*cipherstate, []byte, bool) {
+	/* No encrypted keys */
+	_, plaintext, valid2 := decryptWithAd(cs, []byte{}, message.ciphertext)
+	return cs, plaintext, valid2
+}
+
+/* ---------------------------------------------------------------- *
+ * PROCESSES                                                        *
+ * ---------------------------------------------------------------- */
+
+func InitSession(initiator bool, prologue []byte, s keypair, rs [32]byte) noisesession {
+	var session noisesession
+	psk := emptyKey
+	if initiator {
+		session.hs = initializeInitiator(prologue, s, rs, psk)
+	} else {
+		session.hs = initializeResponder(prologue, s, rs, psk)
+	}
+	session.i = initiator
+	session.mc = 0
+	return session
+}
+
+func SendMessage(session *noisesession, message []byte) (*noisesession, messagebuffer) {
+	var messageBuffer messagebuffer
+	if session.mc == 0 {
+		_, messageBuffer = writeMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, messageBuffer, session.cs1, session.cs2 = writeMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, messageBuffer = writeMessageRegular(&session.cs1, message)
+		} else {
+			_, messageBuffer = writeMessageRegular(&session.cs2, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, messageBuffer
+}
+
+func RecvMessage(session *noisesession, message *messagebuffer) (*noisesession, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if session.mc == 0 {
+		_, plaintext, valid = readMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, plaintext, valid, session.cs1, session.cs2 = readMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, plaintext, valid = readMessageRegular(&session.cs2, message)
+		} else {
+			_, plaintext, valid = readMessageRegular(&session.cs1, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, plaintext, valid
+}
+
+func main() {}

derp/derp_client.go

@@ -210,12 +210,12 @@ func (c *Client) send(dstKey key.NodePublic, pkt []byte) (ret error) {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
 	if c.rate != nil {
-		pktLen := frameHeaderLen + dstKey.RawLen() + len(pkt)
+		pktLen := frameHeaderLen + key.NodePublicRawLen + len(pkt)
 		if !c.rate.AllowN(time.Now(), pktLen) {
 			return nil // drop
 		}
 	}
-	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(dstKey.RawLen()+len(pkt))); err != nil {
+	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(key.NodePublicRawLen+len(pkt))); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {

derp/derp_server.go

@@ -1017,7 +1017,7 @@ func (s *Server) verifyClient(clientKey key.NodePublic, info *clientInfo) error
 }
 
 func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
-	buf := make([]byte, 0, len(magic)+s.publicKey.RawLen())
+	buf := make([]byte, 0, len(magic)+key.NodePublicRawLen)
 	buf = append(buf, magic...)
 	buf = s.publicKey.AppendTo(buf)
 	err := writeFrame(lw.bw(), frameServerKey, buf)
@@ -1469,7 +1469,7 @@ func (c *sclient) sendPacket(srcKey key.NodePublic, contents []byte) (err error)
 	withKey := !srcKey.IsZero()
 	pktLen := len(contents)
 	if withKey {
-		pktLen += srcKey.RawLen()
+		pktLen += key.NodePublicRawLen
 	}
 	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err

disco/disco.go

@@ -25,8 +25,9 @@ import (
 	"fmt"
 	"net"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 // Magic is the 6 byte header of all discovery messages.
@@ -115,19 +116,19 @@ type Ping struct {
 	// It shouldn't be trusted by itself, but can be combined with
 	// netmap data to reduce the discokey:nodekey relation from 1:N to
 	// 1:1.
-	NodeKey tailcfg.NodeKey
+	NodeKey key.NodePublic
 }
 
 func (m *Ping) AppendMarshal(b []byte) []byte {
 	dataLen := 12
 	hasKey := !m.NodeKey.IsZero()
 	if hasKey {
-		dataLen += len(m.NodeKey)
+		dataLen += key.NodePublicRawLen
 	}
 	ret, d := appendMsgHeader(b, TypePing, v0, dataLen)
 	n := copy(d, m.TxID[:])
 	if hasKey {
-		copy(d[n:], m.NodeKey[:])
+		m.NodeKey.AppendTo(d[:n])
 	}
 	return ret
 }
@@ -138,8 +139,10 @@ func parsePing(ver uint8, p []byte) (m *Ping, err error) {
 	}
 	m = new(Ping)
 	p = p[copy(m.TxID[:], p):]
-	if len(p) >= len(m.NodeKey) {
-		copy(m.NodeKey[:], p)
+	// Deliberately lax on longer-than-expected messages, for future
+	// compatibility.
+	if len(p) >= key.NodePublicRawLen {
+		m.NodeKey = key.NodePublicFromRaw32(mem.B(p[:key.NodePublicRawLen]))
 	}
 	return m, nil
 }

disco/disco_test.go

@@ -10,8 +10,9 @@ import (
 	"strings"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 func TestMarshalAndParse(t *testing.T) {
@@ -30,13 +31,8 @@ func TestMarshalAndParse(t *testing.T) {
 		{
 			name: "ping_with_nodekey_src",
 			m: &Ping{
-				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
-				NodeKey: tailcfg.NodeKey{
-					1:  1,
-					2:  2,
-					30: 30,
-					31: 31,
-				},
+				TxID:    [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				NodeKey: key.NodePublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})),
 			},
 			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f",
 		},

docs/k8s/README.md

@@ -32,7 +32,7 @@ There are quite a few ways of running Tailscale inside a Kubernetes Cluster, som
    ```
 
 ### Sample Sidecar
-Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
+Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivity between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
 
 1. Create and login to the sample nginx pod with a Tailscale sidecar
 
@@ -144,4 +144,4 @@ routes for the subnet-router are enabled.
    # INTERNAL_IP="$(kubectl get po <POD_NAME> -o=jsonpath='{.status.podIP}')" 
    INTERNAL_PORT=8080 
    curl http://$INTERNAL_IP:$INTERNAL_PORT
-   ```
+   ```

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	filippo.io/mkcert v1.4.3
+	github.com/akutz/memconn v0.1.0
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
@@ -13,9 +14,8 @@ require (
 	github.com/coreos/go-iptables v0.6.0
 	github.com/creack/pty v1.1.17
 	github.com/dave/jennifer v1.4.1
-	github.com/frankban/quicktest v1.13.1
+	github.com/frankban/quicktest v1.14.0
 	github.com/gliderlabs/ssh v0.3.3
-	github.com/go-multierror/multierror v1.0.2
 	github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1
 	github.com/godbus/dbus/v5 v5.0.5
 	github.com/google/go-cmp v0.5.6
@@ -39,23 +39,24 @@ require (
 	github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
 	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20211020060615-d418f374d309
+	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
+	golang.org/x/net v0.0.0-20211111083644-e5c967477495
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211020174200-9d6173849985
+	golang.org/x/sys v0.0.0-20211110154304-99a53858aa08
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.7
-	golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2
+	golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45
 	golang.zx2c4.com/wireguard/windows v0.4.10
 	honnef.co/go/tools v0.2.1
 	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
-	inet.af/netstack v0.0.0-20211027215559-ec21145de76b
+	inet.af/netstack v0.0.0-20211101182044-1c1bcf452982
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	nhooyr.io/websocket v1.8.7
@@ -192,13 +193,14 @@ require (
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.4 // indirect
 	github.com/uudashr/gocognit v1.0.1 // indirect
-	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
+	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -39,6 +39,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A=
+github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
@@ -127,14 +129,15 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8=
-github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
+github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
@@ -156,16 +159,16 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-multierror/multierror v1.0.2 h1:AwsKbEXkmf49ajdFJgcFXqSG0aLo0HEyAE9zk9JguJo=
-github.com/go-multierror/multierror v1.0.2/go.mod h1:U7SZR/D9jHgt2nkSj8XcbCWdmVM2igraCHQ3HC1HiKY=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1 h1:4dntyT+x6QTOSCIrgczbQ+ockAEha0cfxD5Wi0iCzjY=
 github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -194,8 +197,10 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.5 h1:9Eg0XUhQxtkV8ykTMKtMMYY72g4NgxtRq4jgh4Ih5YM=
@@ -484,6 +489,8 @@ github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -643,6 +650,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -666,6 +675,7 @@ github.com/u-root/uio v0.0.0-20210528114334-82958018845c h1:BFvcl34IGnw8yvJi8hlq
 github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
@@ -681,10 +691,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
 github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6 h1:167a2omrzz+nN9Of6lN/0yOB9itzw+IOioRThNZ30jA=
+github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
@@ -729,8 +739,9 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -798,8 +809,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211111083644-e5c967477495 h1:cjxxlQm6d4kYbhpZ2ghvmI8xnq0AG+jXmzrhzfkyu5A=
+golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -836,7 +848,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -847,11 +858,13 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -877,8 +890,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
-golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w=
@@ -958,9 +972,15 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 h1:Ug9qvr1myri/zFN6xL17LSCBGFDnphBBhzmILHsM5TY=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20210905140043-2ef39d47540c/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
-golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2 h1:mHJssZsxXvTmTP+sxkGZCItVGhaOWo0UnFqrM2lMqOk=
-golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2/go.mod h1:RTjaYEQboNk7+2qfPGBotaMEh/5HIvmPZ6DIe10lTqI=
+golang.zx2c4.com/wireguard v0.0.0-20211115224047-111e0566dce3 h1:7BFThRTwBwTLoMomQ/Y0GqY1VLH9D7kbbTNsfxl2fU0=
+golang.zx2c4.com/wireguard v0.0.0-20211115224047-111e0566dce3/go.mod h1:evxZIqfCetExY5piKXGAxJYwvXWkps9zTCkWpkoGFxw=
+golang.zx2c4.com/wireguard v0.0.0-20211116194326-3cae233d69f7 h1:ZeHUKruJlkbSvafSH7GrDzMDXf7+/0T5sEKE8A9rEiE=
+golang.zx2c4.com/wireguard v0.0.0-20211116194326-3cae233d69f7/go.mod h1:evxZIqfCetExY5piKXGAxJYwvXWkps9zTCkWpkoGFxw=
+golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45 h1:mEVhdMPTuebD9IUXOUB5Q2sjZpcmzkahHWd6DrGpLHA=
+golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45/go.mod h1:evxZIqfCetExY5piKXGAxJYwvXWkps9zTCkWpkoGFxw=
 golang.zx2c4.com/wireguard/windows v0.4.10 h1:HmjzJnb+G4NCdX+sfjsQlsxGPuYaThxRbZUZFLyR0/s=
 golang.zx2c4.com/wireguard/windows v0.4.10/go.mod h1:v7w/8FC48tTBm1IzScDVPEEb0/GjLta+T0ybpP9UWRg=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1036,8 +1056,8 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCU
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw=
 inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8=
-inet.af/netstack v0.0.0-20211027215559-ec21145de76b h1:5aGmvztDCPhMYl/pByQYY0eaqbDzqTtX28180pffKqw=
-inet.af/netstack v0.0.0-20211027215559-ec21145de76b/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
+inet.af/netstack v0.0.0-20211101182044-1c1bcf452982 h1:hYciifHEv98/p8ln52ybKhgQpGouZWALFxxFE65RVdU=
+inet.af/netstack v0.0.0-20211101182044-1c1bcf452982/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU=
 inet.af/wf v0.0.0-20210516214145-a5343001b756 h1:muIT3C1rH3/xpvIH8blKkMvhctV7F+OtZqs7kcwHDBQ=

health/health.go

@@ -16,8 +16,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/multierr"
 )
 
 var (
@@ -268,7 +268,7 @@ func selfCheckLocked() {
 // OverallError returns a summary of the health state.
 //
 // If there are multiple problems, the error will be of type
-// multierror.MultipleErrors.
+// multierr.Error.
 func OverallError() error {
 	mu.Lock()
 	defer mu.Unlock()
@@ -337,7 +337,7 @@ func overallErrorLocked() error {
 		// Not super efficient (stringifying these in a sort), but probably max 2 or 3 items.
 		return errs[i].Error() < errs[j].Error()
 	})
-	return multierror.New(errs)
+	return multierr.New(errs...)
 }
 
 var (

hostinfo/hostinfo.go

@@ -7,11 +7,14 @@
 package hostinfo
 
 import (
+	"bufio"
 	"io"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync/atomic"
+	"time"
 
 	"go4.org/mem"
 	"tailscale.com/tailcfg"
@@ -224,3 +227,55 @@ func inKubernetes() bool {
 	}
 	return false
 }
+
+type etcAptSrcResult struct {
+	mod      time.Time
+	disabled bool
+}
+
+var etcAptSrcCache atomic.Value // of etcAptSrcResult
+
+// DisabledEtcAptSource reports whether Ubuntu (or similar) has disabled
+// the /etc/apt/sources.list.d/tailscale.list file contents upon upgrade
+// to a new release of the distro.
+//
+// See https://github.com/tailscale/tailscale/issues/3177
+func DisabledEtcAptSource() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	const path = "/etc/apt/sources.list.d/tailscale.list"
+	fi, err := os.Stat(path)
+	if err != nil || !fi.Mode().IsRegular() {
+		return false
+	}
+	mod := fi.ModTime()
+	if c, ok := etcAptSrcCache.Load().(etcAptSrcResult); ok && c.mod == mod {
+		return c.disabled
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	v := etcAptSourceFileIsDisabled(f)
+	etcAptSrcCache.Store(etcAptSrcResult{mod: mod, disabled: v})
+	return v
+}
+
+func etcAptSourceFileIsDisabled(r io.Reader) bool {
+	bs := bufio.NewScanner(r)
+	disabled := false // did we find the "disabled on upgrade" comment?
+	for bs.Scan() {
+		line := strings.TrimSpace(bs.Text())
+		if strings.Contains(line, "# disabled on upgrade") {
+			disabled = true
+		}
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		// Well, it has some contents in it at least.
+		return false
+	}
+	return disabled
+}

hostinfo/hostinfo_test.go

@@ -6,6 +6,7 @@ package hostinfo
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 )
 
@@ -27,3 +28,25 @@ func TestOSVersion(t *testing.T) {
 	}
 	t.Logf("Got: %#q", osVersion())
 }
+
+func TestEtcAptSourceFileIsDisabled(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want bool
+	}{
+		{"empty", "", false},
+		{"normal", "deb foo\n", false},
+		{"normal-commented", "# deb foo\n", false},
+		{"normal-disabled-by-ubuntu", "# deb foo # disabled on upgrade to dingus\n", true},
+		{"normal-disabled-then-uncommented", "deb foo # disabled on upgrade to dingus\n", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := etcAptSourceFileIsDisabled(strings.NewReader(tt.in))
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

ipn/backend.go

@@ -12,6 +12,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/structs"
 )
@@ -48,7 +49,7 @@ type EngineStatus struct {
 	RBytes, WBytes int64
 	NumLive        int
 	LiveDERPs      int // number of active DERP connections
-	LivePeers      map[tailcfg.NodeKey]ipnstate.PeerStatusLite
+	LivePeers      map[key.NodePublic]ipnstate.PeerStatusLite
 }
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend

ipn/ipnlocal/local.go

@@ -25,8 +25,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
@@ -50,6 +48,7 @@ import (
 	"tailscale.com/types/preftype"
 	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
@@ -97,6 +96,7 @@ type LocalBackend struct {
 	gotPortPollRes        chan struct{}    // closed upon first readPoller result
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
+	varRoot               string // or empty if SetVarRoot never called
 
 	filterHash deephash.Sum
 
@@ -122,6 +122,7 @@ type LocalBackend struct {
 	engineStatus     ipn.EngineStatus
 	endpoints        []tailcfg.Endpoint
 	blocked          bool
+	keyExpired       bool
 	authURL          string // cleared on Notify
 	authURLSticky    string // not cleared on Notify
 	interact         bool
@@ -335,8 +336,8 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 
 		if err := health.OverallError(); err != nil {
 			switch e := err.(type) {
-			case multierror.MultipleErrors:
-				for _, err := range e {
+			case multierr.Error:
+				for _, err := range e.Errors() {
 					s.Health = append(s.Health, err.Error())
 				}
 			default:
@@ -389,7 +390,7 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 				tailscaleIPs = append(tailscaleIPs, addr.IP())
 			}
 		}
-		sb.AddPeer(key.NodePublicFromRaw32(mem.B(p.Key[:])), &ipnstate.PeerStatus{
+		sb.AddPeer(p.Key, &ipnstate.PeerStatus{
 			InNetworkMap:       true,
 			ID:                 p.StableID,
 			UserID:             p.User,
@@ -452,18 +453,35 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		// TODO(crawshaw): display in the UI.
 		if errors.Is(st.Err, io.EOF) {
 			b.logf("[v1] Received error: EOF")
-		} else {
-			b.logf("Received error: %v", st.Err)
-			e := st.Err.Error()
-			b.send(ipn.Notify{ErrMessage: &e})
+			return
+		}
+		b.logf("Received error: %v", st.Err)
+		var uerr controlclient.UserVisibleError
+		if errors.As(st.Err, &uerr) {
+			s := uerr.UserVisibleError()
+			b.send(ipn.Notify{ErrMessage: &s})
 		}
 		return
 	}
 
 	b.mu.Lock()
 	wasBlocked := b.blocked
+	keyExpiryExtended := false
+	if st.NetMap != nil {
+		wasExpired := b.keyExpired
+		isExpired := !st.NetMap.Expiry.IsZero() && st.NetMap.Expiry.Before(time.Now())
+		if wasExpired && !isExpired {
+			keyExpiryExtended = true
+		}
+		b.keyExpired = isExpired
+	}
 	b.mu.Unlock()
 
+	if keyExpiryExtended && wasBlocked {
+		// Key extended, unblock the engine
+		b.blockEngineUpdates(false)
+	}
+
 	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
@@ -847,7 +865,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		})
 	}
 
-	var discoPublic tailcfg.DiscoKey
+	var discoPublic key.DiscoPublic
 	if controlclient.Debug.Disco {
 		discoPublic = b.e.DiscoPublicKey()
 	}
@@ -1562,7 +1580,7 @@ func (b *LocalBackend) parseWgStatusLocked(s *wgengine.Status) (ret ipn.EngineSt
 	var peerStats, peerKeys strings.Builder
 
 	ret.LiveDERPs = s.DERPs
-	ret.LivePeers = map[tailcfg.NodeKey]ipnstate.PeerStatusLite{}
+	ret.LivePeers = map[key.NodePublic]ipnstate.PeerStatusLite{}
 	for _, p := range s.Peers {
 		if !p.LastHandshake.IsZero() {
 			fmt.Fprintf(&peerStats, "%d/%d ", p.RxBytes, p.TxBytes)
@@ -1771,6 +1789,12 @@ func (b *LocalBackend) NetMap() *netmap.NetworkMap {
 	return b.netMap
 }
 
+func (b *LocalBackend) isEngineBlocked() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.blocked
+}
+
 // blockEngineUpdate sets b.blocked to block, while holding b.mu. Its
 // indirect effect is to turn b.authReconfig() into a no-op if block
 // is true.
@@ -1999,34 +2023,29 @@ func normalizeResolver(cfg dnstype.Resolver) dnstype.Resolver {
 	return cfg
 }
 
+// SetVarRoot sets the root directory of Tailscale's writable
+// storage area . (e.g. "/var/lib/tailscale")
+//
+// It should only be called before the LocalBackend is used.
+func (b *LocalBackend) SetVarRoot(dir string) {
+	b.varRoot = dir
+}
+
 // TailscaleVarRoot returns the root directory of Tailscale's writable
 // storage area. (e.g. "/var/lib/tailscale")
 //
 // It returns an empty string if there's no configured or discovered
 // location.
 func (b *LocalBackend) TailscaleVarRoot() string {
+	if b.varRoot != "" {
+		return b.varRoot
+	}
 	switch runtime.GOOS {
 	case "ios", "android":
 		dir, _ := paths.AppSharedDir.Load().(string)
 		return dir
 	}
-	// Temporary (2021-09-27) transitional fix for #2927 (Synology
-	// cert dir) on the way towards a more complete fix
-	// (#2932). It fixes any case where the state file is provided
-	// to tailscaled explicitly when it's not in the default
-	// location.
-	if fs, ok := b.store.(*ipn.FileStore); ok {
-		if fp := fs.Path(); fp != "" {
-			if dir := filepath.Dir(fp); strings.EqualFold(filepath.Base(dir), "tailscale") {
-				return dir
-			}
-		}
-	}
-	stateFile := paths.DefaultTailscaledStateFile()
-	if stateFile == "" {
-		return ""
-	}
-	return filepath.Dir(stateFile)
+	return ""
 }
 
 func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
@@ -2398,6 +2417,7 @@ func (b *LocalBackend) nextState() ipn.State {
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 		st          = b.engineStatus
+		keyExpired  = b.keyExpired
 	)
 	b.mu.Unlock()
 
@@ -2430,7 +2450,9 @@ func (b *LocalBackend) nextState() ipn.State {
 		}
 	case !wantRunning:
 		return ipn.Stopped
-	case !netMap.Expiry.IsZero() && time.Until(netMap.Expiry) <= 0:
+	case keyExpired:
+		// NetMap must be non-nil for us to get here.
+		// The node key expired, need to relogin.
 		return ipn.NeedsLogin
 	case netMap.MachineStatus != tailcfg.MachineAuthorized:
 		// TODO(crawshaw): handle tailcfg.MachineInvalid
@@ -2511,6 +2533,7 @@ func (b *LocalBackend) ResetForClientDisconnect() {
 	b.userID = ""
 	b.setNetMapLocked(nil)
 	b.prefs = new(ipn.Prefs)
+	b.keyExpired = false
 	b.authURL = ""
 	b.authURLSticky = ""
 	b.activeLogin = ""
@@ -2680,7 +2703,7 @@ func (b *LocalBackend) OperatorUserID() string {
 // TestOnlyPublicKeys returns the current machine and node public
 // keys. Used in tests only to facilitate automated node authorization
 // in the test harness.
-func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey tailcfg.NodeKey) {
+func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey key.NodePublic) {
 	b.mu.Lock()
 	prefs := b.prefs
 	machinePrivKey := b.machinePrivKey
@@ -2692,7 +2715,7 @@ func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeK
 
 	mk := machinePrivKey.Public()
 	nk := prefs.Persist.PrivateNodeKey.Public()
-	return mk, tailcfg.NodeKeyFromNodePublic(nk)
+	return mk, nk
 }
 
 func (b *LocalBackend) WaitingFiles() ([]apitype.WaitingFile, error) {
@@ -2782,7 +2805,7 @@ func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
 	b.mu.Lock()
 	cc := b.cc
 	if prefs := b.prefs; prefs != nil {
-		req.NodeKey = tailcfg.NodeKeyFromNodePublic(prefs.Persist.PrivateNodeKey.Public())
+		req.NodeKey = prefs.Persist.PrivateNodeKey.Public()
 	}
 	b.mu.Unlock()
 	if cc == nil {

ipn/ipnlocal/loglines_test.go

@@ -90,7 +90,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()
@@ -105,7 +105,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       11,
 			RxBytes:       12,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()

ipn/ipnlocal/peerapi.go

@@ -6,6 +6,7 @@ package ipnlocal
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -29,11 +30,13 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine"
 )
 
@@ -500,9 +503,16 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handlePeerPut(w, r)
 		return
 	}
-	if r.URL.Path == "/v0/goroutines" {
+	switch r.URL.Path {
+	case "/v0/goroutines":
 		h.handleServeGoroutines(w, r)
 		return
+	case "/v0/env":
+		h.handleServeEnv(w, r)
+		return
+	case "/v0/metrics":
+		h.handleServeMetrics(w, r)
+		return
 	}
 	who := h.peerUser.DisplayName
 	fmt.Fprintf(w, `<html>
@@ -710,3 +720,32 @@ func (h *peerAPIHandler) handleServeGoroutines(w http.ResponseWriter, r *http.Re
 	}
 	w.Write(buf)
 }
+
+func (h *peerAPIHandler) handleServeEnv(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	var data struct {
+		Hostinfo *tailcfg.Hostinfo
+		Uid      int
+		Args     []string
+		Env      []string
+	}
+	data.Hostinfo = hostinfo.New()
+	data.Uid = os.Getuid()
+	data.Args = os.Args
+	data.Env = os.Environ()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}
+
+func (h *peerAPIHandler) handleServeMetrics(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}

ipn/ipnlocal/state_test.go

@@ -867,6 +867,45 @@ func TestStateMachine(t *testing.T) {
 		// change either.
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
+	t.Logf("\n\nExpireKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(-time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsTrue)
+	}
+
+	t.Logf("\n\nExtendKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Starting, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsFalse)
+	}
+	notifies.expect(1)
+	// Fake a DERP connection.
+	b.setWgengineStatus(&wgengine.Status{DERPs: 1}, nil)
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Running, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Running, qt.Equals, b.State())
+	}
 }
 
 type testStateStorage struct {

ipn/ipnserver/server.go

@@ -52,16 +52,13 @@ import (
 
 // Options is the configuration of the Tailscale node agent.
 type Options struct {
-	// SocketPath, on unix systems, is the unix socket path to listen
-	// on for frontend connections.
-	SocketPath string
-
-	// Port, on windows, is the localhost TCP port to listen on for
-	// frontend connections.
-	Port int
-
-	// StatePath is the path to the stored agent state.
-	StatePath string
+	// VarRoot is the the Tailscale daemon's private writable
+	// directory (usually "/var/lib/tailscale" on Linux) that
+	// contains the "tailscaled.state" file, the "certs" directory
+	// for TLS certs, and the "files" directory for incoming
+	// Taildrop files before they're moved to a user directory.
+	// If empty, Taildrop and TLS certs don't function.
+	VarRoot string
 
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
@@ -83,10 +80,6 @@ type Options struct {
 	// the actual definition of "disconnect" is when the
 	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
-
-	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
-	// to register a debug handler.
-	DebugMux *http.ServeMux
 }
 
 // Server is an IPN backend and its set of 0 or more active localhost
@@ -100,8 +93,8 @@ type Server struct {
 	// being run in "client mode" that requires an active GUI
 	// connection (such as on Windows by default).  Even if this
 	// is true, the ForceDaemon pref can override this.
-	resetOnZero bool
-	opts        Options
+	resetOnZero       bool
+	autostartStateKey ipn.StateKey
 
 	bsMu sync.Mutex // lock order: bsMu, then mu
 	bs   *ipn.BackendServer
@@ -114,6 +107,9 @@ type Server struct {
 	disconnectSub  map[chan<- struct{}]struct{} // keys are subscribers of disconnects
 }
 
+// LocalBackend returns the server's LocalBackend.
+func (s *Server) LocalBackend() *ipnlocal.LocalBackend { return s.b }
+
 // connIdentity represents the owner of a localhost TCP or unix socket connection.
 type connIdentity struct {
 	Conn       net.Conn
@@ -414,13 +410,16 @@ func (s *Server) checkConnIdentityLocked(ci connIdentity) error {
 //
 // s.mu must not be held.
 func (s *Server) localAPIPermissions(ci connIdentity) (read, write bool) {
-	if runtime.GOOS == "windows" {
+	switch runtime.GOOS {
+	case "windows":
 		s.mu.Lock()
 		defer s.mu.Unlock()
 		if s.checkConnIdentityLocked(ci) == nil {
 			return true, true
 		}
 		return false, false
+	case "js":
+		return true, true
 	}
 	if ci.IsUnixSock {
 		return true, !isReadonlyConn(ci, s.b.OperatorUserID(), logger.Discard)
@@ -606,18 +605,57 @@ func tryWindowsAppDataMigration(logf logger.Logf, path string) string {
 	return paths.TryConfigFileMigration(logf, oldFile, path)
 }
 
+// StateStore returns a StateStore from path.
+//
+// The path should be an absolute path to a file.
+//
+// Special cases:
+//
+//   * empty string means to use an in-memory store
+//   * if the string begins with "kube:", the suffix
+//     is a Kubernetes secret name
+//   * if the string begins with "arn:", the value is
+//     an AWS ARN for an SSM.
+func StateStore(path string, logf logger.Logf) (ipn.StateStore, error) {
+	if path == "" {
+		return &ipn.MemoryStore{}, nil
+	}
+	const kubePrefix = "kube:"
+	const arnPrefix = "arn:"
+	switch {
+	case strings.HasPrefix(path, kubePrefix):
+		secretName := strings.TrimPrefix(path, kubePrefix)
+		store, err := ipn.NewKubeStore(secretName)
+		if err != nil {
+			return nil, fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
+		}
+		return store, nil
+	case strings.HasPrefix(path, arnPrefix):
+		store, err := aws.NewStore(path)
+		if err != nil {
+			return nil, fmt.Errorf("aws.NewStore(%q): %v", path, err)
+		}
+		return store, nil
+	}
+	if runtime.GOOS == "windows" {
+		path = tryWindowsAppDataMigration(logf, path)
+	}
+	store, err := ipn.NewFileStore(path)
+	if err != nil {
+		return nil, fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
+	}
+	return store, nil
+}
+
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
-func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+//
+// Deprecated: use New and Server.Run instead.
+func Run(ctx context.Context, logf logger.Logf, ln net.Listener, store ipn.StateStore, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
 	getEngine = getEngineUntilItWorksWrapper(getEngine)
 	runDone := make(chan struct{})
 	defer close(runDone)
 
-	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
-	if err != nil {
-		return fmt.Errorf("safesocket.Listen: %v", err)
-	}
-
 	var serverMu sync.Mutex
 	var serverOrNil *Server
 
@@ -633,57 +671,28 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			s.stopAll()
 		}
 		serverMu.Unlock()
-		listen.Close()
+		ln.Close()
 	}()
-	logf("Listening on %v", listen.Addr())
+	logf("Listening on %v", ln.Addr())
 
 	var serverModeUser *user.User
-	var store ipn.StateStore
-	if opts.StatePath != "" {
-		const kubePrefix = "kube:"
-		const arnPrefix = "arn:"
-		path := opts.StatePath
-		switch {
-		case strings.HasPrefix(path, kubePrefix):
-			secretName := strings.TrimPrefix(path, kubePrefix)
-			store, err = ipn.NewKubeStore(secretName)
-			if err != nil {
-				return fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
-			}
-		case strings.HasPrefix(path, arnPrefix):
-			store, err = aws.NewStore(path)
-			if err != nil {
-				return fmt.Errorf("aws.NewStore(%q): %v", path, err)
-			}
-		default:
-			if runtime.GOOS == "windows" {
-				path = tryWindowsAppDataMigration(logf, path)
-			}
-			store, err = ipn.NewFileStore(path)
-			if err != nil {
-				return fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
-			}
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return fmt.Errorf("calling ReadState on state store: %w", err)
 		}
-		if opts.AutostartStateKey == "" {
-			autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
-			if err != nil && err != ipn.ErrStateNotExist {
-				return fmt.Errorf("calling ReadState on %s: %w", path, err)
-			}
-			key := string(autoStartKey)
-			if strings.HasPrefix(key, "user-") {
-				uid := strings.TrimPrefix(key, "user-")
-				u, err := lookupUserFromID(logf, uid)
-				if err != nil {
-					logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
-				} else {
-					logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
-					serverModeUser = u
-				}
-				opts.AutostartStateKey = ipn.StateKey(key)
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
 			}
+			opts.AutostartStateKey = ipn.StateKey(key)
 		}
-	} else {
-		store = &ipn.MemoryStore{}
 	}
 
 	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
@@ -693,7 +702,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	if err != nil {
 		logf("ipnserver: initial getEngine call: %v", err)
 		for i := 1; ctx.Err() == nil; i++ {
-			c, err := listen.Accept()
+			c, err := ln.Accept()
 			if err != nil {
 				logf("%d: Accept: %v", i, err)
 				bo.BackOff(ctx, err)
@@ -720,8 +729,8 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 		}
 	}
 	if unservedConn != nil {
-		listen = &listenerWithReadyConn{
-			Listener: listen,
+		ln = &listenerWithReadyConn{
+			Listener: ln,
 			c:        unservedConn,
 		}
 	}
@@ -733,49 +742,78 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	serverMu.Lock()
 	serverOrNil = server
 	serverMu.Unlock()
-	return server.Serve(ctx, listen)
+	return server.Run(ctx, ln)
 }
 
 // New returns a new Server.
 //
-// The opts.StatePath option is ignored; it's only used by Run.
+// To start it, use the Server.Run method.
 func New(logf logger.Logf, logid string, store ipn.StateStore, eng wgengine.Engine, serverModeUser *user.User, opts Options) (*Server, error) {
 	b, err := ipnlocal.NewLocalBackend(logf, logid, store, eng)
 	if err != nil {
 		return nil, fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	b.SetVarRoot(opts.VarRoot)
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)
 	})
 
-	if opts.DebugMux != nil {
-		opts.DebugMux.HandleFunc("/debug/ipn", func(w http.ResponseWriter, r *http.Request) {
-			serveHTMLStatus(w, b)
-		})
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return nil, fmt.Errorf("calling ReadState on store: %w", err)
+		}
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
+			}
+			opts.AutostartStateKey = ipn.StateKey(key)
+		}
 	}
 
 	server := &Server{
-		b:              b,
-		backendLogID:   logid,
-		logf:           logf,
-		resetOnZero:    !opts.SurviveDisconnects,
-		serverModeUser: serverModeUser,
-		opts:           opts,
+		b:                 b,
+		backendLogID:      logid,
+		logf:              logf,
+		resetOnZero:       !opts.SurviveDisconnects,
+		serverModeUser:    serverModeUser,
+		autostartStateKey: opts.AutostartStateKey,
 	}
 	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 	return server, nil
 }
 
-// Serve accepts connections from ln forever.
+// Run runs the server, accepting connections from ln forever.
 //
-// The context is only used to suppress errors
-func (s *Server) Serve(ctx context.Context, ln net.Listener) error {
+// If the context is done, the listener is closed.
+func (s *Server) Run(ctx context.Context, ln net.Listener) error {
 	defer s.b.Shutdown()
-	if s.opts.AutostartStateKey != "" {
+
+	runDone := make(chan struct{})
+	defer close(runDone)
+
+	// When the context is closed or when we return, whichever is first, close our listener
+	// and all open connections.
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-runDone:
+		}
+		s.stopAll()
+		ln.Close()
+	}()
+
+	if s.autostartStateKey != "" {
 		s.bs.GotCommand(ctx, &ipn.Command{
 			Version: version.Long,
 			Start: &ipn.StartArgs{
-				Opts: ipn.Options{StateKey: s.opts.AutostartStateKey},
+				Opts: ipn.Options{StateKey: s.autostartStateKey},
 			},
 		})
 	}
@@ -1013,13 +1051,13 @@ func (s *Server) localhostHandler(ci connIdentity) http.Handler {
 			io.WriteString(w, "<html><title>Tailscale</title><body><h1>Tailscale</h1>This is the local Tailscale daemon.")
 			return
 		}
-		serveHTMLStatus(w, s.b)
+		s.ServeHTMLStatus(w, r)
 	})
 }
 
-func serveHTMLStatus(w http.ResponseWriter, b *ipnlocal.LocalBackend) {
+func (s *Server) ServeHTMLStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	st := b.Status()
+	st := s.b.Status()
 	// TODO(bradfitz): add LogID and opts to st?
 	st.WriteHTML(w)
 }

ipn/ipnserver/server_test.go

@@ -62,10 +62,16 @@ func TestRunMultipleAccepts(t *testing.T) {
 	}
 	t.Cleanup(eng.Close)
 
-	opts := ipnserver.Options{
-		SocketPath: socketPath,
-	}
+	opts := ipnserver.Options{}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
+	store := new(ipn.MemoryStore)
+
+	ln, _, err := safesocket.Listen(socketPath, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	err = ipnserver.Run(ctx, logTriggerTestf, ln, store, "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -72,7 +72,7 @@ func (s *Status) Peers() []key.NodePublic {
 type PeerStatusLite struct {
 	TxBytes, RxBytes int64
 	LastHandshake    time.Time
-	NodeKey          tailcfg.NodeKey
+	NodeKey          key.NodePublic
 }
 
 type PeerStatus struct {

ipn/localapi/localapi.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/net/netknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/version"
 )
 
@@ -113,6 +114,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveSetDNS(w, r)
 	case "/localapi/v0/derpmap":
 		h.serveDERPMap(w, r)
+	case "/localapi/v0/metrics":
+		h.serveMetrics(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -184,6 +187,17 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+func (h *Handler) serveMetrics(w http.ResponseWriter, r *http.Request) {
+	// Require write access out of paranoia that the metrics
+	// might contain something sensitive.
+	if !h.PermitWrite {
+		http.Error(w, "metric access denied", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 // serveProfileFunc is the implementation of Handler.serveProfile, after auth,
 // for platforms where we want to link it in.
 var serveProfileFunc func(http.ResponseWriter, *http.Request)

log/filelogger/log.go

@@ -106,6 +106,7 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	if w.fday != day {
 		w.startNewFileLocked()
 	}
+	out = removeDatePrefix(out)
 	if w.f != nil {
 		// RFC3339Nano but with a fixed number (3) of nanosecond digits:
 		const formatPre = "2006-01-02T15:04:05"
@@ -118,6 +119,30 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	}
 }
 
+func isNum(b byte) bool { return '0' <= b && b <= '9' }
+
+// removeDatePrefix returns a subslice of v with the log package's
+// standard datetime prefix format removed, if present.
+func removeDatePrefix(v []byte) []byte {
+	const format = "2009/01/23 01:23:23 "
+	if len(v) < len(format) {
+		return v
+	}
+	for i, b := range v[:len(format)] {
+		fb := format[i]
+		if isNum(fb) {
+			if !isNum(b) {
+				return v
+			}
+			continue
+		}
+		if b != fb {
+			return v
+		}
+	}
+	return v[len(format):]
+}
+
 // startNewFileLocked opens a new log file for writing
 // and also cleans up any old files.
 //

log/filelogger/log_test.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package filelogger
+
+import "testing"
+
+func TestRemoveDatePrefix(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"\n", "\n"},
+		{"2009/01/23 01:23:23", "2009/01/23 01:23:23"},
+		{"2009/01/23 01:23:23 \n", "\n"},
+		{"2009/01/23 01:23:23 foo\n", "foo\n"},
+		{"9999/01/23 01:23:23 foo\n", "foo\n"},
+		{"2009_01/23 01:23:23 had an underscore\n", "2009_01/23 01:23:23 had an underscore\n"},
+	}
+	for i, tt := range tests {
+		got := removeDatePrefix([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Logf("[%d] removeDatePrefix(%q) = %q; want %q", i, tt.in, got, tt.want)
+		}
+	}
+
+}

logpolicy/logpolicy.go

@@ -31,6 +31,8 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
+	"tailscale.com/net/dnscache"
+	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/netknob"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
@@ -38,6 +40,7 @@ import (
 	"tailscale.com/paths"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/racebuild"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
@@ -500,6 +503,9 @@ func New(collection string) *Policy {
 		},
 		HTTPC: &http.Client{Transport: newLogtailTransport(logtail.DefaultHost)},
 	}
+	if collection == logtail.CollectionNode {
+		c.MetricsDelta = clientmetric.EncodeLogTailMetricsDelta
+	}
 
 	if val := getLogTarget(); val != "" {
 		log.Println("You have enabled a non-default log target. Doing without being told to by Tailscale staff or your network administrator will make getting support difficult.")
@@ -588,10 +594,22 @@ func newLogtailTransport(host string) *http.Transport {
 		t0 := time.Now()
 		c, err := nd.DialContext(ctx, netw, addr)
 		d := time.Since(t0).Round(time.Millisecond)
-		if err != nil {
-			log.Printf("logtail: dial %q failed: %v (in %v)", addr, err, d)
-		} else {
+		if err == nil {
 			log.Printf("logtail: dialed %q in %v", addr, d)
+			return c, nil
+		}
+
+		// If we failed to dial, try again with bootstrap DNS.
+		log.Printf("logtail: dial %q failed: %v (in %v), trying bootstrap...", addr, err, d)
+		dnsCache := &dnscache.Resolver{
+			Forward:          dnscache.Get().Forward, // use default cache's forwarder
+			UseLastGood:      true,
+			LookupIPFallback: dnsfallback.Lookup,
+		}
+		dialer := dnscache.Dialer(nd.DialContext, dnsCache)
+		c, err = dialer(ctx, netw, addr)
+		if err == nil {
+			log.Printf("logtail: bootstrap dial succeeded")
 		}
 		return c, err
 	}

logtail/logtail.go

@@ -28,6 +28,12 @@ import (
 // Config.BaseURL isn't provided.
 const DefaultHost = "log.tailscale.io"
 
+const (
+	// CollectionNode is the name of a logtail Config.Collection
+	// for tailscaled (or equivalent: IPNExtension, Android app).
+	CollectionNode = "tailnode.log.tailscale.io"
+)
+
 type Encoder interface {
 	EncodeAll(src, dst []byte) []byte
 	Close() error
@@ -46,6 +52,12 @@ type Config struct {
 	Buffer         Buffer           // temp storage, if nil a MemoryBuffer
 	NewZstdEncoder func() Encoder   // if set, used to compress logs for transmission
 
+	// MetricsDelta, if non-nil, is a func that returns an encoding
+	// delta in clientmetrics to upload alongside existing logs.
+	// It can return either an empty string (for nothing) or a string
+	// that's safe to embed in a JSON string literal without further escaping.
+	MetricsDelta func() string
+
 	// DrainLogs, if non-nil, disables automatic uploading of new logs,
 	// so that logs are only uploaded when a token is sent to DrainLogs.
 	DrainLogs <-chan struct{}
@@ -84,6 +96,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 		drainLogs:      cfg.DrainLogs,
 		timeNow:        cfg.TimeNow,
 		bo:             backoff.NewBackoff("logtail", logf, 30*time.Second),
+		metricsDelta:   cfg.MetricsDelta,
 
 		shutdownStart: make(chan struct{}),
 		shutdownDone:  make(chan struct{}),
@@ -119,6 +132,7 @@ type Logger struct {
 	zstdEncoder    Encoder
 	uploadCancel   func()
 	explainedRaw   bool
+	metricsDelta   func() string // or nil
 
 	shutdownStart chan struct{} // closed when shutdown begins
 	shutdownDone  chan struct{} // closed when shutdown complete
@@ -426,6 +440,14 @@ func (l *Logger) encodeText(buf []byte, skipClientTime bool) []byte {
 		b = append(b, "\"}, "...)
 	}
 
+	if l.metricsDelta != nil {
+		if d := l.metricsDelta(); d != "" {
+			b = append(b, `"metrics": "`...)
+			b = append(b, d...)
+			b = append(b, `",`...)
+		}
+	}
+
 	b = append(b, "\"text\": \""...)
 	for i, c := range buf {
 		switch c {

net/dns/manager_linux.go

@@ -180,20 +180,55 @@ func dnsMode(logf logger.Logf, env newOSConfigEnv) (ret string, err error) {
 			return "direct", nil
 		}
 	case "NetworkManager":
-		// You'd think we would use newNMManager somewhere in
-		// here. However, as explained in
-		// https://github.com/tailscale/tailscale/issues/1699 , using
-		// NetworkManager for DNS configuration carries with it the
-		// cost of losing IPv6 configuration on the Tailscale network
-		// interface. So, when we can avoid it, we bypass
-		// NetworkManager by replacing resolv.conf directly.
-		//
-		// If you ever try to put NMManager back here, keep in mind
-		// that versions >=1.26.6 will ignore DNS configuration
-		// anyway, so you still need a fallback path that uses
-		// directManager.
 		dbg("rc", "nm")
-		return "direct", nil
+		// Sometimes, NetworkManager owns the configuration but points
+		// it at systemd-resolved.
+		if err := resolvedIsActuallyResolver(bs); err != nil {
+			dbg("resolved", "not-in-use")
+			// You'd think we would use newNMManager here. However, as
+			// explained in
+			// https://github.com/tailscale/tailscale/issues/1699 ,
+			// using NetworkManager for DNS configuration carries with
+			// it the cost of losing IPv6 configuration on the
+			// Tailscale network interface. So, when we can avoid it,
+			// we bypass NetworkManager by replacing resolv.conf
+			// directly.
+			//
+			// If you ever try to put NMManager back here, keep in mind
+			// that versions >=1.26.6 will ignore DNS configuration
+			// anyway, so you still need a fallback path that uses
+			// directManager.
+			return "direct", nil
+		}
+		dbg("nm-resolved", "yes")
+
+		if err := env.dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
+			dbg("resolved", "no")
+			return "direct", nil
+		}
+
+		// See large comment above for reasons we'd use NM rather than
+		// resolved. systemd-resolved is actually in charge of DNS
+		// configuration, but in some cases we might need to configure
+		// it via NetworkManager. All the logic below is probing for
+		// that case: is NetworkManager running? If so, is it one of
+		// the versions that requires direct interaction with it?
+		if err := env.dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return "systemd-resolved", nil
+		}
+		safe, err := env.nmVersionBetween("1.26.0", "1.26.5")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return "", fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if safe {
+			dbg("nm-safe", "yes")
+			return "network-manager", nil
+		}
+		dbg("nm-safe", "no")
+		return "systemd-resolved", nil
 	default:
 		dbg("rc", "unknown")
 		return "direct", nil
@@ -244,6 +279,13 @@ func nmIsUsingResolved() error {
 	return nil
 }
 
+// resolvedIsActuallyResolver reports whether the given resolv.conf
+// bytes describe a configuration where systemd-resolved (127.0.0.53)
+// is the only configured nameserver.
+//
+// Returns an error if the configuration is something other than
+// exclusively systemd-resolved, or nil if the config is only
+// systemd-resolved.
 func resolvedIsActuallyResolver(bs []byte) error {
 	cfg, err := readResolv(bytes.NewBuffer(bs))
 	if err != nil {

net/dns/manager_linux_test.go

@@ -34,7 +34,7 @@ func TestLinuxDNSMode(t *testing.T) {
 				resolvDotConf(
 					"# Managed by NetworkManager",
 					"nameserver 10.0.0.1")),
-			wantLog: "dns: [rc=nm ret=direct]",
+			wantLog: "dns: [rc=nm resolved=not-in-use ret=direct]",
 			want:    "direct",
 		},
 		{
@@ -172,6 +172,52 @@ func TestLinuxDNSMode(t *testing.T) {
 			wantLog: "dns: [rc=resolved resolved=no ret=direct]",
 			want:    "direct",
 		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm-safe=no ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_but_no_resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes resolved=no ret=direct]",
+			want:    "direct",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_safe_nm",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.26.3", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm-safe=yes ret=network-manager]",
+			want:    "network-manager",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_no_networkmanager",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning()),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm=no ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

net/dns/resolved.go

@@ -12,6 +12,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
@@ -171,6 +172,14 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
 		m.ifidx, linkDomains,
 	).Store()
+	if err != nil && err.Error() == "Argument list too long" { // TODO: better error match
+		// Issue 3188: older systemd-resolved had argument length limits.
+		// Trim out the *.arpa. entries and try again.
+		err = m.resolved.CallWithContext(
+			ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
+			m.ifidx, linkDomainsWithoutReverseDNS(linkDomains),
+		).Store()
+	}
 	if err != nil {
 		return fmt.Errorf("setLinkDomains: %w", err)
 	}
@@ -234,3 +243,16 @@ func (m *resolvedManager) Close() error {
 
 	return nil
 }
+
+// linkDomainsWithoutReverseDNS returns a copy of v without
+// *.arpa. entries.
+func linkDomainsWithoutReverseDNS(v []resolvedLinkDomain) (ret []resolvedLinkDomain) {
+	for _, d := range v {
+		if strings.HasSuffix(d.Domain, ".arpa.") {
+			// Oh well. At least the rest will work.
+			continue
+		}
+		ret = append(ret, d)
+	}
+	return ret
+}

net/interfaces/interfaces_windows.go

@@ -116,8 +116,12 @@ func notTailscaleInterface(iface *winipcfg.IPAdapterAddresses) bool {
 	// TODO(bradfitz): do this without the Description method's
 	// utf16-to-string allocation. But at least we only do it for
 	// the virtual interfaces, for which there won't be many.
-	return !(iface.IfType == winipcfg.IfTypePropVirtual &&
-		iface.Description() == tsconst.WintunInterfaceDesc)
+	if iface.IfType != winipcfg.IfTypePropVirtual {
+		return true
+	}
+	desc := iface.Description()
+	return !(strings.Contains(desc, tsconst.WintunInterfaceDesc) ||
+		strings.Contains(desc, tsconst.WintunInterfaceDesc0_14))
 }
 
 // NonTailscaleInterfaces returns a map of interface LUID to interface

net/netcheck/netcheck.go

@@ -34,6 +34,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/opt"
+	"tailscale.com/util/clientmetric"
 )
 
 // Debugging and experimentation tweakables.
@@ -232,6 +233,12 @@ func (c *Client) MakeNextReportFull() {
 func (c *Client) ReceiveSTUNPacket(pkt []byte, src netaddr.IPPort) {
 	c.vlogf("received STUN packet from %s", src)
 
+	if src.IP().Is4() {
+		metricSTUNRecv4.Add(1)
+	} else if src.IP().Is6() {
+		metricSTUNRecv6.Add(1)
+	}
+
 	c.mu.Lock()
 	if c.handleHairSTUNLocked(pkt, src) {
 		c.mu.Unlock()
@@ -737,7 +744,13 @@ func (c *Client) udpBindAddr() string {
 // GetReport gets a report.
 //
 // It may not be called concurrently with itself.
-func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, error) {
+func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (_ *Report, reterr error) {
+	defer func() {
+		if reterr != nil {
+			metricNumGetReportError.Add(1)
+		}
+	}()
+	metricNumGetReport.Add(1)
 	// Mask user context with ours that we guarantee to cancel so
 	// we can depend on it being closed in goroutines later.
 	// (User ctx might be context.Background, etc)
@@ -769,6 +782,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		last = nil // causes makeProbePlan below to do a full (initial) plan
 		c.nextFull = false
 		c.lastFull = now
+		metricNumGetReportFull.Add(1)
 	}
 	rs.incremental = last != nil
 	c.mu.Unlock()
@@ -983,6 +997,7 @@ func (c *Client) runHTTPOnlyChecks(ctx context.Context, last *Report, rs *report
 }
 
 func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netaddr.IP, error) {
+	metricHTTPSend.Add(1)
 	var result httpstat.Result
 	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), overallProbeTimeout)
 	defer cancel()
@@ -1217,6 +1232,7 @@ func (rs *reportState) runProbe(ctx context.Context, dm *tailcfg.DERPMap, probe
 
 	switch probe.proto {
 	case probeIPv4:
+		metricSTUNSend4.Add(1)
 		n, err := rs.pc4.WriteTo(req, addr)
 		if n == len(req) && err == nil {
 			rs.mu.Lock()
@@ -1224,6 +1240,7 @@ func (rs *reportState) runProbe(ctx context.Context, dm *tailcfg.DERPMap, probe
 			rs.mu.Unlock()
 		}
 	case probeIPv6:
+		metricSTUNSend6.Add(1)
 		n, err := rs.pc6.WriteTo(req, addr)
 		if n == len(req) && err == nil {
 			rs.mu.Lock()
@@ -1322,3 +1339,15 @@ func conciseOptBool(b opt.Bool, trueVal string) string {
 	}
 	return ""
 }
+
+var (
+	metricNumGetReport      = clientmetric.NewCounter("netcheck_report")
+	metricNumGetReportFull  = clientmetric.NewCounter("netcheck_report_full")
+	metricNumGetReportError = clientmetric.NewCounter("netcheck_report_error")
+
+	metricSTUNSend4 = clientmetric.NewCounter("netcheck_stun_send_ipv4")
+	metricSTUNSend6 = clientmetric.NewCounter("netcheck_stun_send_ipv6")
+	metricSTUNRecv4 = clientmetric.NewCounter("netcheck_stun_recv_ipv4")
+	metricSTUNRecv6 = clientmetric.NewCounter("netcheck_stun_recv_ipv6")
+	metricHTTPSend  = clientmetric.NewCounter("netcheck_https_measure")
+)

net/tstun/wrap.go

@@ -7,7 +7,6 @@
 package tstun
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -17,16 +16,18 @@ import (
 	"sync/atomic"
 	"time"
 
+	"go4.org/mem"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/disco"
 	"tailscale.com/net/packet"
-	"tailscale.com/tailcfg"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/pad32"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -79,7 +80,7 @@ type Wrapper struct {
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 	destMACAtomic  atomic.Value // of [6]byte
-	discoKey       atomic.Value // of tailcfg.DiscoKey
+	discoKey       atomic.Value // of key.DiscoPublic
 
 	// buffer stores the oldest unconsumed packet from tdev.
 	// It is made a static buffer in order to avoid allocations.
@@ -204,7 +205,7 @@ func (t *Wrapper) SetDestIPActivityFuncs(m map[netaddr.IP]func()) {
 //
 // It is only used for filtering out bogus traffic when network
 // stack(s) get confused; see Issue 1526.
-func (t *Wrapper) SetDiscoKey(k tailcfg.DiscoKey) {
+func (t *Wrapper) SetDiscoKey(k key.DiscoPublic) {
 	t.discoKey.Store(k)
 }
 
@@ -216,12 +217,13 @@ func (t *Wrapper) isSelfDisco(p *packet.Parsed) bool {
 		return false
 	}
 	pkt := p.Payload()
-	discoSrc, ok := disco.Source(pkt)
+	discobs, ok := disco.Source(pkt)
 	if !ok {
 		return false
 	}
-	selfDiscoPub, ok := t.discoKey.Load().(tailcfg.DiscoKey)
-	return ok && bytes.Equal(selfDiscoPub[:], discoSrc)
+	discoSrc := key.DiscoPublicFromRaw32(mem.B(discobs))
+	selfDiscoPub, ok := t.discoKey.Load().(key.DiscoPublic)
+	return ok && selfDiscoPub == discoSrc
 }
 
 func (t *Wrapper) Close() error {
@@ -420,11 +422,13 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 	if p.IPProto == ipproto.UDP && // disco is over UDP; avoid isSelfDisco call for TCP/etc
 		t.isSelfDisco(p) {
 		t.logf("[unexpected] received self disco out packet over tstun; dropping")
+		metricPacketOutDropSelfDisco.Add(1)
 		return filter.DropSilently
 	}
 
 	if t.PreFilterOut != nil {
 		if res := t.PreFilterOut(p, t); res.IsDrop() {
+			// Handled by userspaceEngine.handleLocalPackets (quad-100 DNS primarily).
 			return res
 		}
 	}
@@ -436,6 +440,7 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 	}
 
 	if filt.RunOut(p, t.filterFlags) != filter.Accept {
+		metricPacketOutDropFilter.Add(1)
 		return filter.Drop
 	}
 
@@ -470,6 +475,8 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	if res.err != nil {
 		return 0, res.err
 	}
+
+	metricPacketOut.Add(1)
 	pkt := res.data
 	n := copy(buf[offset:], pkt)
 	// t.buffer has a fixed location in memory.
@@ -495,6 +502,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	if !isInjectedPacket && !t.disableFilter {
 		response := t.filterOut(p)
 		if response != filter.Accept {
+			metricPacketOutDrop.Add(1)
 			// Wireguard considers read errors fatal; pretend nothing was read
 			return 0, nil
 		}
@@ -528,6 +536,7 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 	if p.IPProto == ipproto.UDP && // disco is over UDP; avoid isSelfDisco call for TCP/etc
 		t.isSelfDisco(p) {
 		t.logf("[unexpected] received self disco in packet over tstun; dropping")
+		metricPacketInDropSelfDisco.Add(1)
 		return filter.DropSilently
 	}
 
@@ -557,6 +566,7 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 	}
 
 	if outcome != filter.Accept {
+		metricPacketInDropFilter.Add(1)
 
 		// Tell them, via TSMP, we're dropping them due to the ACL.
 		// Their host networking stack can translate this into ICMP
@@ -595,8 +605,10 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 // Write accepts an incoming packet. The packet begins at buf[offset:],
 // like wireguard-go/tun.Device.Write.
 func (t *Wrapper) Write(buf []byte, offset int) (int, error) {
+	metricPacketIn.Add(1)
 	if !t.disableFilter {
 		if t.filterIn(buf[offset:]) != filter.Accept {
+			metricPacketInDrop.Add(1)
 			// If we're not accepting the packet, lie to wireguard-go and pretend
 			// that everything is okay with a nil error, so wireguard-go
 			// doesn't log about this Write "failure".
@@ -720,3 +732,15 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 func (t *Wrapper) Unwrap() tun.Device {
 	return t.tdev
 }
+
+var (
+	metricPacketIn              = clientmetric.NewGauge("tstun_in_from_wg")
+	metricPacketInDrop          = clientmetric.NewGauge("tstun_in_from_wg_drop")
+	metricPacketInDropFilter    = clientmetric.NewGauge("tstun_in_from_wg_drop_filter")
+	metricPacketInDropSelfDisco = clientmetric.NewGauge("tstun_in_from_wg_drop_self_disco")
+
+	metricPacketOut              = clientmetric.NewGauge("tstun_out_to_wg")
+	metricPacketOutDrop          = clientmetric.NewGauge("tstun_out_to_wg_drop")
+	metricPacketOutDropFilter    = clientmetric.NewGauge("tstun_out_to_wg_drop_filter")
+	metricPacketOutDropSelfDisco = clientmetric.NewGauge("tstun_out_to_wg_drop_self_disco")
+)

net/tstun/wrap_test.go

@@ -13,14 +13,15 @@ import (
 	"testing"
 	"unsafe"
 
+	"go4.org/mem"
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/disco"
 	"tailscale.com/net/packet"
-	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
 )
@@ -493,7 +494,7 @@ func TestPeerAPIBypass(t *testing.T) {
 // Issue 1526: drop disco frames from ourselves.
 func TestFilterDiscoLoop(t *testing.T) {
 	var memLog tstest.MemLogger
-	discoPub := tailcfg.DiscoKey{1: 1, 2: 2}
+	discoPub := key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 31: 0}))
 	tw := &Wrapper{logf: memLog.Logf}
 	tw.SetDiscoKey(discoPub)
 	uh := packet.UDP4Header{
@@ -505,7 +506,8 @@ func TestFilterDiscoLoop(t *testing.T) {
 		SrcPort: 9,
 		DstPort: 10,
 	}
-	discoPayload := fmt.Sprintf("%s%s%s", disco.Magic, discoPub[:], [disco.NonceLen]byte{})
+	discobs := discoPub.Raw32()
+	discoPayload := fmt.Sprintf("%s%s%s", disco.Magic, discobs[:], [disco.NonceLen]byte{})
 	pkt := make([]byte, uh.Len()+len(discoPayload))
 	uh.Marshal(pkt)
 	copy(pkt[uh.Len():], discoPayload)

paths/paths_js.go

@@ -7,3 +7,5 @@ package paths
 func ensureStateDirPerms(dirPath string) error {
 	return nil
 }
+
+func LegacyStateFilePath() string { return "" }

safesocket/safesocket.go

@@ -13,6 +13,10 @@ import (
 	"time"
 )
 
+// WindowsLocalPort is the default localhost TCP port
+// used by safesocket on Windows.
+const WindowsLocalPort = 41112
+
 type closeable interface {
 	CloseRead() error
 	CloseWrite() error

safesocket/safesocket_js.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package safesocket
+
+import (
+	"net"
+
+	"github.com/akutz/memconn"
+)
+
+const memName = "Tailscale-IPN"
+
+func listen(path string, port uint16) (_ net.Listener, gotPort uint16, _ error) {
+	ln, err := memconn.Listen("memu", memName)
+	return ln, 1, err
+}
+
+func connect(path string, port uint16) (net.Conn, error) {
+	return memconn.Dial("memu", memName)
+}

safesocket/unixsocket.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !windows
-// +build !windows
+//go:build !windows && !js
+// +build !windows,!js
 
 package safesocket
 

scripts/check_license_headers.sh

@@ -38,6 +38,12 @@ for file in $(find $1 -name '*.go' -not -path '*/.git/*'); do
         $1/wgengine/router/ifconfig_windows.go)
             # WireGuard copyright.
         ;;
+		*_string.go)
+			# Generated file from go:generate stringer
+		;;
+		$1/control/noise/noiseexplorer_test.go)
+			# Noiseexplorer.com copyright.
+		;;
         *)
             header="$(head -3 $file)"
             if ! check_file "$header"; then

scripts/installer.sh

@@ -250,6 +250,9 @@ main() {
 				OS_UNSUPPORTED=1
 			fi
 			;;
+		fedora)
+			# All versions supported, no version checking required.
+			;;
 		arch)
 			# Rolling release, no version checking needed.
 			;;

tailcfg/tailcfg.go

@@ -14,7 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
@@ -48,7 +47,8 @@ import (
 //    22: 2021-06-16: added MapResponse.DNSConfig.ExtraRecords
 //    23: 2021-08-25: DNSConfig.Routes values may be empty (for ExtraRecords support in 1.14.1+)
 //    24: 2021-09-18: MapResponse.Health from control to node; node shows in "tailscale status"
-const CurrentMapRequestVersion = 24
+//    25: 2021-11-01: MapResponse.Debug.Exit
+const CurrentMapRequestVersion = 25
 
 type StableID string
 
@@ -78,34 +78,6 @@ func (u StableNodeID) IsZero() bool {
 	return u == ""
 }
 
-// NodeKey is the WireGuard public key for a node.
-//
-// Deprecated: prefer to use key.NodePublic instead. If you must have
-// a NodeKey, use NodePublic.AsNodeKey.
-type NodeKey = key.NodeKey
-
-// NodeKeyFromNodePublic returns k converted to a NodeKey.
-//
-// Deprecated: exists only as a compatibility bridge while NodeKey
-// gets removed from the codebase. Do not introduce new uses that
-// aren't related to #3206.
-func NodeKeyFromNodePublic(k key.NodePublic) NodeKey {
-	return k.AsNodeKey()
-}
-
-// DiscoKey is the curve25519 public key for path discovery key.
-// It's never written to disk or reused between network start-ups.
-type DiscoKey [32]byte
-
-// DiscoKeyFromNodePublic returns k converted to a DiscoKey.
-//
-// Deprecated: exists only as a compatibility bridge while DiscoKey
-// gets removed from the codebase. Do not introduce new uses that
-// aren't related to #3206.
-func DiscoKeyFromDiscoPublic(k key.DiscoPublic) DiscoKey {
-	return k.Raw32()
-}
-
 // User is an IPN user.
 //
 // A user can have multiple logins associated with it (e.g. gmail and github oauth).
@@ -174,10 +146,10 @@ type Node struct {
 	// Sharer, if non-zero, is the user who shared this node, if different than User.
 	Sharer UserID `json:",omitempty"`
 
-	Key        NodeKey
+	Key        key.NodePublic
 	KeyExpiry  time.Time
 	Machine    key.MachinePublic
-	DiscoKey   DiscoKey
+	DiscoKey   key.DiscoPublic
 	Addresses  []netaddr.IPPrefix // IP addresses of this Node directly
 	AllowedIPs []netaddr.IPPrefix // range of IP addresses to route to this node
 	Endpoints  []string           `json:",omitempty"` // IP+port (public via STUN, and local LANs)
@@ -646,8 +618,8 @@ func (st SignatureType) String() string {
 type RegisterRequest struct {
 	_          structs.Incomparable
 	Version    int // currently 1
-	NodeKey    NodeKey
-	OldNodeKey NodeKey
+	NodeKey    key.NodePublic
+	OldNodeKey key.NodePublic
 	Auth       struct {
 		_ structs.Incomparable
 		// One of Provider/LoginName, Oauth2Token, or AuthKey is set.
@@ -764,8 +736,8 @@ type MapRequest struct {
 
 	Compress    string // "zstd" or "" (no compression)
 	KeepAlive   bool   // whether server should send keep-alives back to us
-	NodeKey     NodeKey
-	DiscoKey    DiscoKey
+	NodeKey     key.NodePublic
+	DiscoKey    key.DiscoPublic
 	IncludeIPv6 bool `json:",omitempty"` // include IPv6 endpoints in returned Node Endpoints (for Version 4 clients)
 	Stream      bool // if true, multiple MapResponse objects are returned
 	Hostinfo    *Hostinfo
@@ -1128,12 +1100,16 @@ type Debug struct {
 	// fixed port.
 	RandomizeClientPort bool `json:",omitempty"`
 
-	/// DisableUPnP is whether the client will attempt to perform a UPnP portmapping.
+	// DisableUPnP is whether the client will attempt to perform a UPnP portmapping.
 	// By default, we want to enable it to see if it works on more clients.
 	//
 	// If UPnP catastrophically fails for people, this should be set to True to kill
 	// new attempts at UPnP connections.
 	DisableUPnP opt.Bool `json:",omitempty"`
+
+	// Exit optionally specifies that the client should os.Exit
+	// with this code.
+	Exit *int `json:",omitempty"`
 }
 
 func appendKey(base []byte, prefix string, k [32]byte) []byte {
@@ -1148,25 +1124,6 @@ func keyMarshalText(prefix string, k [32]byte) []byte {
 	return appendKey(nil, prefix, k)
 }
 
-func (k DiscoKey) String() string { return fmt.Sprintf("discokey:%x", k[:]) }
-func (k DiscoKey) MarshalText() ([]byte, error) {
-	dk := key.DiscoPublicFromRaw32(mem.B(k[:]))
-	return dk.MarshalText()
-}
-func (k *DiscoKey) UnmarshalText(text []byte) error {
-	var dk key.DiscoPublic
-	if err := dk.UnmarshalText(text); err != nil {
-		return err
-	}
-	dk.AppendTo(k[:0])
-	return nil
-}
-func (k DiscoKey) ShortString() string      { return fmt.Sprintf("d:%x", k[:8]) }
-func (k DiscoKey) AppendTo(b []byte) []byte { return appendKey(b, "discokey:", k) }
-
-// IsZero reports whether k is the zero value.
-func (k DiscoKey) IsZero() bool { return k == DiscoKey{} }
-
 func (id ID) String() string      { return fmt.Sprintf("id:%x", int64(id)) }
 func (id UserID) String() string  { return fmt.Sprintf("userid:%x", int64(id)) }
 func (id LoginID) String() string { return fmt.Sprintf("loginid:%x", int64(id)) }
@@ -1288,7 +1245,7 @@ type SetDNSRequest struct {
 	Version int
 
 	// NodeKey is the client's current node key.
-	NodeKey NodeKey
+	NodeKey key.NodePublic
 
 	// Name is the domain name for which to create a record.
 	// For ACME DNS-01 challenges, it should be one of the domains

tailcfg/tailcfg_clone.go

@@ -72,10 +72,10 @@ var _NodeCloneNeedsRegeneration = Node(struct {
 	Name                    string
 	User                    UserID
 	Sharer                  UserID
-	Key                     NodeKey
+	Key                     key.NodePublic
 	KeyExpiry               time.Time
 	Machine                 key.MachinePublic
-	DiscoKey                DiscoKey
+	DiscoKey                key.DiscoPublic
 	Addresses               []netaddr.IPPrefix
 	AllowedIPs              []netaddr.IPPrefix
 	Endpoints               []string

tailcfg/tailcfg_test.go

@@ -264,13 +264,13 @@ func TestNodeEqual(t *testing.T) {
 			true,
 		},
 		{
-			&Node{Key: n1.AsNodeKey()},
-			&Node{Key: key.NewNode().Public().AsNodeKey()},
+			&Node{Key: n1},
+			&Node{Key: key.NewNode().Public()},
 			false,
 		},
 		{
-			&Node{Key: n1.AsNodeKey()},
-			&Node{Key: n1.AsNodeKey()},
+			&Node{Key: n1},
+			&Node{Key: n1},
 			true,
 		},
 		{
@@ -407,14 +407,6 @@ func TestNetInfoFields(t *testing.T) {
 	}
 }
 
-func TestDiscoKeyMarshal(t *testing.T) {
-	var k1, k2 DiscoKey
-	for i := range k1 {
-		k1[i] = byte(i)
-	}
-	testKey(t, "discokey:", k1, &k2)
-}
-
 type keyIn interface {
 	String() string
 	MarshalText() ([]byte, error)
@@ -542,15 +534,6 @@ func TestAppendKeyAllocs(t *testing.T) {
 	}
 }
 
-func TestDiscoKeyAppend(t *testing.T) {
-	d := DiscoKey{1: 1, 2: 2}
-	got := string(d.AppendTo([]byte("foo")))
-	want := "foodiscokey:0001020000000000000000000000000000000000000000000000000000000000"
-	if got != want {
-		t.Errorf("got %q; want %q", got, want)
-	}
-}
-
 func TestRegisterRequestNilClone(t *testing.T) {
 	var nilReq *RegisterRequest
 	got := nilReq.Clone()

tsconst/interface.go

@@ -7,5 +7,6 @@
 package tsconst
 
 // WintunInterfaceDesc is the description attached to Tailscale
-// interfaces on Windows. This is set by our modified WinTun driver.
+// interfaces on Windows. This is set by the WinTun driver.
 const WintunInterfaceDesc = "Tailscale Tunnel"
+const WintunInterfaceDesc0_14 = "Wintun Userspace Tunnel"

tsnet/tsnet.go

@@ -127,10 +127,11 @@ func (s *Server) start() error {
 		return fmt.Errorf("%T is not a wgengine.InternalsGetter", eng)
 	}
 
-	ns, err := netstack.Create(logf, tunDev, eng, magicConn, false)
+	ns, err := netstack.Create(logf, tunDev, eng, magicConn)
 	if err != nil {
 		return fmt.Errorf("netstack.Create: %w", err)
 	}
+	ns.ProcessLocalIPs = true
 	ns.ForwardTCPIn = s.forwardTCP
 	if err := ns.Start(); err != nil {
 		return fmt.Errorf("failed to start netstack: %w", err)
@@ -147,6 +148,7 @@ func (s *Server) start() error {
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	lb.SetVarRoot(s.dir)
 	s.lb = lb
 	lb.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)

tstest/archtest/archtest_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package archtest
+
+import (
+	"runtime"
+	"testing"
+
+	"inet.af/netstack/atomicbitops"
+)
+
+// tests netstack's AlignedAtomicInt64.
+func TestAlignedAtomicInt64(t *testing.T) {
+	type T struct {
+		A atomicbitops.AlignedAtomicInt64
+		x int32
+		B atomicbitops.AlignedAtomicInt64
+	}
+
+	t.Logf("I am %v/%v\n", runtime.GOOS, runtime.GOARCH)
+	var x T
+	x.A.Store(1)
+	x.B.Store(2)
+	if got, want := x.A.Load(), int64(1); got != want {
+		t.Errorf("A = %v; want %v", got, want)
+	}
+	if got, want := x.B.Load(), int64(2); got != want {
+		t.Errorf("A = %v; want %v", got, want)
+	}
+}

tstest/archtest/qemu_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && amd64 && !race
+// +build linux,amd64,!race
+
+package archtest
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestInQemu(t *testing.T) {
+	t.Parallel()
+	type Arch struct {
+		Goarch string // GOARCH value
+		Qarch  string // qemu name
+	}
+	arches := []Arch{
+		{"arm", "arm"},
+		{"arm64", "aarch64"},
+		{"mips", "mips"},
+		{"mipsle", "mipsel"},
+		{"mips64", "mips64"},
+		{"mips64le", "mips64el"},
+		{"386", "386"},
+	}
+	inCI := os.Getenv("CI") == "true"
+	for _, arch := range arches {
+		arch := arch
+		t.Run(arch.Goarch, func(t *testing.T) {
+			t.Parallel()
+			qemuUser := "qemu-" + arch.Qarch
+			execVia := qemuUser
+			if arch.Goarch == "386" {
+				execVia = "" // amd64 can run it fine
+			} else {
+				look, err := exec.LookPath(qemuUser)
+				if err != nil {
+					if inCI {
+						t.Fatalf("in CI and qemu not available: %v", err)
+					}
+					t.Skipf("%s not found; skipping test. error was: %v", qemuUser, err)
+				}
+				t.Logf("using %v", look)
+			}
+			cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"),
+				"test",
+				"--exec="+execVia,
+				"-v",
+				"tailscale.com/tstest/archtest",
+			)
+			cmd.Env = append(os.Environ(), "GOARCH="+arch.Goarch)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				if strings.Contains(string(out), "fatal error: sigaction failed") && !inCI {
+					t.Skip("skipping; qemu too old. use 5.x.")
+				}
+				t.Errorf("failed: %s", out)
+			}
+			sub := fmt.Sprintf("I am linux/%s", arch.Goarch)
+			if !bytes.Contains(out, []byte(sub)) {
+				t.Errorf("output didn't contain %q: %s", sub, out)
+			}
+		})
+	}
+}

tstest/integration/gen_deps.go

@@ -15,6 +15,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"strings"
 )
 
 func main() {
@@ -52,6 +53,10 @@ import (
 	// process and can cache a prior success when a dependency changes.
 `)
 	for _, dep := range x.Imports {
+		if !strings.Contains(dep, ".") {
+			// Omit stanard library deps.
+			continue
+		}
 		fmt.Fprintf(&out, "\t_ %q\n", dep)
 	}
 	fmt.Fprintf(&out, ")\n")

tstest/integration/integration_test.go

@@ -84,6 +84,47 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+func TestOneNodeExpiredKey(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp()
+	n1.AwaitRunning(t)
+
+	nodes := env.Control.AllNodes()
+	if len(nodes) != 1 {
+		t.Fatalf("expected 1 node, got %d nodes", len(nodes))
+	}
+
+	nodeKey := nodes[0].Key
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	env.Control.SetExpireAllNodes(true)
+	n1.AwaitNeedsLogin(t)
+	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
+	if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	env.Control.SetExpireAllNodes(false)
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+}
+
 func TestCollectPanic(t *testing.T) {
 	t.Parallel()
 	bins := BuildTestBinaries(t)
@@ -713,7 +754,7 @@ func (n *testNode) MustDown() {
 // over its localhost IPC mechanism. (Unix socket, etc)
 func (n *testNode) AwaitListening(t testing.TB) {
 	if err := tstest.WaitFor(20*time.Second, func() (err error) {
-		c, err := safesocket.Connect(n.sockFile, 41112)
+		c, err := safesocket.Connect(n.sockFile, safesocket.WindowsLocalPort)
 		if err != nil {
 			return err
 		}
@@ -780,6 +821,23 @@ func (n *testNode) AwaitRunning(t testing.TB) {
 	}
 }
 
+// AwaitNeedsLogin waits for n to reach the IPN state "NeedsLogin".
+func (n *testNode) AwaitNeedsLogin(t testing.TB) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		st, err := n.Status()
+		if err != nil {
+			return err
+		}
+		if st.BackendState != "NeedsLogin" {
+			return fmt.Errorf("in state %q", st.BackendState)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failure/timeout waiting for transition to NeedsLogin status: %v", err)
+	}
+}
+
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {

tstest/integration/tailscaled_deps_test_darwin.go

@@ -11,37 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/exec"
-	_ "os/signal"
-	_ "path/filepath"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -50,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -61,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -11,35 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -48,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -59,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_linux.go

@@ -11,35 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -48,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -59,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -11,35 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -48,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -59,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_windows.go

@@ -11,38 +11,16 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "golang.org/x/sys/windows"
 	_ "golang.org/x/sys/windows/svc"
 	_ "golang.org/x/sys/windows/svc/mgr"
 	_ "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/logtail/backoff"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
@@ -52,10 +30,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/util/winutil"
 	_ "tailscale.com/version"
@@ -65,5 +46,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/testcontrol/testcontrol.go

@@ -58,13 +58,14 @@ type Server struct {
 	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        key.MachinePublic
 	privKey       key.ControlPrivate // not strictly needed vs. MachinePrivate, but handy to test type interactions.
-	nodes         map[tailcfg.NodeKey]*tailcfg.Node
-	users         map[tailcfg.NodeKey]*tailcfg.User
-	logins        map[tailcfg.NodeKey]*tailcfg.Login
+	nodes         map[key.NodePublic]*tailcfg.Node
+	users         map[key.NodePublic]*tailcfg.User
+	logins        map[key.NodePublic]*tailcfg.Login
 	updates       map[tailcfg.NodeID]chan updateType
 	authPath      map[string]*AuthPath
-	nodeKeyAuthed map[tailcfg.NodeKey]bool // key => true once authenticated
-	pingReqsToAdd map[tailcfg.NodeKey]*tailcfg.PingRequest
+	nodeKeyAuthed map[key.NodePublic]bool // key => true once authenticated
+	pingReqsToAdd map[key.NodePublic]*tailcfg.PingRequest
+	allExpired    bool // All nodes will be told their node key is expired.
 }
 
 // BaseURL returns the server's base URL, without trailing slash.
@@ -103,7 +104,7 @@ func (s *Server) condLocked() *sync.Cond {
 
 // AwaitNodeInMapRequest waits for node k to be stuck in a map poll.
 // It returns an error if and only if the context is done first.
-func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k tailcfg.NodeKey) error {
+func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k key.NodePublic) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	cond := s.condLocked()
@@ -135,11 +136,11 @@ func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k tailcfg.NodeKey) e
 
 // AddPingRequest sends the ping pr to nodeKeyDst. It reports whether it did so. That is,
 // it reports whether nodeKeyDst was connected.
-func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequest) bool {
+func (s *Server) AddPingRequest(nodeKeyDst key.NodePublic, pr *tailcfg.PingRequest) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.pingReqsToAdd == nil {
-		s.pingReqsToAdd = map[tailcfg.NodeKey]*tailcfg.PingRequest{}
+		s.pingReqsToAdd = map[key.NodePublic]*tailcfg.PingRequest{}
 	}
 	// Now send the update to the channel
 	node := s.nodeLocked(nodeKeyDst)
@@ -153,8 +154,20 @@ func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequ
 	return sendUpdate(oldUpdatesCh, updateDebugInjection)
 }
 
+// Mark the Node key of every node as expired
+func (s *Server) SetExpireAllNodes(expired bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.allExpired = expired
+
+	for _, node := range s.nodes {
+		sendUpdate(s.updates[node.ID], updateSelfChanged)
+	}
+}
+
 type AuthPath struct {
-	nodeKey tailcfg.NodeKey
+	nodeKey key.NodePublic
 
 	closeOnce sync.Once
 	ch        chan struct{}
@@ -254,7 +267,7 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
 }
 
 // Node returns the node for nodeKey. It's always nil or cloned memory.
-func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
+func (s *Server) Node(nodeKey key.NodePublic) *tailcfg.Node {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	return s.nodeLocked(nodeKey)
@@ -263,7 +276,7 @@ func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 // nodeLocked returns the node for nodeKey. It's always nil or cloned memory.
 //
 // s.mu must be held.
-func (s *Server) nodeLocked(nodeKey tailcfg.NodeKey) *tailcfg.Node {
+func (s *Server) nodeLocked(nodeKey key.NodePublic) *tailcfg.Node {
 	return s.nodes[nodeKey].Clone()
 }
 
@@ -272,13 +285,14 @@ func (s *Server) AddFakeNode() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.nodes == nil {
-		s.nodes = make(map[tailcfg.NodeKey]*tailcfg.Node)
+		s.nodes = make(map[key.NodePublic]*tailcfg.Node)
 	}
-	nk := tailcfg.NodeKeyFromNodePublic(key.NewNode().Public())
+	nk := key.NewNode().Public()
 	mk := key.NewMachine().Public()
-	dk := tailcfg.DiscoKeyFromDiscoPublic(key.NewDisco().Public())
-	id := int64(binary.LittleEndian.Uint64(nk[:]))
-	ip := netaddr.IPv4(nk[0], nk[1], nk[2], nk[3])
+	dk := key.NewDisco().Public()
+	r := nk.Raw32()
+	id := int64(binary.LittleEndian.Uint64(r[:]))
+	ip := netaddr.IPv4(r[0], r[1], r[2], r[3])
 	addr := netaddr.IPPrefixFrom(ip, 32)
 	s.nodes[nk] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(id),
@@ -306,14 +320,14 @@ func (s *Server) AllNodes() (nodes []*tailcfg.Node) {
 	return nodes
 }
 
-func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login) {
+func (s *Server) getUser(nodeKey key.NodePublic) (*tailcfg.User, *tailcfg.Login) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.users == nil {
-		s.users = map[tailcfg.NodeKey]*tailcfg.User{}
+		s.users = map[key.NodePublic]*tailcfg.User{}
 	}
 	if s.logins == nil {
-		s.logins = map[tailcfg.NodeKey]*tailcfg.Login{}
+		s.logins = map[key.NodePublic]*tailcfg.Login{}
 	}
 	if u, ok := s.users[nodeKey]; ok {
 		return u, s.logins[nodeKey]
@@ -353,7 +367,7 @@ func (s *Server) authPathDone(authPath string) <-chan struct{} {
 	return nil
 }
 
-func (s *Server) addAuthPath(authPath string, nodeKey tailcfg.NodeKey) {
+func (s *Server) addAuthPath(authPath string, nodeKey key.NodePublic) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.authPath == nil {
@@ -385,7 +399,7 @@ func (s *Server) CompleteAuth(authPathOrURL string) bool {
 		panic("zero AuthPath.NodeKey")
 	}
 	if s.nodeKeyAuthed == nil {
-		s.nodeKeyAuthed = map[tailcfg.NodeKey]bool{}
+		s.nodeKeyAuthed = map[key.NodePublic]bool{}
 	}
 	s.nodeKeyAuthed[ap.nodeKey] = true
 	ap.CompleteSuccessfully()
@@ -433,10 +447,12 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		// some follow-ups? For now all are successes.
 	}
 
-	user, login := s.getUser(req.NodeKey)
+	nk := req.NodeKey
+
+	user, login := s.getUser(nk)
 	s.mu.Lock()
 	if s.nodes == nil {
-		s.nodes = map[tailcfg.NodeKey]*tailcfg.Node{}
+		s.nodes = map[key.NodePublic]*tailcfg.Node{}
 	}
 
 	machineAuthorized := true // TODO: add Server.RequireMachineAuth
@@ -449,7 +465,7 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		v6Prefix,
 	}
 
-	s.nodes[req.NodeKey] = &tailcfg.Node{
+	s.nodes[nk] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(user.ID),
 		StableID:          tailcfg.StableNodeID(fmt.Sprintf("TESTCTRL%08x", int(user.ID))),
 		User:              user.ID,
@@ -461,9 +477,10 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		Hostinfo:          *req.Hostinfo,
 	}
 	requireAuth := s.RequireAuth
-	if requireAuth && s.nodeKeyAuthed[req.NodeKey] {
+	if requireAuth && s.nodeKeyAuthed[nk] {
 		requireAuth = false
 	}
+	allExpired := s.allExpired
 	s.mu.Unlock()
 
 	authURL := ""
@@ -471,14 +488,14 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		randHex := make([]byte, 10)
 		crand.Read(randHex)
 		authPath := fmt.Sprintf("/auth/%x", randHex)
-		s.addAuthPath(authPath, req.NodeKey)
+		s.addAuthPath(authPath, nk)
 		authURL = s.BaseURL() + authPath
 	}
 
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
 		User:              *user,
 		Login:             *login,
-		NodeKeyExpired:    false,
+		NodeKeyExpired:    allExpired,
 		MachineAuthorized: machineAuthorized,
 		AuthURL:           authURL,
 	})
@@ -639,6 +656,13 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey key.Machi
 		if res == nil {
 			return // done
 		}
+
+		s.mu.Lock()
+		allExpired := s.allExpired
+		s.mu.Unlock()
+		if allExpired {
+			res.Node.KeyExpiry = time.Now().Add(-1 * time.Minute)
+		}
 		// TODO: add minner if/when needed
 		resBytes, err := json.Marshal(res)
 		if err != nil {
@@ -690,12 +714,13 @@ var keepAliveMsg = &struct {
 //
 // No updates to s are done here.
 func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, err error) {
-	node := s.Node(req.NodeKey)
+	nk := req.NodeKey
+	node := s.Node(nk)
 	if node == nil {
 		// node key rotated away (once test server supports that)
 		return nil, nil
 	}
-	user, _ := s.getUser(req.NodeKey)
+	user, _ := s.getUser(nk)
 	res = &tailcfg.MapResponse{
 		Node:            node,
 		DERPMap:         s.DERPMap,
@@ -727,9 +752,9 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 
 	// Consume the PingRequest while protected by mutex if it exists
 	s.mu.Lock()
-	if pr, ok := s.pingReqsToAdd[node.Key]; ok {
+	if pr, ok := s.pingReqsToAdd[nk]; ok {
 		res.PingRequest = pr
-		delete(s.pingReqsToAdd, node.Key)
+		delete(s.pingReqsToAdd, nk)
 	}
 	s.mu.Unlock()
 	return res, nil

types/key/disco.go

@@ -21,6 +21,10 @@ const (
 	// This prefix is used in the control protocol, so cannot be
 	// changed.
 	discoPublicHexPrefix = "discokey:"
+
+	// DiscoPublicRawLen is the length in bytes of a DiscoPublic, when
+	// serialized with AppendTo, Raw32 or WriteRawWithoutAllocating.
+	DiscoPublicRawLen = 32
 )
 
 // DiscoPrivate is a disco key, used for peer-to-peer path discovery.
@@ -115,12 +119,6 @@ func (k DiscoPublic) AppendTo(buf []byte) []byte {
 	return append(buf, k.k[:]...)
 }
 
-// RawLen returns the length of k when to the format handled by
-// ReadRawWithoutAllocating and WriteRawWithoutAllocating.
-func (k DiscoPublic) RawLen() int {
-	return 32
-}
-
 // String returns the output of MarshalText as a string.
 func (k DiscoPublic) String() string {
 	bs, err := k.MarshalText()

types/key/machine.go

@@ -77,6 +77,19 @@ func (k *MachinePrivate) UnmarshalText(b []byte) error {
 	return parseHex(k.k[:], mem.B(b), mem.S(machinePrivateHexPrefix))
 }
 
+// UntypedBytes returns k, encoded as an untyped 64-character hex
+// string.
+//
+// Deprecated: this function is risky to use, because it produces
+// serialized values that do not identify themselves as a
+// MachinePrivate, allowing other code to potentially parse it back in
+// as the wrong key type. For new uses that don't require this
+// specific raw byte serialization, please use
+// MarshalText/UnmarshalText.
+func (k MachinePrivate) UntypedBytes() []byte {
+	return append([]byte(nil), k.k[:]...)
+}
+
 // SealTo wraps cleartext into a NaCl box (see
 // golang.org/x/crypto/nacl) to p, authenticated from k, using a
 // random nonce.
@@ -112,6 +125,19 @@ type MachinePublic struct {
 	k [32]byte
 }
 
+// MachinePublicFromRaw32 parses a 32-byte raw value as a MachinePublic.
+//
+// This should be used only when deserializing a MachinePublic from a
+// binary protocol.
+func MachinePublicFromRaw32(raw mem.RO) MachinePublic {
+	if raw.Len() != 32 {
+		panic("input has wrong size")
+	}
+	var ret MachinePublic
+	raw.Copy(ret.k[:])
+	return ret
+}
+
 // ParseMachinePublicUntyped parses an untyped 64-character hex value
 // as a MachinePublic.
 //
@@ -153,6 +179,19 @@ func (k MachinePublic) UntypedHexString() string {
 	return hex.EncodeToString(k.k[:])
 }
 
+// UntypedBytes returns k, encoded as an untyped 64-character hex
+// string.
+//
+// Deprecated: this function is risky to use, because it produces
+// serialized values that do not identify themselves as a
+// MachinePublic, allowing other code to potentially parse it back in
+// as the wrong key type. For new uses that don't require this
+// specific raw byte serialization, please use
+// MarshalText/UnmarshalText.
+func (k MachinePublic) UntypedBytes() []byte {
+	return append([]byte(nil), k.k[:]...)
+}
+
 // String returns the output of MarshalText as a string.
 func (k MachinePublic) String() string {
 	bs, err := k.MarshalText()

types/key/node.go

@@ -33,6 +33,10 @@ const (
 	// This prefix is used in the control protocol, so cannot be
 	// changed.
 	nodePublicHexPrefix = "nodekey:"
+
+	// NodePublicRawLen is the length in bytes of a NodePublic, when
+	// serialized with AppendTo, Raw32 or WriteRawWithoutAllocating.
+	NodePublicRawLen = 32
 )
 
 // NodePrivate is a node key, used for WireGuard tunnels and
@@ -190,12 +194,6 @@ func (k NodePublic) AppendTo(buf []byte) []byte {
 	return append(buf, k.k[:]...)
 }
 
-// RawLen returns the length of k when to the format handled by
-// ReadRawWithoutAllocating and WriteRawWithoutAllocating.
-func (k NodePublic) RawLen() int {
-	return 32
-}
-
 // ReadRawWithoutAllocating initializes k with bytes read from br.
 // The reading is done ~4x slower than io.ReadFull, but in exchange is
 // allocation-free.
@@ -307,10 +305,3 @@ func (k NodePublic) WireGuardGoString() string {
 	b[second+3] = b64((k.k[31] << 2) & 63)
 	return string(b)
 }
-
-// AsNodeKey returns k converted to a NodeKey.
-//
-// Cross-compatibility shim as part of #3206.
-func (k NodePublic) AsNodeKey() NodeKey {
-	return k.Raw32()
-}

types/key/node_legacy.go

@@ -1,28 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package key
-
-import (
-	"go4.org/mem"
-)
-
-// NodeKey is the legacy form of NodePublic.
-// See #3206 for removal effort.
-type NodeKey [32]byte
-
-func (k NodeKey) ShortString() string          { return k.AsNodePublic().ShortString() }
-func (k NodeKey) String() string               { return k.AsNodePublic().String() }
-func (k NodeKey) MarshalText() ([]byte, error) { return k.AsNodePublic().MarshalText() }
-func (k NodeKey) AsNodePublic() NodePublic     { return NodePublicFromRaw32(mem.B(k[:])) }
-func (k NodeKey) IsZero() bool                 { return k == NodeKey{} }
-
-func (k *NodeKey) UnmarshalText(text []byte) error {
-	var nk NodePublic
-	if err := nk.UnmarshalText(text); err != nil {
-		return err
-	}
-	*k = nk.AsNodeKey()
-	return nil
-}

types/key/node_legacy_test.go

@@ -1,75 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package key
-
-import (
-	"bytes"
-	"encoding/json"
-	"strings"
-	"testing"
-)
-
-func TestNodeKeyMarshal(t *testing.T) {
-	var k1, k2 NodeKey
-	for i := range k1 {
-		k1[i] = byte(i)
-	}
-
-	const prefix = "nodekey:"
-	got, err := k1.MarshalText()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := k2.UnmarshalText(got); err != nil {
-		t.Fatal(err)
-	}
-	if s := k1.String(); string(got) != s {
-		t.Errorf("MarshalText = %q != String %q", got, s)
-	}
-	if !strings.HasPrefix(string(got), prefix) {
-		t.Errorf("%q didn't start with prefix %q", got, prefix)
-	}
-	if k2 != k1 {
-		t.Errorf("mismatch after unmarshal")
-	}
-}
-
-func TestNodeKeyRoundTrip(t *testing.T) {
-	serialized := `{
-      "Pub":"nodekey:50d20b455ecf12bc453f83c2cfdb2a24925d06cf2598dcaa54e91af82ce9f765"
-    }`
-
-	// Carefully check that the expected serialized data decodes and
-	// re-encodes to the expected keys. These types are serialized to
-	// disk all over the place and need to be stable.
-	pub := NodeKey{
-		0x50, 0xd2, 0xb, 0x45, 0x5e, 0xcf, 0x12, 0xbc, 0x45, 0x3f, 0x83,
-		0xc2, 0xcf, 0xdb, 0x2a, 0x24, 0x92, 0x5d, 0x6, 0xcf, 0x25, 0x98,
-		0xdc, 0xaa, 0x54, 0xe9, 0x1a, 0xf8, 0x2c, 0xe9, 0xf7, 0x65,
-	}
-
-	type key struct {
-		Pub NodeKey
-	}
-
-	var a key
-	if err := json.Unmarshal([]byte(serialized), &a); err != nil {
-		t.Fatal(err)
-	}
-	if a.Pub != pub {
-		t.Errorf("wrong deserialization of public key, got %#v want %#v", a.Pub, pub)
-	}
-
-	bs, err := json.MarshalIndent(a, "", "  ")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var b bytes.Buffer
-	json.Indent(&b, []byte(serialized), "", "  ")
-	if got, want := string(bs), b.String(); got != want {
-		t.Error("json serialization doesn't roundtrip")
-	}
-}

types/netmap/netmap.go

@@ -26,7 +26,7 @@ type NetworkMap struct {
 	// Core networking
 
 	SelfNode   *tailcfg.Node
-	NodeKey    tailcfg.NodeKey
+	NodeKey    key.NodePublic
 	PrivateKey key.NodePrivate
 	Expiry     time.Time
 	// Name is the DNS name assigned to this node.

types/netmap/netmap_test.go

@@ -8,24 +8,30 @@ import (
 	"encoding/hex"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
-func testNodeKey(b byte) (ret tailcfg.NodeKey) {
-	for i := range ret {
-		ret[i] = b
+func testNodeKey(b byte) (ret key.NodePublic) {
+	var bs [key.NodePublicRawLen]byte
+	for i := range bs {
+		bs[i] = b
 	}
-	return
+	return key.NodePublicFromRaw32(mem.B(bs[:]))
 }
 
-func testDiscoKey(hexPrefix string) (ret tailcfg.DiscoKey) {
+func testDiscoKey(hexPrefix string) (ret key.DiscoPublic) {
 	b, err := hex.DecodeString(hexPrefix)
 	if err != nil {
 		panic(err)
 	}
-	copy(ret[:], b)
-	return
+	// this function is used with short hexes, so zero-extend the raw
+	// value.
+	var bs [32]byte
+	copy(bs[:], b)
+	return key.DiscoPublicFromRaw32(mem.B(bs[:]))
 }
 
 func TestNetworkMapConcise(t *testing.T) {

util/clientmetric/clientmetric.go

@@ -0,0 +1,306 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package clientmetric provides client-side metrics whose values
+// get occasionally logged.
+package clientmetric
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	mu          sync.Mutex // guards vars in this block
+	metrics     = map[string]*Metric{}
+	numWireID   int         // how many wireIDs have been allocated
+	lastDelta   time.Time   // time of last call to EncodeLogTailMetricsDelta
+	sortedDirty bool        // whether sorted needs to be rebuilt
+	sorted      []*Metric   // by name
+	lastLogVal  []scanEntry // by Metric.regIdx
+	unsorted    []*Metric   // by Metric.regIdx
+
+	// valFreeList is a set of free contiguous int64s whose
+	// element addresses get assigned to Metric.v.
+	// Any memory address in len(valFreeList) is free for use.
+	// They're contiguous to reduce cache churn during diff scans.
+	// When out of length, a new backing array is made.
+	valFreeList []int64
+)
+
+// scanEntry contains the minimal data needed for quickly scanning
+// memory for changed values. It's small to reduce memory pressure.
+type scanEntry struct {
+	v          *int64 // Metric.v
+	lastLogged int64  // last logged value
+}
+
+// Type is a metric type: counter or gauge.
+type Type uint8
+
+const (
+	TypeGauge Type = iota
+	TypeCounter
+)
+
+// Metric is an integer metric value that's tracked over time.
+//
+// It's safe for concurrent use.
+type Metric struct {
+	v      *int64 // atomic; the metric value
+	regIdx int    // index into lastLogVal and unsorted
+	name   string
+	typ    Type
+
+	// The following fields are owned by the package-level 'mu':
+
+	// wireID is the lazily-allocated "wire ID". Until a metric is encoded
+	// in the logs (by EncodeLogTailMetricsDelta), it has no wireID. This
+	// ensures that unused metrics don't waste valuable low numbers, which
+	// encode with varints with fewer bytes.
+	wireID int
+
+	// lastNamed is the last time the name of this metric was
+	// written on the wire.
+	lastNamed time.Time
+}
+
+func (m *Metric) Name() string { return m.name }
+func (m *Metric) Value() int64 { return atomic.LoadInt64(m.v) }
+func (m *Metric) Type() Type   { return m.typ }
+
+// Add increments m's value by n.
+//
+// If m is of type counter, n should not be negative.
+func (m *Metric) Add(n int64) {
+	atomic.AddInt64(m.v, n)
+}
+
+// Set sets m's value to v.
+//
+// If m is of type counter, Set should not be used.
+func (m *Metric) Set(v int64) {
+	atomic.StoreInt64(m.v, v)
+}
+
+// Publish registers a metric in the global map.
+// It panics if the name is a duplicate anywhere in the process.
+func (m *Metric) Publish() {
+	mu.Lock()
+	defer mu.Unlock()
+	if m.name == "" {
+		panic("unnamed Metric")
+	}
+	if _, dup := metrics[m.name]; dup {
+		panic("duplicate metric " + m.name)
+	}
+	metrics[m.name] = m
+	sortedDirty = true
+
+	if len(valFreeList) == 0 {
+		valFreeList = make([]int64, 256)
+	}
+	m.v = &valFreeList[0]
+	valFreeList = valFreeList[1:]
+
+	m.regIdx = len(unsorted)
+	unsorted = append(unsorted, m)
+	lastLogVal = append(lastLogVal, scanEntry{v: m.v})
+}
+
+// Metrics returns the sorted list of metrics.
+//
+// The returned slice should not be mutated.
+func Metrics() []*Metric {
+	mu.Lock()
+	defer mu.Unlock()
+	if sortedDirty {
+		sortedDirty = false
+		sorted = make([]*Metric, 0, len(metrics))
+		for _, m := range metrics {
+			sorted = append(sorted, m)
+		}
+		sort.Slice(sorted, func(i, j int) bool {
+			return sorted[i].name < sorted[j].name
+		})
+	}
+	return sorted
+}
+
+// NewUnpublished initializes a new Metric without calling Publish on
+// it.
+func NewUnpublished(name string, typ Type) *Metric {
+	if i := strings.IndexFunc(name, isIllegalMetricRune); name == "" || i != -1 {
+		panic(fmt.Sprintf("illegal metric name %q (index %v)", name, i))
+	}
+	return &Metric{
+		name: name,
+		typ:  typ,
+	}
+}
+
+func isIllegalMetricRune(r rune) bool {
+	return !(r >= 'a' && r <= 'z' ||
+		r >= 'A' && r <= 'Z' ||
+		r >= '0' && r <= '9' ||
+		r == '_')
+}
+
+// NewCounter returns a new metric that can only increment.
+func NewCounter(name string) *Metric {
+	m := NewUnpublished(name, TypeCounter)
+	m.Publish()
+	return m
+}
+
+// NewGauge returns a new metric that can both increment and decrement.
+func NewGauge(name string) *Metric {
+	m := NewUnpublished(name, TypeGauge)
+	m.Publish()
+	return m
+}
+
+// WritePrometheusExpositionFormat writes all client metrics to w in
+// the Prometheus text-based exposition format.
+//
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md
+func WritePrometheusExpositionFormat(w io.Writer) {
+	for _, m := range Metrics() {
+		switch m.Type() {
+		case TypeGauge:
+			fmt.Fprintf(w, "# TYPE %s gauge\n", m.Name())
+		case TypeCounter:
+			fmt.Fprintf(w, "# TYPE %s counter\n", m.Name())
+		}
+		fmt.Fprintf(w, "%s %v\n", m.Name(), m.Value())
+	}
+}
+
+const (
+	// metricLogNameFrequency is how often a metric's name=>id
+	// mapping is redundantly put in the logs. In other words,
+	// this is how how far in the logs you need to fetch from a
+	// given point in time to recompute the metrics at that point
+	// in time.
+	metricLogNameFrequency = 4 * time.Hour
+
+	// minMetricEncodeInterval is the minimum interval that the
+	// metrics will be scanned for changes before being encoded
+	// for logtail.
+	minMetricEncodeInterval = 15 * time.Second
+)
+
+// EncodeLogTailMetricsDelta return an encoded string representing the metrics
+// differences since the previous call.
+//
+// It implements the requirements of a logtail.Config.MetricsDelta
+// func. Notably, its output is safe to embed in a JSON string literal
+// without further escaping.
+//
+// The current encoding is:
+//   * name immediately following metric:
+//     'N' + hex(varint(len(name))) + name
+//   * set value of a metric:
+//     'S' + hex(varint(wireid)) + hex(varint(value))
+//   * increment a metric: (decrements if negative)
+//     'I' + hex(varint(wireid)) + hex(varint(value))
+func EncodeLogTailMetricsDelta() string {
+	mu.Lock()
+	defer mu.Unlock()
+
+	now := time.Now()
+	if !lastDelta.IsZero() && now.Sub(lastDelta) < minMetricEncodeInterval {
+		return ""
+	}
+	lastDelta = now
+
+	var enc *deltaEncBuf // lazy
+	for i, ent := range lastLogVal {
+		val := atomic.LoadInt64(ent.v)
+		delta := val - ent.lastLogged
+		if delta == 0 {
+			continue
+		}
+		lastLogVal[i].lastLogged = val
+		m := unsorted[i]
+		if enc == nil {
+			enc = deltaPool.Get().(*deltaEncBuf)
+			enc.buf.Reset()
+		}
+		if m.wireID == 0 {
+			numWireID++
+			m.wireID = numWireID
+		}
+		if m.lastNamed.IsZero() || now.Sub(m.lastNamed) > metricLogNameFrequency {
+			enc.writeName(m.Name())
+			m.lastNamed = now
+			enc.writeValue(m.wireID, val)
+		} else {
+			enc.writeDelta(m.wireID, delta)
+		}
+	}
+	if enc == nil {
+		return ""
+	}
+	defer deltaPool.Put(enc)
+	return enc.buf.String()
+}
+
+var deltaPool = &sync.Pool{
+	New: func() interface{} {
+		return new(deltaEncBuf)
+	},
+}
+
+// deltaEncBuf encodes metrics per the format described
+// on EncodeLogTailMetricsDelta above.
+type deltaEncBuf struct {
+	buf     bytes.Buffer
+	scratch [binary.MaxVarintLen64]byte
+}
+
+// writeName writes a "name" (N) record to the buffer, which notes
+// that the immediately following record's wireID has the provided
+// name.
+func (b *deltaEncBuf) writeName(name string) {
+	b.buf.WriteByte('N')
+	b.writeHexVarint(int64(len(name)))
+	b.buf.WriteString(name)
+}
+
+// writeDelta writes a "set" (S) record to the buffer, noting that the
+// metric with the given wireID now has value v.
+func (b *deltaEncBuf) writeValue(wireID int, v int64) {
+	b.buf.WriteByte('S')
+	b.writeHexVarint(int64(wireID))
+	b.writeHexVarint(v)
+}
+
+// writeDelta writes an "increment" (I) delta value record to the
+// buffer, noting that the metric with the given wireID now has a
+// value that's v larger (or smaller if v is negative).
+func (b *deltaEncBuf) writeDelta(wireID int, v int64) {
+	b.buf.WriteByte('I')
+	b.writeHexVarint(int64(wireID))
+	b.writeHexVarint(v)
+}
+
+// writeHexVarint writes v to the buffer as a hex-encoded varint.
+func (b *deltaEncBuf) writeHexVarint(v int64) {
+	n := binary.PutVarint(b.scratch[:], v)
+	hexLen := n * 2
+	oldLen := b.buf.Len()
+	b.buf.Grow(hexLen)
+	hexBuf := b.buf.Bytes()[oldLen : oldLen+hexLen]
+	hex.Encode(hexBuf, b.scratch[:n])
+	b.buf.Write(hexBuf)
+}

util/deephash/deephash_test.go

@@ -15,6 +15,7 @@ import (
 	"runtime"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/dnstype"
@@ -163,11 +164,11 @@ func getVal() []interface{} {
 			dnsname.FQDN("d."): {netaddr.MustParseIPPort("8.8.8.8:13"), netaddr.MustParseIPPort("9.9.9.9:24")},
 			dnsname.FQDN("e."): {netaddr.MustParseIPPort("8.8.8.8:14"), netaddr.MustParseIPPort("9.9.9.9:25")},
 		},
-		map[tailcfg.DiscoKey]bool{
-			{1: 1}: true,
-			{1: 2}: false,
-			{2: 3}: true,
-			{3: 4}: false,
+		map[key.DiscoPublic]bool{
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 31: 0})): true,
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 2, 31: 0})): false,
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 3, 31: 0})): true,
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 4, 31: 0})): false,
 		},
 		&tailcfg.MapResponse{
 			DERPMap: &tailcfg.DERPMap{

util/multierr/multierr.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package multierr provides a simple multiple-error type.
+// It was inspired by github.com/go-multierror/multierror.
+package multierr
+
+import (
+	"errors"
+	"strings"
+)
+
+// An Error represents multiple errors.
+type Error struct {
+	errs []error
+}
+
+// Error implements the error interface.
+func (e Error) Error() string {
+	s := new(strings.Builder)
+	s.WriteString("multiple errors:")
+	for _, err := range e.errs {
+		s.WriteString("\n\t")
+		s.WriteString(err.Error())
+	}
+	return s.String()
+}
+
+// Errors returns a slice containing all errors in e.
+func (e Error) Errors() []error {
+	return append(e.errs[:0:0], e.errs...)
+}
+
+// New returns an error composed from errs.
+// Some errors in errs get special treatment:
+//   * nil errors are discarded
+//   * errors of type Error are expanded into the top level
+// If the resulting slice has length 0, New returns nil.
+// If the resulting slice has length 1, New returns that error.
+// If the resulting slice has length > 1, New returns that slice as an Error.
+func New(errs ...error) error {
+	dst := make([]error, 0, len(errs))
+	for _, e := range errs {
+		switch e := e.(type) {
+		case nil:
+			continue
+		case Error:
+			dst = append(dst, e.errs...)
+		default:
+			dst = append(dst, e)
+		}
+	}
+	// dst has been filtered and splatted.
+	switch len(dst) {
+	case 0:
+		return nil
+	case 1:
+		return dst[0]
+	}
+	// Zero any remaining elements of dst left over from errs, for GC.
+	tail := dst[len(dst):cap(dst)]
+	for i := range tail {
+		tail[i] = nil
+	}
+	return Error{errs: dst}
+}
+
+// Is reports whether any error in e matches target.
+func (e Error) Is(target error) bool {
+	for _, err := range e.errs {
+		if errors.Is(err, target) {
+			return true
+		}
+	}
+	return false
+}
+
+// As finds the first error in e that matches target, and if any is found,
+// sets target to that error value and returns true. Otherwise, it returns false.
+func (e Error) As(target interface{}) bool {
+	for _, err := range e.errs {
+		if ok := errors.As(err, target); ok {
+			return true
+		}
+	}
+	return false
+}

util/multierr/multierr_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package multierr_test
+
+import (
+	"errors"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"tailscale.com/util/multierr"
+)
+
+func TestAll(t *testing.T) {
+	C := qt.New(t)
+	eqErr := qt.CmpEquals(cmpopts.EquateErrors())
+
+	type E = []error
+	N := multierr.New
+
+	a := errors.New("a")
+	b := errors.New("b")
+	c := errors.New("c")
+	d := errors.New("d")
+	x := errors.New("x")
+	abcd := E{a, b, c, d}
+
+	tests := []struct {
+		In         E       // input to New
+		WantNil    bool    // want nil returned?
+		WantSingle error   // if non-nil, want this single error returned
+		WantErrors []error // if non-nil, want an Error composed of these errors returned
+	}{
+		{In: nil, WantNil: true},
+
+		{In: E{nil}, WantNil: true},
+		{In: E{nil, nil}, WantNil: true},
+
+		{In: E{a}, WantSingle: a},
+		{In: E{a, nil}, WantSingle: a},
+		{In: E{nil, a}, WantSingle: a},
+		{In: E{nil, a, nil}, WantSingle: a},
+
+		{In: E{a, b}, WantErrors: E{a, b}},
+		{In: E{nil, a, nil, b, nil}, WantErrors: E{a, b}},
+
+		{In: E{a, b, N(c, d)}, WantErrors: E{a, b, c, d}},
+		{In: E{a, N(b, c), d}, WantErrors: E{a, b, c, d}},
+		{In: E{N(a, b), c, d}, WantErrors: E{a, b, c, d}},
+		{In: E{N(a, b), N(c, d)}, WantErrors: E{a, b, c, d}},
+		{In: E{nil, N(a, nil, b), nil, N(c, d)}, WantErrors: E{a, b, c, d}},
+
+		{In: E{N(a, N(b, N(c, N(d))))}, WantErrors: E{a, b, c, d}},
+		{In: E{N(N(N(N(a), b), c), d)}, WantErrors: E{a, b, c, d}},
+
+		{In: E{N(abcd...)}, WantErrors: E{a, b, c, d}},
+		{In: E{N(abcd...), N(abcd...)}, WantErrors: E{a, b, c, d, a, b, c, d}},
+	}
+
+	for _, test := range tests {
+		got := multierr.New(test.In...)
+		if test.WantNil {
+			C.Assert(got, qt.IsNil)
+			continue
+		}
+		if test.WantSingle != nil {
+			C.Assert(got, eqErr, test.WantSingle)
+			continue
+		}
+		ee, _ := got.(multierr.Error)
+		C.Assert(ee.Errors(), eqErr, test.WantErrors)
+
+		for _, e := range test.WantErrors {
+			C.Assert(ee.Is(e), qt.IsTrue)
+		}
+		C.Assert(ee.Is(x), qt.IsFalse)
+	}
+}

version/version.go

@@ -14,7 +14,7 @@ import (
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-var Long = "date.20211022"
+var Long = "date.20211101"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.

wgengine/bench/wg.go

@@ -105,7 +105,7 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netadd
 			Endpoints:  eps,
 		}
 		e2.SetNetworkMap(&netmap.NetworkMap{
-			NodeKey:    tailcfg.NodeKeyFromNodePublic(k2.Public()),
+			NodeKey:    k2.Public(),
 			PrivateKey: k2,
 			Peers:      []*tailcfg.Node{&n},
 		})
@@ -142,7 +142,7 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netadd
 			Endpoints:  eps,
 		}
 		e1.SetNetworkMap(&netmap.NetworkMap{
-			NodeKey:    tailcfg.NodeKeyFromNodePublic(k1.Public()),
+			NodeKey:    k1.Public(),
 			PrivateKey: k1,
 			Peers:      []*tailcfg.Node{&n},
 		})

wgengine/magicsock/magicsock.go

@@ -51,6 +51,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/nettype"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/uniq"
 	"tailscale.com/version"
 	"tailscale.com/wgengine/monitor"
@@ -92,19 +93,19 @@ func newPeerInfo(ep *endpoint) *peerInfo {
 //
 // Doesn't do any locking, all access must be done with Conn.mu held.
 type peerMap struct {
-	byNodeKey map[tailcfg.NodeKey]*peerInfo
+	byNodeKey map[key.NodePublic]*peerInfo
 	byIPPort  map[netaddr.IPPort]*peerInfo
 
-	// nodesOfDisco are contains the set of nodes that are using a
+	// nodesOfDisco contains the set of nodes that are using a
 	// DiscoKey. Usually those sets will be just one node.
-	nodesOfDisco map[key.DiscoPublic]map[tailcfg.NodeKey]bool
+	nodesOfDisco map[key.DiscoPublic]map[key.NodePublic]bool
 }
 
 func newPeerMap() peerMap {
 	return peerMap{
-		byNodeKey:    map[tailcfg.NodeKey]*peerInfo{},
+		byNodeKey:    map[key.NodePublic]*peerInfo{},
 		byIPPort:     map[netaddr.IPPort]*peerInfo{},
-		nodesOfDisco: map[key.DiscoPublic]map[tailcfg.NodeKey]bool{},
+		nodesOfDisco: map[key.DiscoPublic]map[key.NodePublic]bool{},
 	}
 }
 
@@ -121,7 +122,7 @@ func (m *peerMap) anyEndpointForDiscoKey(dk key.DiscoPublic) bool {
 
 // endpointForNodeKey returns the endpoint for nk, or nil if
 // nk is not known to us.
-func (m *peerMap) endpointForNodeKey(nk tailcfg.NodeKey) (ep *endpoint, ok bool) {
+func (m *peerMap) endpointForNodeKey(nk key.NodePublic) (ep *endpoint, ok bool) {
 	if nk.IsZero() {
 		return nil, false
 	}
@@ -167,22 +168,17 @@ func (m *peerMap) forEachEndpointWithDiscoKey(dk key.DiscoPublic, f func(ep *end
 // upsertEndpoint stores endpoint in the peerInfo for
 // ep.publicKey, and updates indexes. m must already have a
 // tailcfg.Node for ep.publicKey.
-func (m *peerMap) upsertEndpoint(ep *endpoint) {
-	pi := m.byNodeKey[ep.publicKey]
-	if pi == nil {
-		pi = newPeerInfo(ep)
-		m.byNodeKey[ep.publicKey] = pi
-	} else {
-		old := pi.ep
-		pi.ep = ep
-		if old.discoKey != ep.discoKey {
-			delete(m.nodesOfDisco[old.discoKey], ep.publicKey)
-		}
+func (m *peerMap) upsertEndpoint(ep *endpoint, oldDiscoKey key.DiscoPublic) {
+	if m.byNodeKey[ep.publicKey] == nil {
+		m.byNodeKey[ep.publicKey] = newPeerInfo(ep)
+	}
+	if oldDiscoKey != ep.discoKey {
+		delete(m.nodesOfDisco[oldDiscoKey], ep.publicKey)
 	}
 	if !ep.discoKey.IsZero() {
 		set := m.nodesOfDisco[ep.discoKey]
 		if set == nil {
-			set = map[tailcfg.NodeKey]bool{}
+			set = map[key.NodePublic]bool{}
 			m.nodesOfDisco[ep.discoKey] = set
 		}
 		set[ep.publicKey] = true
@@ -195,7 +191,7 @@ func (m *peerMap) upsertEndpoint(ep *endpoint) {
 // This should only be called with a fully verified mapping of ipp to
 // nk, because calling this function defines the endpoint we hand to
 // WireGuard for packets received from ipp.
-func (m *peerMap) setNodeKeyForIPPort(ipp netaddr.IPPort, nk tailcfg.NodeKey) {
+func (m *peerMap) setNodeKeyForIPPort(ipp netaddr.IPPort, nk key.NodePublic) {
 	if pi := m.byIPPort[ipp]; pi != nil {
 		delete(pi.ipPorts, ipp)
 		delete(m.byIPPort, ipp)
@@ -237,7 +233,7 @@ type Conn struct {
 	derpActiveFunc         func()
 	idleFunc               func() time.Duration // nil means unknown
 	testOnlyPacketListener nettype.PacketListener
-	noteRecvActivity       func(tailcfg.NodeKey) // or nil, see Options.NoteRecvActivity
+	noteRecvActivity       func(key.NodePublic) // or nil, see Options.NoteRecvActivity
 
 	// ================================================================
 	// No locking required to access these fields, either because
@@ -299,7 +295,7 @@ type Conn struct {
 
 	// havePrivateKey is whether privateKey is non-zero.
 	havePrivateKey  syncs.AtomicBool
-	publicKeyAtomic atomic.Value // of tailcfg.NodeKey (or NodeKey zero value if !havePrivateKey)
+	publicKeyAtomic atomic.Value // of key.NodePublic (or NodeKey zero value if !havePrivateKey)
 
 	// port is the preferred port from opts.Port; 0 means auto.
 	port syncs.AtomicUint32
@@ -492,7 +488,7 @@ type Options struct {
 	// The provided func is likely to call back into
 	// Conn.ParseEndpoint, which acquires Conn.mu. As such, you should
 	// not hold Conn.mu while calling it.
-	NoteRecvActivity func(tailcfg.NodeKey)
+	NoteRecvActivity func(key.NodePublic)
 
 	// LinkMonitor is the link monitor to use.
 	// With one, the portmapper won't be used.
@@ -841,7 +837,7 @@ func (c *Conn) callNetInfoCallbackLocked(ni *tailcfg.NetInfo) {
 // discoKey. It's used in tests to enable receiving of packets from
 // addr without having to spin up the entire active discovery
 // machinery.
-func (c *Conn) addValidDiscoPathForTest(nodeKey tailcfg.NodeKey, addr netaddr.IPPort) {
+func (c *Conn) addValidDiscoPathForTest(nodeKey key.NodePublic, addr netaddr.IPPort) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.peerMap.setNodeKeyForIPPort(addr, nodeKey)
@@ -863,7 +859,7 @@ func (c *Conn) SetNetInfoCallback(fn func(*tailcfg.NetInfo)) {
 
 // LastRecvActivityOfNodeKey describes the time we last got traffic from
 // this endpoint (updated every ~10 seconds).
-func (c *Conn) LastRecvActivityOfNodeKey(nk tailcfg.NodeKey) string {
+func (c *Conn) LastRecvActivityOfNodeKey(nk key.NodePublic) string {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	de, ok := c.peerMap.endpointForNodeKey(nk)
@@ -944,7 +940,7 @@ func (c *Conn) DiscoPublicKey() key.DiscoPublic {
 }
 
 // PeerHasDiscoKey reports whether peer k supports discovery keys (client version 0.100.0+).
-func (c *Conn) PeerHasDiscoKey(k tailcfg.NodeKey) bool {
+func (c *Conn) PeerHasDiscoKey(k key.NodePublic) bool {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if ep, ok := c.peerMap.endpointForNodeKey(k); ok {
@@ -966,6 +962,9 @@ func (c *Conn) setNearestDERP(derpNum int) (wantDERP bool) {
 		// No change.
 		return true
 	}
+	if c.myDerp != 0 && derpNum != 0 {
+		metricDERPHomeChange.Add(1)
+	}
 	c.myDerp = derpNum
 	health.SetMagicSockDERPHome(derpNum)
 
@@ -1167,7 +1166,9 @@ var errNetworkDown = errors.New("magicsock: network down")
 func (c *Conn) networkDown() bool { return !c.networkUp.Get() }
 
 func (c *Conn) Send(b []byte, ep conn.Endpoint) error {
+	metricSendData.Add(1)
 	if c.networkDown() {
+		metricSendDataNetworkDown.Add(1)
 		return errNetworkDown
 	}
 	return ep.(*endpoint).send(b)
@@ -1191,9 +1192,14 @@ func (c *Conn) sendUDP(ipp netaddr.IPPort, b []byte) (sent bool, err error) {
 	}
 	ua := udpAddrPool.Get().(*net.UDPAddr)
 	sent, err = c.sendUDPStd(ipp.UDPAddrAt(ua), b)
-	if err == nil {
+	if err != nil {
+		metricSendUDPError.Add(1)
+	} else {
 		// Only return it to the pool on success; Issue 3122.
 		udpAddrPool.Put(ua)
+		if sent {
+			metricSendUDP.Add(1)
+		}
 	}
 	return
 }
@@ -1239,6 +1245,7 @@ func (c *Conn) sendAddr(addr netaddr.IPPort, pubKey key.NodePublic, b []byte) (s
 
 	ch := c.derpWriteChanOfAddr(addr, pubKey)
 	if ch == nil {
+		metricSendDERPErrorChan.Add(1)
 		return false, nil
 	}
 
@@ -1252,10 +1259,13 @@ func (c *Conn) sendAddr(addr netaddr.IPPort, pubKey key.NodePublic, b []byte) (s
 
 	select {
 	case <-c.donec:
+		metricSendDERPErrorClosed.Add(1)
 		return false, errConnClosed
 	case ch <- derpWriteRequest{addr, pubKey, pkt}:
+		metricSendDERPQueued.Add(1)
 		return true, nil
 	default:
+		metricSendDERPErrorQueue.Add(1)
 		// Too many writes queued. Drop packet.
 		return false, errDropDerpPacket
 	}
@@ -1367,6 +1377,7 @@ func (c *Conn) derpWriteChanOfAddr(addr netaddr.IPPort, peer key.NodePublic) cha
 	*ad.lastWrite = time.Now()
 	ad.createTime = time.Now()
 	c.activeDerp[regionID] = ad
+	metricNumDERPConns.Set(int64(len(c.activeDerp)))
 	c.logActiveDerpLocked()
 	c.setPeerLastDerpLocked(peer, regionID, regionID)
 	c.scheduleCleanStaleDerpLocked()
@@ -1603,6 +1614,9 @@ func (c *Conn) runDerpWriter(ctx context.Context, dc *derphttp.Client, ch <-chan
 			err := dc.Send(wr.pubKey, wr.b)
 			if err != nil {
 				c.logf("magicsock: derp.Send(%v): %v", wr.addr, err)
+				metricSendDERPError.Add(1)
+			} else {
+				metricSendDERP.Add(1)
 			}
 		}
 	}
@@ -1618,6 +1632,7 @@ func (c *Conn) receiveIPv6(b []byte) (int, conn.Endpoint, error) {
 			return 0, nil, err
 		}
 		if ep, ok := c.receiveIP(b[:n], ipp, &c.ippEndpoint6); ok {
+			metricRecvDataIPv6.Add(1)
 			return n, ep, nil
 		}
 	}
@@ -1633,6 +1648,7 @@ func (c *Conn) receiveIPv4(b []byte) (n int, ep conn.Endpoint, err error) {
 			return 0, nil, err
 		}
 		if ep, ok := c.receiveIP(b[:n], ipp, &c.ippEndpoint4); ok {
+			metricRecvDataIPv4.Add(1)
 			return n, ep, nil
 		}
 	}
@@ -1647,7 +1663,7 @@ func (c *Conn) receiveIP(b []byte, ipp netaddr.IPPort, cache *ippEndpointCache)
 		c.stunReceiveFunc.Load().(func([]byte, netaddr.IPPort))(b, ipp)
 		return nil, false
 	}
-	if c.handleDiscoMessage(b, ipp, tailcfg.NodeKey{}) {
+	if c.handleDiscoMessage(b, ipp, key.NodePublic{}) {
 		return nil, false
 	}
 	if !c.havePrivateKey.Get() {
@@ -1691,6 +1707,7 @@ func (c *connBind) receiveDERP(b []byte) (n int, ep conn.Endpoint, err error) {
 			// No data read occurred. Wait for another packet.
 			continue
 		}
+		metricRecvDataDERP.Add(1)
 		return n, ep, nil
 	}
 	return 0, nil, net.ErrClosed
@@ -1710,13 +1727,13 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en
 	}
 
 	ipp := netaddr.IPPortFrom(derpMagicIPAddr, uint16(regionID))
-	if c.handleDiscoMessage(b[:n], ipp, tailcfg.NodeKeyFromNodePublic(dm.src)) {
+	if c.handleDiscoMessage(b[:n], ipp, dm.src) {
 		return 0, nil
 	}
 
 	var ok bool
 	c.mu.Lock()
-	ep, ok = c.peerMap.endpointForNodeKey(tailcfg.NodeKeyFromNodePublic(dm.src))
+	ep, ok = c.peerMap.endpointForNodeKey(dm.src)
 	c.mu.Unlock()
 	if !ok {
 		// We don't know anything about this node key, nothing to
@@ -1746,7 +1763,7 @@ const (
 //
 // The dstKey should only be non-zero if the dstDisco key
 // unambiguously maps to exactly one peer.
-func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey tailcfg.NodeKey, dstDisco key.DiscoPublic, m disco.Message, logLevel discoLogLevel) (sent bool, err error) {
+func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey key.NodePublic, dstDisco key.DiscoPublic, m disco.Message, logLevel discoLogLevel) (sent bool, err error) {
 	c.mu.Lock()
 	if c.closed {
 		c.mu.Unlock()
@@ -1762,9 +1779,16 @@ func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey tailcfg.NodeKey, dstD
 	di := c.discoInfoLocked(dstDisco)
 	c.mu.Unlock()
 
+	isDERP := dst.IP() == derpMagicIPAddr
+	if isDERP {
+		metricSendDiscoDERP.Add(1)
+	} else {
+		metricSendDiscoUDP.Add(1)
+	}
+
 	box := di.sharedKey.Seal(m.AppendMarshal(nil))
 	pkt = append(pkt, box...)
-	sent, err = c.sendAddr(dst, key.NodePublicFromRaw32(mem.B(dstKey[:])), pkt)
+	sent, err = c.sendAddr(dst, dstKey, pkt)
 	if sent {
 		if logLevel == discoLog || (logLevel == discoVerboseLog && debugDisco) {
 			node := "?"
@@ -1773,6 +1797,11 @@ func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey tailcfg.NodeKey, dstD
 			}
 			c.logf("[v1] magicsock: disco: %v->%v (%v, %v) sent %v", c.discoShort, dstDisco.ShortString(), node, derpStr(dst.String()), disco.MessageSummary(m))
 		}
+		if isDERP {
+			metricSentDiscoDERP.Add(1)
+		} else {
+			metricSentDiscoUDP.Add(1)
+		}
 	} else if err == nil {
 		// Can't send. (e.g. no IPv6 locally)
 	} else {
@@ -1797,8 +1826,8 @@ func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey tailcfg.NodeKey, dstD
 // src.Port() being the region ID) and the derpNodeSrc will be the node key
 // it was received from at the DERP layer. derpNodeSrc is zero when received
 // over UDP.
-func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc tailcfg.NodeKey) (isDiscoMsg bool) {
-	headerLen := len(disco.Magic) + key.DiscoPublic{}.RawLen()
+func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc key.NodePublic) (isDiscoMsg bool) {
+	const headerLen = len(disco.Magic) + key.DiscoPublicRawLen
 	if len(msg) < headerLen || string(msg[:len(disco.Magic)]) != disco.Magic {
 		return false
 	}
@@ -1809,7 +1838,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 	// Use naked returns for all following paths.
 	isDiscoMsg = true
 
-	sender := key.DiscoPublicFromRaw32(mem.B(msg[len(disco.Magic) : len(disco.Magic)+key.DiscoPublic{}.RawLen()]))
+	sender := key.DiscoPublicFromRaw32(mem.B(msg[len(disco.Magic):headerLen]))
 
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -1833,6 +1862,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 	}
 
 	if !c.peerMap.anyEndpointForDiscoKey(sender) {
+		metricRecvDiscoBadPeer.Add(1)
 		if debugDisco {
 			c.logf("magicsock: disco: ignoring disco-looking frame, don't know endpoint for %v", sender.ShortString())
 		}
@@ -1860,7 +1890,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 		if debugDisco {
 			c.logf("magicsock: disco: failed to open naclbox from %v (wrong rcpt?)", sender)
 		}
-		// TODO(bradfitz): add some counter for this that logs rarely
+		metricRecvDiscoBadKey.Add(1)
 		return
 	}
 
@@ -1874,14 +1904,23 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 		// newer version of Tailscale that we don't
 		// understand. Not even worth logging about, lest it
 		// be too spammy for old clients.
-		// TODO(bradfitz): add some counter for this that logs rarely
+		metricRecvDiscoBadParse.Add(1)
 		return
 	}
 
+	isDERP := src.IP() == derpMagicIPAddr
+	if isDERP {
+		metricRecvDiscoDERP.Add(1)
+	} else {
+		metricRecvDiscoUDP.Add(1)
+	}
+
 	switch dm := dm.(type) {
 	case *disco.Ping:
+		metricRecvDiscoPing.Add(1)
 		c.handlePingLocked(dm, src, di, derpNodeSrc)
 	case *disco.Pong:
+		metricRecvDiscoPong.Add(1)
 		// There might be multiple nodes for the sender's DiscoKey.
 		// Ask each to handle it, stopping once one reports that
 		// the Pong's TxID was theirs.
@@ -1892,14 +1931,16 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 			}
 		})
 	case *disco.CallMeMaybe:
-		if src.IP() != derpMagicIPAddr || derpNodeSrc.IsZero() {
+		metricRecvDiscoCallMeMaybe.Add(1)
+		if !isDERP || derpNodeSrc.IsZero() {
 			// CallMeMaybe messages should only come via DERP.
 			c.logf("[unexpected] CallMeMaybe packets should only come via DERP")
 			return
 		}
-		nodeKey := tailcfg.NodeKey(derpNodeSrc)
+		nodeKey := derpNodeSrc
 		ep, ok := c.peerMap.endpointForNodeKey(nodeKey)
 		if !ok {
+			metricRecvDiscoCallMeMaybeBadNode.Add(1)
 			c.logf("magicsock: disco: ignoring CallMeMaybe from %v; %v is unknown", sender.ShortString(), derpNodeSrc.ShortString())
 			return
 		}
@@ -1907,6 +1948,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 			return
 		}
 		if ep.discoKey != di.discoKey {
+			metricRecvDiscoCallMeMaybeBadDisco.Add(1)
 			c.logf("[unexpected] CallMeMaybe from peer via DERP whose netmap discokey != disco source")
 			return
 		}
@@ -1927,7 +1969,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort, derpNodeSrc ta
 // derpNodeSrc is non-zero if the disco ping arrived via DERP.
 //
 // c.mu must be held.
-func (c *Conn) unambiguousNodeKeyOfPingLocked(dm *disco.Ping, dk key.DiscoPublic, derpNodeSrc tailcfg.NodeKey) (nk tailcfg.NodeKey, ok bool) {
+func (c *Conn) unambiguousNodeKeyOfPingLocked(dm *disco.Ping, dk key.DiscoPublic, derpNodeSrc key.NodePublic) (nk key.NodePublic, ok bool) {
 	if !derpNodeSrc.IsZero() {
 		if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok && ep.discoKey == dk {
 			return derpNodeSrc, true
@@ -1954,7 +1996,7 @@ func (c *Conn) unambiguousNodeKeyOfPingLocked(dm *disco.Ping, dk key.DiscoPublic
 
 // di is the discoInfo of the source of the ping.
 // derpNodeSrc is non-zero if the ping arrived via DERP.
-func (c *Conn) handlePingLocked(dm *disco.Ping, src netaddr.IPPort, di *discoInfo, derpNodeSrc tailcfg.NodeKey) {
+func (c *Conn) handlePingLocked(dm *disco.Ping, src netaddr.IPPort, di *discoInfo, derpNodeSrc key.NodePublic) {
 	likelyHeartBeat := src == di.lastPingFrom && time.Since(di.lastPingTime) < 5*time.Second
 	di.lastPingFrom = src
 	di.lastPingTime = time.Now()
@@ -1999,7 +2041,7 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netaddr.IPPort, di *discoInf
 		if numNodes > 1 {
 			// Zero it out if it's ambiguous, so sendDiscoMessage logging
 			// isn't confusing.
-			dstKey = tailcfg.NodeKey{}
+			dstKey = key.NodePublic{}
 		}
 	}
 
@@ -2060,7 +2102,7 @@ func (c *Conn) enqueueCallMeMaybe(derpAddr netaddr.IPPort, de *endpoint) {
 	for _, ep := range c.lastEndpoints {
 		eps = append(eps, ep.Addr)
 	}
-	go de.sendDiscoMessage(derpAddr, &disco.CallMeMaybe{MyNumber: eps}, discoLog)
+	go de.c.sendDiscoMessage(derpAddr, de.publicKey, de.discoKey, &disco.CallMeMaybe{MyNumber: eps}, discoLog)
 }
 
 // discoInfoLocked returns the previous or new discoInfo for k.
@@ -2130,9 +2172,9 @@ func (c *Conn) SetPrivateKey(privateKey key.NodePrivate) error {
 	c.havePrivateKey.Set(!newKey.IsZero())
 
 	if newKey.IsZero() {
-		c.publicKeyAtomic.Store(tailcfg.NodeKey{})
+		c.publicKeyAtomic.Store(key.NodePublic{})
 	} else {
-		c.publicKeyAtomic.Store(tailcfg.NodeKeyFromNodePublic(newKey.Public()))
+		c.publicKeyAtomic.Store(newKey.Public())
 	}
 
 	if oldKey.IsZero() {
@@ -2244,6 +2286,8 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 		}
 	}
 
+	metricNumPeers.Set(int64(len(nm.Peers)))
+
 	c.logf("[v1] magicsock: got updated network map; %d peers", len(nm.Peers))
 	if numNoDisco != 0 {
 		c.logf("[v1] magicsock: %d DERP-only peers (no discokey)", numNoDisco)
@@ -2257,8 +2301,9 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 	// handle full set updates.
 	for _, n := range nm.Peers {
 		if ep, ok := c.peerMap.endpointForNodeKey(n.Key); ok {
+			oldDiscoKey := ep.discoKey
 			ep.updateFromNode(n)
-			c.peerMap.upsertEndpoint(ep) // maybe update discokey mappings in peerMap
+			c.peerMap.upsertEndpoint(ep, oldDiscoKey) // maybe update discokey mappings in peerMap
 			continue
 		}
 
@@ -2269,36 +2314,38 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 			endpointState: map[netaddr.IPPort]*endpointState{},
 		}
 		if !n.DiscoKey.IsZero() {
-			ep.discoKey = key.DiscoPublicFromRaw32(mem.B(n.DiscoKey[:]))
+			ep.discoKey = n.DiscoKey
 			ep.discoShort = n.DiscoKey.ShortString()
 		}
-		ep.wgEndpoint = key.NodePublicFromRaw32(mem.B(n.Key[:])).UntypedHexString()
+		ep.wgEndpoint = n.Key.UntypedHexString()
 		ep.initFakeUDPAddr()
-		c.logf("magicsock: created endpoint key=%s: disco=%s; %v", n.Key.ShortString(), n.DiscoKey.ShortString(), logger.ArgWriter(func(w *bufio.Writer) {
-			const derpPrefix = "127.3.3.40:"
-			if strings.HasPrefix(n.DERP, derpPrefix) {
-				ipp, _ := netaddr.ParseIPPort(n.DERP)
-				regionID := int(ipp.Port())
-				code := c.derpRegionCodeLocked(regionID)
-				if code != "" {
-					code = "(" + code + ")"
+		if debugDisco { // rather than making a new knob
+			c.logf("magicsock: created endpoint key=%s: disco=%s; %v", n.Key.ShortString(), n.DiscoKey.ShortString(), logger.ArgWriter(func(w *bufio.Writer) {
+				const derpPrefix = "127.3.3.40:"
+				if strings.HasPrefix(n.DERP, derpPrefix) {
+					ipp, _ := netaddr.ParseIPPort(n.DERP)
+					regionID := int(ipp.Port())
+					code := c.derpRegionCodeLocked(regionID)
+					if code != "" {
+						code = "(" + code + ")"
+					}
+					fmt.Fprintf(w, "derp=%v%s ", regionID, code)
 				}
-				fmt.Fprintf(w, "derp=%v%s ", regionID, code)
-			}
 
-			for _, a := range n.AllowedIPs {
-				if a.IsSingleIP() {
-					fmt.Fprintf(w, "aip=%v ", a.IP())
-				} else {
-					fmt.Fprintf(w, "aip=%v ", a)
+				for _, a := range n.AllowedIPs {
+					if a.IsSingleIP() {
+						fmt.Fprintf(w, "aip=%v ", a.IP())
+					} else {
+						fmt.Fprintf(w, "aip=%v ", a)
+					}
 				}
-			}
-			for _, ep := range n.Endpoints {
-				fmt.Fprintf(w, "ep=%v ", ep)
-			}
-		}))
+				for _, ep := range n.Endpoints {
+					fmt.Fprintf(w, "ep=%v ", ep)
+				}
+			}))
+		}
 		ep.updateFromNode(n)
-		c.peerMap.upsertEndpoint(ep)
+		c.peerMap.upsertEndpoint(ep, key.DiscoPublic{})
 	}
 
 	// If the set of nodes changed since the last SetNetworkMap, the
@@ -2307,7 +2354,7 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 	// current netmap. If that happens, go through the allocful
 	// deletion path to clean up moribund nodes.
 	if c.peerMap.nodeCount() != len(nm.Peers) {
-		keep := make(map[tailcfg.NodeKey]bool, len(nm.Peers))
+		keep := make(map[key.NodePublic]bool, len(nm.Peers))
 		for _, n := range nm.Peers {
 			keep[n.Key] = true
 		}
@@ -2347,6 +2394,7 @@ func (c *Conn) closeDerpLocked(node int, why string) {
 		go ad.c.Close()
 		ad.cancel()
 		delete(c.activeDerp, node)
+		metricNumDERPConns.Set(int64(len(c.activeDerp)))
 	}
 }
 
@@ -2818,19 +2866,18 @@ func (c *Conn) ParseEndpoint(nodeKeyStr string) (conn.Endpoint, error) {
 	if err != nil {
 		return nil, fmt.Errorf("magicsock: ParseEndpoint: parse failed on %q: %w", nodeKeyStr, err)
 	}
-	pk := tailcfg.NodeKeyFromNodePublic(k)
 
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.closed {
 		return nil, errConnClosed
 	}
-	ep, ok := c.peerMap.endpointForNodeKey(pk)
+	ep, ok := c.peerMap.endpointForNodeKey(k)
 	if !ok {
 		// We should never be telling WireGuard about a new peer
 		// before magicsock knows about it.
-		c.logf("[unexpected] magicsock: ParseEndpoint: unknown node key=%s", pk.ShortString())
-		return nil, fmt.Errorf("magicsock: ParseEndpoint: unknown peer %q", pk.ShortString())
+		c.logf("[unexpected] magicsock: ParseEndpoint: unknown node key=%s", k.ShortString())
+		return nil, fmt.Errorf("magicsock: ParseEndpoint: unknown peer %q", k.ShortString())
 	}
 
 	return ep, nil
@@ -3108,7 +3155,7 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 		ps := &ipnstate.PeerStatus{InMagicSock: true}
 		//ps.Addrs = append(ps.Addrs, n.Endpoints...)
 		ep.populatePeerStatus(ps)
-		sb.AddPeer(key.NodePublicFromRaw32(mem.B(ep.publicKey[:])), ps)
+		sb.AddPeer(ep.publicKey, ps)
 	})
 
 	c.foreachActiveDerpSortedLocked(func(node int, ad activeDerp) {
@@ -3134,9 +3181,9 @@ type endpoint struct {
 
 	// These fields are initialized once and never modified.
 	c          *Conn
-	publicKey  tailcfg.NodeKey // peer public key (for WireGuard + DERP)
-	fakeWGAddr netaddr.IPPort  // the UDP address we tell wireguard-go we're using
-	wgEndpoint string          // string from ParseEndpoint, holds a JSON-serialized wgcfg.Endpoints
+	publicKey  key.NodePublic // peer public key (for WireGuard + DERP)
+	fakeWGAddr netaddr.IPPort // the UDP address we tell wireguard-go we're using
+	wgEndpoint string         // string from ParseEndpoint, holds a JSON-serialized wgcfg.Endpoints
 
 	// mu protects all following fields.
 	mu sync.Mutex // Lock ordering: Conn.mu, then endpoint.mu
@@ -3458,10 +3505,10 @@ func (de *endpoint) send(b []byte) error {
 	}
 	var err error
 	if !udpAddr.IsZero() {
-		_, err = de.c.sendAddr(udpAddr, key.NodePublicFromRaw32(mem.B(de.publicKey[:])), b)
+		_, err = de.c.sendAddr(udpAddr, de.publicKey, b)
 	}
 	if !derpAddr.IsZero() {
-		if ok, _ := de.c.sendAddr(derpAddr, key.NodePublicFromRaw32(mem.B(de.publicKey[:])), b); ok && err != nil {
+		if ok, _ := de.c.sendAddr(derpAddr, de.publicKey, b); ok && err != nil {
 			// UDP failed but DERP worked, so good enough:
 			return nil
 		}
@@ -3499,13 +3546,16 @@ func (de *endpoint) removeSentPingLocked(txid stun.TxID, sp sentPing) {
 	delete(de.sentPing, txid)
 }
 
-// sendDiscoPing sends a ping with the provided txid to ep.
+// sendDiscoPing sends a ping with the provided txid to ep using de's discoKey.
 //
-// The caller (startPingLocked) should've already been recorded the ping in
+// The caller (startPingLocked) should've already recorded the ping in
 // sentPing and set up the timer.
-func (de *endpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLevel discoLogLevel) {
-	selfPubKey, _ := de.c.publicKeyAtomic.Load().(tailcfg.NodeKey)
-	sent, _ := de.sendDiscoMessage(ep, &disco.Ping{
+//
+// The caller should use de.discoKey as the discoKey argument.
+// It is passed in so that sendDiscoPing doesn't need to lock de.mu.
+func (de *endpoint) sendDiscoPing(ep netaddr.IPPort, discoKey key.DiscoPublic, txid stun.TxID, logLevel discoLogLevel) {
+	selfPubKey, _ := de.c.publicKeyAtomic.Load().(key.NodePublic)
+	sent, _ := de.c.sendDiscoMessage(ep, de.publicKey, discoKey, &disco.Ping{
 		TxID:    [12]byte(txid),
 		NodeKey: selfPubKey,
 	}, logLevel)
@@ -3561,7 +3611,7 @@ func (de *endpoint) startPingLocked(ep netaddr.IPPort, now mono.Time, purpose di
 	if purpose == pingHeartbeat {
 		logLevel = discoVerboseLog
 	}
-	go de.sendDiscoPing(ep, txid, logLevel)
+	go de.sendDiscoPing(ep, de.discoKey, txid, logLevel)
 }
 
 func (de *endpoint) sendPingsLocked(now mono.Time, sendCallMeMaybe bool) {
@@ -3599,10 +3649,6 @@ func (de *endpoint) sendPingsLocked(now mono.Time, sendCallMeMaybe bool) {
 	}
 }
 
-func (de *endpoint) sendDiscoMessage(dst netaddr.IPPort, dm disco.Message, logLevel discoLogLevel) (sent bool, err error) {
-	return de.c.sendDiscoMessage(dst, de.publicKey, de.discoKey, dm, logLevel)
-}
-
 func (de *endpoint) updateFromNode(n *tailcfg.Node) {
 	if n == nil {
 		panic("nil node when updating disco ep")
@@ -3610,10 +3656,9 @@ func (de *endpoint) updateFromNode(n *tailcfg.Node) {
 	de.mu.Lock()
 	defer de.mu.Unlock()
 
-	tnk := key.DiscoPublicFromRaw32(mem.B(n.DiscoKey[:]))
-	if de.discoKey != tnk {
+	if de.discoKey != n.DiscoKey {
 		de.c.logf("[v1] magicsock: disco: node %s changed from discokey %s to %s", de.publicKey.ShortString(), de.discoKey, n.DiscoKey)
-		de.discoKey = tnk
+		de.discoKey = n.DiscoKey
 		de.discoShort = de.discoKey.ShortString()
 		de.resetLocked()
 	}
@@ -3986,7 +4031,7 @@ type discoInfo struct {
 
 	// lastNodeKey is the last NodeKey seen using discoKey.
 	// It's only updated if the NodeKey is unambiguous.
-	lastNodeKey tailcfg.NodeKey
+	lastNodeKey key.NodePublic
 
 	// lastNodeKeyTime is the time a NodeKey was last seen using
 	// this discoKey. It's only updated if the NodeKey is
@@ -3996,7 +4041,51 @@ type discoInfo struct {
 
 // setNodeKey sets the most recent mapping from di.discoKey to the
 // NodeKey nk.
-func (di *discoInfo) setNodeKey(nk tailcfg.NodeKey) {
+func (di *discoInfo) setNodeKey(nk key.NodePublic) {
 	di.lastNodeKey = nk
 	di.lastNodeKeyTime = time.Now()
 }
+
+var (
+	metricNumPeers     = clientmetric.NewGauge("magicsock_netmap_num_peers")
+	metricNumDERPConns = clientmetric.NewGauge("magicsock_num_derp_conns")
+
+	// Sends (data or disco)
+	metricSendDERPQueued      = clientmetric.NewCounter("magicsock_send_derp_queued")
+	metricSendDERPErrorChan   = clientmetric.NewCounter("magicsock_send_derp_error_chan")
+	metricSendDERPErrorClosed = clientmetric.NewCounter("magicsock_send_derp_error_closed")
+	metricSendDERPErrorQueue  = clientmetric.NewCounter("magicsock_send_derp_error_queue")
+	metricSendUDP             = clientmetric.NewCounter("magicsock_send_udp")
+	metricSendUDPError        = clientmetric.NewCounter("magicsock_send_udp_error")
+	metricSendDERP            = clientmetric.NewCounter("magicsock_send_derp")
+	metricSendDERPError       = clientmetric.NewCounter("magicsock_send_derp_error")
+
+	// Data packets (non-disco)
+	metricSendData            = clientmetric.NewCounter("magicsock_send_data")
+	metricSendDataNetworkDown = clientmetric.NewCounter("magicsock_send_data_network_down")
+	metricRecvData            = clientmetric.NewCounter("magicsock_recv_data")
+	metricRecvDataDERP        = clientmetric.NewCounter("magicsock_recv_data_derp")
+	metricRecvDataIPv4        = clientmetric.NewCounter("magicsock_recv_data_ipv4")
+	metricRecvDataIPv6        = clientmetric.NewCounter("magicsock_recv_data_ipv6")
+
+	// Disco packets
+	metricSendDiscoUDP      = clientmetric.NewCounter("magicsock_disco_send_udp")
+	metricSendDiscoDERP     = clientmetric.NewCounter("magicsock_disco_send_derp")
+	metricSentDiscoUDP      = clientmetric.NewCounter("magicsock_disco_sent_udp")
+	metricSentDiscoDERP     = clientmetric.NewCounter("magicsock_disco_sent_derp")
+	metricRecvDiscoBadPeer  = clientmetric.NewCounter("magicsock_disco_recv_bad_peer")
+	metricRecvDiscoBadKey   = clientmetric.NewCounter("magicsock_disco_recv_bad_key")
+	metricRecvDiscoBadParse = clientmetric.NewCounter("magicsock_disco_recv_bad_parse")
+
+	metricRecvDiscoUDP                 = clientmetric.NewCounter("magicsock_disco_recv_udp")
+	metricRecvDiscoDERP                = clientmetric.NewCounter("magicsock_disco_recv_derp")
+	metricRecvDiscoPing                = clientmetric.NewCounter("magicsock_disco_recv_ping")
+	metricRecvDiscoPong                = clientmetric.NewCounter("magicsock_disco_recv_pong")
+	metricRecvDiscoCallMeMaybe         = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe")
+	metricRecvDiscoCallMeMaybeBadNode  = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node")
+	metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco")
+
+	// metricDERPHomeChange is how many times our DERP home region DI has
+	// changed from non-zero to a different non-zero.
+	metricDERPHomeChange = clientmetric.NewCounter("derp_home_change")
+)

wgengine/magicsock/magicsock_test.go

@@ -9,9 +9,11 @@ import (
 	"context"
 	crand "crypto/rand"
 	"crypto/tls"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -166,7 +168,7 @@ func newMagicStackWithKey(t testing.TB, logf logger.Logf, l nettype.PacketListen
 	tsTun.SetFilter(filter.NewAllowAllForTest(logf))
 
 	wgLogger := wglog.NewLogger(logf)
-	dev := device.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger)
+	dev := wgcfg.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger)
 	dev.Up()
 
 	// Wait for magicsock to connect up to DERP.
@@ -247,7 +249,7 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM
 		me := ms[myIdx]
 		nm := &netmap.NetworkMap{
 			PrivateKey: me.privateKey,
-			NodeKey:    tailcfg.NodeKeyFromNodePublic(me.privateKey.Public()),
+			NodeKey:    me.privateKey.Public(),
 			Addresses:  []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(1, 0, 0, byte(myIdx+1)), 32)},
 		}
 		for i, peer := range ms {
@@ -258,8 +260,8 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM
 			peer := &tailcfg.Node{
 				ID:         tailcfg.NodeID(i + 1),
 				Name:       fmt.Sprintf("node%d", i+1),
-				Key:        tailcfg.NodeKeyFromNodePublic(peer.privateKey.Public()),
-				DiscoKey:   tailcfg.DiscoKeyFromDiscoPublic(peer.conn.DiscoPublicKey()),
+				Key:        peer.privateKey.Public(),
+				DiscoKey:   peer.conn.DiscoPublicKey(),
 				Addresses:  addrs,
 				AllowedIPs: addrs,
 				Endpoints:  epStrings(eps[i]),
@@ -285,7 +287,7 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM
 			m.conn.SetNetworkMap(nm)
 			peerSet := make(map[key.NodePublic]struct{}, len(nm.Peers))
 			for _, peer := range nm.Peers {
-				peerSet[key.NodePublicFromRaw32(mem.B(peer.Key[:]))] = struct{}{}
+				peerSet[peer.Key] = struct{}{}
 			}
 			m.conn.UpdatePeers(peerSet)
 			wg, err := nmcfg.WGCfg(nm, logf, netmap.AllowSingleHosts, "")
@@ -469,7 +471,7 @@ func TestDeviceStartStop(t *testing.T) {
 
 	tun := tuntest.NewChannelTUN()
 	wgLogger := wglog.NewLogger(t.Logf)
-	dev := device.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger)
+	dev := wgcfg.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger)
 	dev.Up()
 	dev.Close()
 }
@@ -618,7 +620,7 @@ func TestNoDiscoKey(t *testing.T) {
 
 	removeDisco := func(idx int, nm *netmap.NetworkMap) {
 		for _, p := range nm.Peers {
-			p.DiscoKey = tailcfg.DiscoKey{}
+			p.DiscoKey = key.DiscoPublic{}
 		}
 	}
 
@@ -680,7 +682,7 @@ func TestDiscokeyChange(t *testing.T) {
 		}
 		mu.Lock()
 		defer mu.Unlock()
-		nm.Peers[0].DiscoKey = tailcfg.DiscoKeyFromDiscoPublic(m1DiscoKey)
+		nm.Peers[0].DiscoKey = m1DiscoKey
 	}
 
 	cleanupMesh := meshStacks(t.Logf, setm1Key, m1, m2)
@@ -955,8 +957,9 @@ func testActiveDiscovery(t *testing.T, d *devices) {
 
 func mustDirect(t *testing.T, logf logger.Logf, m1, m2 *magicStack) {
 	lastLog := time.Now().Add(-time.Minute)
-	// See https://github.com/tailscale/tailscale/issues/654 for a discussion of this deadline.
-	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
+	// See https://github.com/tailscale/tailscale/issues/654
+	// and https://github.com/tailscale/tailscale/issues/3247 for discussions of this deadline.
+	for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
 		pst := m1.Status().Peer[m2.Public()]
 		if pst.CurAddr != "" {
 			logf("direct link %s->%s found with addr %s", m1, m2, pst.CurAddr)
@@ -1136,13 +1139,13 @@ func TestDiscoMessage(t *testing.T) {
 	peer1Pub := c.DiscoPublicKey()
 	peer1Priv := c.discoPrivate
 	n := &tailcfg.Node{
-		Key:      tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
-		DiscoKey: tailcfg.DiscoKeyFromDiscoPublic(peer1Pub),
+		Key:      key.NewNode().Public(),
+		DiscoKey: peer1Pub,
 	}
 	c.peerMap.upsertEndpoint(&endpoint{
 		publicKey: n.Key,
-		discoKey:  key.DiscoPublicFromRaw32(mem.B(n.DiscoKey[:])),
-	})
+		discoKey:  n.DiscoKey,
+	}, key.DiscoPublic{})
 
 	const payload = "why hello"
 
@@ -1153,7 +1156,7 @@ func TestDiscoMessage(t *testing.T) {
 
 	box := peer1Priv.Shared(c.discoPrivate.Public()).Seal([]byte(payload))
 	pkt = append(pkt, box...)
-	got := c.handleDiscoMessage(pkt, netaddr.IPPort{}, tailcfg.NodeKey{})
+	got := c.handleDiscoMessage(pkt, netaddr.IPPort{}, key.NodePublic{})
 	if !got {
 		t.Error("failed to open it")
 	}
@@ -1184,7 +1187,7 @@ func Test32bitAlignment(t *testing.T) {
 	called := 0
 	de := endpoint{
 		c: &Conn{
-			noteRecvActivity: func(tailcfg.NodeKey) { called++ },
+			noteRecvActivity: func(key.NodePublic) { called++ },
 		},
 	}
 
@@ -1223,18 +1226,17 @@ func newTestConn(t testing.TB) *Conn {
 // addTestEndpoint sets conn's network map to a single peer expected
 // to receive packets from sendConn (or DERP), and returns that peer's
 // nodekey and discokey.
-func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (tailcfg.NodeKey, key.DiscoPublic) {
+func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (key.NodePublic, key.DiscoPublic) {
 	// Give conn just enough state that it'll recognize sendConn as a
 	// valid peer and not fall through to the legacy magicsock
 	// codepath.
 	discoKey := key.DiscoPublicFromRaw32(mem.B([]byte{31: 1}))
 	nodeKey := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 31: 0}))
-	tnk := tailcfg.NodeKeyFromNodePublic(nodeKey)
 	conn.SetNetworkMap(&netmap.NetworkMap{
 		Peers: []*tailcfg.Node{
 			{
-				Key:       tnk,
-				DiscoKey:  tailcfg.DiscoKeyFromDiscoPublic(discoKey),
+				Key:       nodeKey,
+				DiscoKey:  discoKey,
 				Endpoints: []string{sendConn.LocalAddr().String()},
 			},
 		},
@@ -1244,8 +1246,8 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (tailcf
 	if err != nil {
 		tb.Fatal(err)
 	}
-	conn.addValidDiscoPathForTest(tnk, netaddr.MustParseIPPort(sendConn.LocalAddr().String()))
-	return tnk, discoKey
+	conn.addValidDiscoPathForTest(nodeKey, netaddr.MustParseIPPort(sendConn.LocalAddr().String()))
+	return nodeKey, discoKey
 }
 
 func setUpReceiveFrom(tb testing.TB) (roundTrip func()) {
@@ -1405,19 +1407,19 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 	conn.SetPrivateKey(key.NodePrivateFromRaw32(mem.B([]byte{0: 1, 31: 0})))
 
 	discoKey := key.DiscoPublicFromRaw32(mem.B([]byte{31: 1}))
-	nodeKey1 := tailcfg.NodeKey{0: 'N', 1: 'K', 2: '1'}
-	nodeKey2 := tailcfg.NodeKey{0: 'N', 1: 'K', 2: '2'}
+	nodeKey1 := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 2: '1', 31: 0}))
+	nodeKey2 := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 2: '2', 31: 0}))
 
 	conn.SetNetworkMap(&netmap.NetworkMap{
 		Peers: []*tailcfg.Node{
 			{
 				Key:       nodeKey1,
-				DiscoKey:  tailcfg.DiscoKeyFromDiscoPublic(discoKey),
+				DiscoKey:  discoKey,
 				Endpoints: []string{"192.168.1.2:345"},
 			},
 		},
 	})
-	_, err := conn.ParseEndpoint(key.NodePublicFromRaw32(mem.B(nodeKey1[:])).UntypedHexString())
+	_, err := conn.ParseEndpoint(nodeKey1.UntypedHexString())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1427,7 +1429,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 			Peers: []*tailcfg.Node{
 				{
 					Key:       nodeKey2,
-					DiscoKey:  tailcfg.DiscoKeyFromDiscoPublic(discoKey),
+					DiscoKey:  discoKey,
 					Endpoints: []string{"192.168.1.2:345"},
 				},
 			},
@@ -1436,7 +1438,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 
 	de, ok := conn.peerMap.endpointForNodeKey(nodeKey2)
 	if ok && de.publicKey != nodeKey2 {
-		t.Fatalf("discoEndpoint public key = %q; want %q", de.publicKey[:], nodeKey2[:])
+		t.Fatalf("discoEndpoint public key = %q; want %q", de.publicKey, nodeKey2)
 	}
 	if de.discoKey != discoKey {
 		t.Errorf("discoKey = %v; want %v", de.discoKey, discoKey)
@@ -1639,3 +1641,125 @@ func epStrings(eps []tailcfg.Endpoint) (ret []string) {
 	}
 	return
 }
+
+func TestStressSetNetworkMap(t *testing.T) {
+	t.Parallel()
+
+	conn := newTestConn(t)
+	t.Cleanup(func() { conn.Close() })
+	var buf tstest.MemLogger
+	conn.logf = buf.Logf
+
+	conn.SetPrivateKey(key.NewNode())
+
+	const npeers = 5
+	present := make([]bool, npeers)
+	allPeers := make([]*tailcfg.Node, npeers)
+	for i := range allPeers {
+		present[i] = true
+		allPeers[i] = &tailcfg.Node{
+			DiscoKey:  randDiscoKey(),
+			Key:       randNodeKey(),
+			Endpoints: []string{fmt.Sprintf("192.168.1.2:%d", i)},
+		}
+	}
+
+	// Get a PRNG seed. If not provided, generate a new one to get extra coverage.
+	seed, err := strconv.ParseUint(os.Getenv("TS_STRESS_SET_NETWORK_MAP_SEED"), 10, 64)
+	if err != nil {
+		var buf [8]byte
+		crand.Read(buf[:])
+		seed = binary.LittleEndian.Uint64(buf[:])
+	}
+	t.Logf("TS_STRESS_SET_NETWORK_MAP_SEED=%d", seed)
+	prng := rand.New(rand.NewSource(int64(seed)))
+
+	const iters = 1000 // approx 0.5s on an m1 mac
+	for i := 0; i < iters; i++ {
+		for j := 0; j < npeers; j++ {
+			// Randomize which peers are present.
+			if prng.Int()&1 == 0 {
+				present[j] = !present[j]
+			}
+			// Randomize some peer disco keys and node keys.
+			if prng.Int()&1 == 0 {
+				allPeers[j].DiscoKey = randDiscoKey()
+			}
+			if prng.Int()&1 == 0 {
+				allPeers[j].Key = randNodeKey()
+			}
+		}
+		// Clone existing peers into a new netmap.
+		peers := make([]*tailcfg.Node, 0, len(allPeers))
+		for peerIdx, p := range allPeers {
+			if present[peerIdx] {
+				peers = append(peers, p.Clone())
+			}
+		}
+		// Set the netmap.
+		conn.SetNetworkMap(&netmap.NetworkMap{
+			Peers: peers,
+		})
+		// Check invariants.
+		if err := conn.peerMap.validate(); err != nil {
+			t.Error(err)
+		}
+	}
+}
+
+func randDiscoKey() (k key.DiscoPublic) { return key.NewDisco().Public() }
+func randNodeKey() (k key.NodePublic)   { return key.NewNode().Public() }
+
+// validate checks m for internal consistency and reports the first error encountered.
+// It is used in tests only, so it doesn't need to be efficient.
+func (m *peerMap) validate() error {
+	seenEps := make(map[*endpoint]bool)
+	for pub, pi := range m.byNodeKey {
+		if got := pi.ep.publicKey; got != pub {
+			return fmt.Errorf("byNodeKey[%v].publicKey = %v", pub, got)
+		}
+		if got, want := pi.ep.wgEndpoint, pub.UntypedHexString(); got != want {
+			return fmt.Errorf("byNodeKey[%v].wgEndpoint = %q, want %q", pub, got, want)
+		}
+		if _, ok := seenEps[pi.ep]; ok {
+			return fmt.Errorf("duplicate endpoint present: %v", pi.ep.publicKey)
+		}
+		seenEps[pi.ep] = true
+		for ipp, v := range pi.ipPorts {
+			if !v {
+				return fmt.Errorf("m.byIPPort[%v] is false, expected map to be set-like", ipp)
+			}
+			if got := m.byIPPort[ipp]; got != pi {
+				return fmt.Errorf("m.byIPPort[%v] = %v, want %v", ipp, got, pi)
+			}
+		}
+	}
+
+	for ipp, pi := range m.byIPPort {
+		if !pi.ipPorts[ipp] {
+			return fmt.Errorf("ipPorts[%v] for %v is false", ipp, pi.ep.publicKey)
+		}
+		pi2 := m.byNodeKey[pi.ep.publicKey]
+		if pi != pi2 {
+			return fmt.Errorf("byNodeKey[%v]=%p doesn't match byIPPort[%v]=%p", pi, pi, pi.ep.publicKey, pi2)
+		}
+	}
+
+	publicToDisco := make(map[key.NodePublic]key.DiscoPublic)
+	for disco, nodes := range m.nodesOfDisco {
+		for pub, v := range nodes {
+			if !v {
+				return fmt.Errorf("m.nodeOfDisco[%v][%v] is false, expected map to be set-like", disco, pub)
+			}
+			if _, ok := m.byNodeKey[pub]; !ok {
+				return fmt.Errorf("nodesOfDisco refers to public key %v, which is not present in byNodeKey", pub)
+			}
+			if _, ok := publicToDisco[pub]; ok {
+				return fmt.Errorf("publicKey %v refers to multiple disco keys", pub)
+			}
+			publicToDisco[pub] = disco
+		}
+	}
+
+	return nil
+}

wgengine/monitor/monitor_windows.go

@@ -7,9 +7,13 @@ package monitor
 import (
 	"context"
 	"errors"
+	"strings"
+	"sync"
 	"time"
 
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
 )
 
@@ -31,6 +35,9 @@ type winMon struct {
 	addressChangeCallback *winipcfg.UnicastAddressChangeCallback
 	routeChangeCallback   *winipcfg.RouteChangeCallback
 
+	mu      sync.Mutex
+	lastLog time.Time // time we last logged about any windows change event
+
 	// noDeadlockTicker exists just to have something scheduled as
 	// far as the Go runtime is concerned. Otherwise "tailscaled
 	// debug --monitor" thinks it's deadlocked with nothing to do,
@@ -100,7 +107,20 @@ func (m *winMon) Receive() (message, error) {
 
 	select {
 	case msg := <-m.messagec:
-		m.logf("got windows change event after %v: evt=%s", time.Since(t0).Round(time.Millisecond), msg.eventType)
+		now := time.Now()
+		m.mu.Lock()
+		sinceLast := now.Sub(m.lastLog)
+		m.lastLog = now
+		m.mu.Unlock()
+		// If it's either been awhile since we last logged
+		// anything, or if this some route/addr that's not
+		// about a Tailscale IP ("ts" prefix), then log. This
+		// is mainly limited to suppress the flood about our own
+		// route updates after connecting to a large tailnet
+		// and all the IPv4 /32 routes.
+		if sinceLast > 5*time.Second || !strings.HasPrefix(msg.eventType, "ts") {
+			m.logf("got windows change event after %v: evt=%s", time.Since(t0).Round(time.Millisecond), msg.eventType)
+		}
 		return msg, nil
 	case <-m.ctx.Done():
 		return nil, errClosed
@@ -108,15 +128,25 @@ func (m *winMon) Receive() (message, error) {
 }
 
 // unicastAddressChanged is the callback we register with Windows to call when unicast address changes.
-func (m *winMon) unicastAddressChanged(_ winipcfg.MibNotificationType, _ *winipcfg.MibUnicastIPAddressRow) {
+func (m *winMon) unicastAddressChanged(_ winipcfg.MibNotificationType, row *winipcfg.MibUnicastIPAddressRow) {
+	what := "addr"
+	if ip, ok := netaddr.FromStdIP(row.Address.IP()); ok && tsaddr.IsTailscaleIP(ip) {
+		what = "tsaddr"
+	}
+
 	// start a goroutine to finish our work, to return to Windows out of this callback
-	go m.somethingChanged("addr")
+	go m.somethingChanged(what)
 }
 
 // routeChanged is the callback we register with Windows to call when route changes.
-func (m *winMon) routeChanged(_ winipcfg.MibNotificationType, _ *winipcfg.MibIPforwardRow2) {
+func (m *winMon) routeChanged(_ winipcfg.MibNotificationType, row *winipcfg.MibIPforwardRow2) {
+	what := "route"
+	ipn := row.DestinationPrefix.IPNet()
+	if cidr, ok := netaddr.FromStdIPNet(&ipn); ok && tsaddr.IsTailscaleIP(cidr.IP()) {
+		what = "tsroute"
+	}
 	// start a goroutine to finish our work, to return to Windows out of this callback
-	go m.somethingChanged("route")
+	go m.somethingChanged(what)
 }
 
 // somethingChanged gets called from OS callbacks whenever address or route changes.

wgengine/netstack/netstack.go

@@ -54,13 +54,23 @@ type Impl struct {
 	// port other than accepting it and closing it.
 	ForwardTCPIn func(c net.Conn, port uint16)
 
-	ipstack     *stack.Stack
-	linkEP      *channel.Endpoint
-	tundev      *tstun.Wrapper
-	e           wgengine.Engine
-	mc          *magicsock.Conn
-	logf        logger.Logf
-	onlySubnets bool // whether we only want to handle subnet relaying
+	// ProcessLocalIPs is whether netstack should handle incoming
+	// traffic directed at the Node.Addresses (local IPs).
+	// It can only be set before calling Start.
+	ProcessLocalIPs bool
+
+	// ProcessSubnets is whether netstack should handle incoming
+	// traffic destined to non-local IPs (i.e. whether it should
+	// be a subnet router).
+	// It can only be set before calling Start.
+	ProcessSubnets bool
+
+	ipstack *stack.Stack
+	linkEP  *channel.Endpoint
+	tundev  *tstun.Wrapper
+	e       wgengine.Engine
+	mc      *magicsock.Conn
+	logf    logger.Logf
 
 	// atomicIsLocalIPFunc holds a func that reports whether an IP
 	// is a local (non-subnet) Tailscale IP address of this
@@ -81,7 +91,7 @@ const nicID = 1
 const mtu = 1500
 
 // Create creates and populates a new Impl.
-func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magicsock.Conn, onlySubnets bool) (*Impl, error) {
+func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magicsock.Conn) (*Impl, error) {
 	if mc == nil {
 		return nil, errors.New("nil magicsock.Conn")
 	}
@@ -130,7 +140,6 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 		e:                   e,
 		mc:                  mc,
 		connsOpenBySubnetIP: make(map[netaddr.IP]int),
-		onlySubnets:         onlySubnets,
 	}
 	ns.atomicIsLocalIPFunc.Store(tsaddr.NewContainsIPFunc(nil))
 	return ns, nil
@@ -275,10 +284,10 @@ func (ns *Impl) updateIPs(nm *netmap.NetworkMap) {
 		isAddr[ipp] = true
 	}
 	for _, ipp := range nm.SelfNode.AllowedIPs {
-		if ns.onlySubnets && isAddr[ipp] {
-			continue
+		local := isAddr[ipp]
+		if local && ns.ProcessLocalIPs || !local && ns.ProcessSubnets {
+			newIPs[ipPrefixToAddressWithPrefix(ipp)] = true
 		}
-		newIPs[ipPrefixToAddressWithPrefix(ipp)] = true
 	}
 
 	ipsToBeAdded := make(map[tcpip.AddressWithPrefix]bool)
@@ -446,11 +455,27 @@ func (ns *Impl) isLocalIP(ip netaddr.IP) bool {
 	return ns.atomicIsLocalIPFunc.Load().(func(netaddr.IP) bool)(ip)
 }
 
+// shouldProcessInbound reports whether an inbound packet should be
+// handled by netstack.
+func (ns *Impl) shouldProcessInbound(p *packet.Parsed, t *tstun.Wrapper) bool {
+	if !ns.ProcessLocalIPs && !ns.ProcessSubnets {
+		// Fast path for common case (e.g. Linux server in TUN mode) where
+		// netstack isn't used at all; don't even do an isLocalIP lookup.
+		return false
+	}
+	isLocal := ns.isLocalIP(p.Dst.IP())
+	if ns.ProcessLocalIPs && isLocal {
+		return true
+	}
+	if ns.ProcessSubnets && !isLocal {
+		return true
+	}
+	return false
+}
+
 func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
-	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP()) {
-		// In hybrid ("only subnets") mode, bail out early if
-		// the traffic is destined for an actual Tailscale
-		// address. The real host OS interface will handle it.
+	if !ns.shouldProcessInbound(p, t) {
+		// Let the host network stack (if any) deal with it.
 		return filter.Accept
 	}
 	var pn tcpip.NetworkProtocolNumber

wgengine/router/ifconfig_windows.go

@@ -15,7 +15,6 @@ import (
 	"sort"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	ole "github.com/go-ole/go-ole"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/tun"
@@ -24,6 +23,7 @@ import (
 	"tailscale.com/health"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/util/multierr"
 	"tailscale.com/wgengine/winnet"
 )
 
@@ -809,5 +809,5 @@ func syncRoutes(ifc *winipcfg.IPAdapterAddresses, want []*winipcfg.RouteData, do
 		}
 	}
 
-	return multierror.New(errs)
+	return multierr.New(errs...)
 }

wgengine/router/router_linux.go

@@ -17,8 +17,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
-	"github.com/go-multierror/multierror"
-	"github.com/vishvananda/netlink"
+	"github.com/tailscale/netlink"
 	"golang.org/x/sys/unix"
 	"golang.org/x/time/rate"
 	"golang.zx2c4.com/wireguard/tun"
@@ -27,6 +26,7 @@ import (
 	"tailscale.com/syncs"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
+	"tailscale.com/util/multierr"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine/monitor"
 )
@@ -147,17 +147,14 @@ func newUserspaceRouter(logf logger.Logf, tunDev tun.Device, linkMon *monitor.Mo
 }
 
 func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monitor.Mon, netfilter4, netfilter6 netfilterRunner, cmd commandRunner, supportsV6, supportsV6NAT bool) (Router, error) {
-	ipRuleAvailable := (cmd.run("ip", "rule") == nil)
-
 	r := &linuxRouter{
 		logf:          logf,
 		tunname:       tunname,
 		netfilterMode: netfilterOff,
 		linkMon:       linkMon,
 
-		ipRuleAvailable: ipRuleAvailable,
-		v6Available:     supportsV6,
-		v6NATAvailable:  supportsV6NAT,
+		v6Available:    supportsV6,
+		v6NATAvailable: supportsV6NAT,
 
 		ipt4: netfilter4,
 		ipt6: netfilter6,
@@ -165,6 +162,12 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monit
 
 		ipRuleFixLimiter: rate.NewLimiter(rate.Every(5*time.Second), 10),
 	}
+	if r.useIPCommand() {
+		r.ipRuleAvailable = (cmd.run("ip", "rule") == nil)
+	} else {
+		// Pretend it is.
+		r.ipRuleAvailable = true
+	}
 
 	return r, nil
 }
@@ -180,9 +183,17 @@ func useAmbientCaps() bool {
 	return v >= 7
 }
 
+var forceIPCommand, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_USE_IP_COMMAND"))
+
 // useIPCommand reports whether r should use the "ip" command (or its
 // fake commandRunner for tests) instead of netlink.
 func (r *linuxRouter) useIPCommand() bool {
+	if r.cmd == nil {
+		panic("invalid init")
+	}
+	if forceIPCommand {
+		return true
+	}
 	// In the future we might need to fall back to using the "ip"
 	// command if, say, netlink is blocked somewhere but the ip
 	// command is allowed to use netlink. For now we only use the ip
@@ -314,7 +325,7 @@ func (r *linuxRouter) Set(cfg *Config) error {
 	}
 	r.snatSubnetRoutes = cfg.SNATSubnetRoutes
 
-	return multierror.New(errs)
+	return multierr.New(errs...)
 }
 
 // setNetfilterMode switches the router to the given netfilter
@@ -604,7 +615,11 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro
 	return err
 }
 
-var errESRCH error = syscall.ESRCH
+var (
+	errESRCH  error = syscall.ESRCH
+	errENOENT error = syscall.ENOENT
+	errEEXIST error = syscall.EEXIST
+)
 
 // delRoute removes the route for cidr pointing to the tunnel
 // interface. Fails if the route doesn't exist, or if removing the
@@ -766,6 +781,16 @@ func (f addrFamily) dashArg() string {
 	panic("illegal")
 }
 
+func (f addrFamily) netlinkInt() int {
+	switch f {
+	case 4:
+		return netlink.FAMILY_V4
+	case 6:
+		return netlink.FAMILY_V6
+	}
+	panic("illegal")
+}
+
 func (r *linuxRouter) addrFamilies() []addrFamily {
 	if r.v6Available {
 		return []addrFamily{v4, v6}
@@ -878,7 +903,7 @@ var ipRules = []netlink.Rule{
 	{
 		Priority: 5250,
 		Mark:     tailscaleBypassMarkNum,
-		Table:    0, // unreachable
+		Type:     unix.RTN_UNREACHABLE,
 	},
 	// If we get to this point, capture all packets and send them
 	// through to the tailscale route table. For apps other than us
@@ -898,7 +923,34 @@ func (r *linuxRouter) justAddIPRules() error {
 	if !r.ipRuleAvailable {
 		return nil
 	}
+	if r.useIPCommand() {
+		return r.addIPRulesWithIPCommand()
+	}
+	var errAcc error
+	for _, family := range r.addrFamilies() {
+		for _, ru := range ipRules {
+			// Note: r is a value type here; safe to mutate it.
+			ru.Family = family.netlinkInt()
+			ru.Mask = -1
+			ru.Goto = -1
+			ru.SuppressIfgroup = -1
+			ru.SuppressPrefixlen = -1
+			ru.Flow = -1
 
+			err := netlink.RuleAdd(&ru)
+			if errors.Is(err, errEEXIST) {
+				// Ignore dups.
+				continue
+			}
+			if err != nil && errAcc == nil {
+				errAcc = err
+			}
+		}
+	}
+	return errAcc
+}
+
+func (r *linuxRouter) addIPRulesWithIPCommand() error {
 	rg := newRunGroup(nil, r.cmd)
 
 	for _, family := range r.addrFamilies() {
@@ -913,7 +965,8 @@ func (r *linuxRouter) justAddIPRules() error {
 			}
 			if r.Table != 0 {
 				args = append(args, "table", mustRouteTable(r.Table).ipCmdArg())
-			} else {
+			}
+			if r.Type == unix.RTN_UNREACHABLE {
 				args = append(args, "type", "unreachable")
 			}
 			rg.Run(args...)
@@ -940,7 +993,39 @@ func (r *linuxRouter) delIPRules() error {
 	if !r.ipRuleAvailable {
 		return nil
 	}
+	if r.useIPCommand() {
+		return r.delIPRulesWithIPCommand()
+	}
+	var errAcc error
+	for _, family := range r.addrFamilies() {
+		for _, ru := range ipRules {
+			// Note: r is a value type here; safe to mutate it.
+			// When deleting rules, we want to be a bit specific (mention which
+			// table we were routing to) but not *too* specific (fwmarks, etc).
+			// That leaves us some flexibility to change these values in later
+			// versions without having ongoing hacks for every possible
+			// combination.
+			ru.Family = family.netlinkInt()
+			ru.Mark = -1
+			ru.Mask = -1
+			ru.Goto = -1
+			ru.SuppressIfgroup = -1
+			ru.SuppressPrefixlen = -1
 
+			err := netlink.RuleDel(&ru)
+			if errors.Is(err, errENOENT) {
+				// Didn't exist to begin with.
+				continue
+			}
+			if err != nil && errAcc == nil {
+				errAcc = err
+			}
+		}
+	}
+	return errAcc
+}
+
+func (r *linuxRouter) delIPRulesWithIPCommand() error {
 	// Error codes: 'ip rule' returns error code 2 if the rule is a
 	// duplicate (add) or not found (del). It returns a different code
 	// for syntax errors. This is also true of busybox.
@@ -1463,29 +1548,17 @@ func supportsV6NAT() bool {
 }
 
 func checkIPRuleSupportsV6() error {
-	add := []string{"-6", "rule", "add", "pref", "1234", "fwmark", tailscaleBypassMark, "table", tailscaleRouteTable.ipCmdArg()}
-	del := []string{"-6", "rule", "del", "pref", "1234", "fwmark", tailscaleBypassMark, "table", tailscaleRouteTable.ipCmdArg()}
-
+	rule := netlink.NewRule()
+	rule.Priority = 1234
+	rule.Mark = tailscaleBypassMarkNum
+	rule.Table = tailscaleRouteTable.num
 	// First delete the rule unconditionally, and don't check for
 	// errors. This is just cleaning up anything that might be already
 	// there.
-	exec.Command("ip", del...).Run()
-
-	// Try adding the rule. This will fail on systems that support
-	// IPv6, but not IPv6 policy routing.
-	out, err := exec.Command("ip", add...).CombinedOutput()
-	if err != nil {
-		out = bytes.TrimSpace(out)
-		var detail interface{} = out
-		if len(out) == 0 {
-			detail = err.Error()
-		}
-		return fmt.Errorf("ip -6 rule failed: %s", detail)
-	}
-
-	// Delete again.
-	exec.Command("ip", del...).Run()
-	return nil
+	netlink.RuleDel(rule)
+	// And clean up on exit.
+	defer netlink.RuleDel(rule)
+	return netlink.RuleAdd(rule)
 }
 
 func nlAddrOfPrefix(p netaddr.IPPrefix) *netlink.Addr {

wgengine/router/router_linux_test.go

@@ -654,62 +654,110 @@ func createTestTUN(t *testing.T) tun.Device {
 	return tun
 }
 
-func TestDelRouteIdempotent(t *testing.T) {
+type linuxTest struct {
+	tun       tun.Device
+	mon       *monitor.Mon
+	r         *linuxRouter
+	logOutput tstest.MemLogger
+}
+
+func (lt *linuxTest) Close() error {
+	if lt.tun != nil {
+		lt.tun.Close()
+	}
+	if lt.mon != nil {
+		lt.mon.Close()
+	}
+	return nil
+}
+
+func newLinuxRootTest(t *testing.T) *linuxTest {
 	if os.Getuid() != 0 {
 		t.Skip("test requires root")
 	}
-	tun := createTestTUN(t)
-	defer tun.Close()
 
-	var logOutput tstest.MemLogger
-	logf := logOutput.Logf
+	lt := new(linuxTest)
+	lt.tun = createTestTUN(t)
+
+	logf := lt.logOutput.Logf
 
 	mon, err := monitor.New(logger.Discard)
 	if err != nil {
+		lt.Close()
 		t.Fatal(err)
 	}
 	mon.Start()
-	defer mon.Close()
+	lt.mon = mon
 
-	r, err := newUserspaceRouter(logf, tun, mon)
+	r, err := newUserspaceRouter(logf, lt.tun, mon)
 	if err != nil {
+		lt.Close()
 		t.Fatal(err)
 	}
 	lr := r.(*linuxRouter)
 	if err := lr.upInterface(); err != nil {
+		lt.Close()
 		t.Fatal(err)
 	}
+	lt.r = lr
+	return lt
+}
+
+func TestDelRouteIdempotent(t *testing.T) {
+	lt := newLinuxRootTest(t)
+	defer lt.Close()
+
 	for _, s := range []string{
 		"192.0.2.0/24",  // RFC 5737
 		"2001:DB8::/32", // RFC 3849
 	} {
 		cidr := netaddr.MustParseIPPrefix(s)
-		if err := lr.addRoute(cidr); err != nil {
-			t.Fatal(err)
+		if err := lt.r.addRoute(cidr); err != nil {
+			t.Error(err)
+			continue
 		}
 		for i := 0; i < 2; i++ {
-			if err := lr.delRoute(cidr); err != nil {
-				t.Fatalf("delRoute(i=%d): %v", i, err)
+			if err := lt.r.delRoute(cidr); err != nil {
+				t.Errorf("delRoute(i=%d): %v", i, err)
 			}
 		}
 	}
 
-	wantSubs := map[string]int{
-		"warning: tried to delete route 192.0.2.0/24 but it was already gone; ignoring error":  1,
-		"warning: tried to delete route 2001:db8::/32 but it was already gone; ignoring error": 1,
-	}
-	out := logOutput.String()
-	for sub, want := range wantSubs {
-		got := strings.Count(out, sub)
-		if got != want {
-			t.Errorf("log output substring %q occurred %d time; want %d", sub, got, want)
-		}
-	}
 	if t.Failed() {
+		out := lt.logOutput.String()
 		t.Logf("Log output:\n%s", out)
 	}
 }
 
+func TestAddRemoveRules(t *testing.T) {
+	lt := newLinuxRootTest(t)
+	defer lt.Close()
+	r := lt.r
+
+	step := func(name string, f func() error) {
+		t.Logf("Doing %v ...", name)
+		if err := f(); err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		rules, err := netlink.RuleList(netlink.FAMILY_ALL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, r := range rules {
+			if r.Priority >= 5000 && r.Priority <= 5999 {
+				t.Logf("Rule: %+v", r)
+			}
+		}
+
+	}
+
+	step("init_del_and_add", r.addIPRules)
+	step("dup_add", r.justAddIPRules)
+	step("del", r.delIPRules)
+	step("dup_del", r.delIPRules)
+
+}
+
 func TestDebugListLinks(t *testing.T) {
 	links, err := netlink.LinkList()
 	if err != nil {
@@ -753,3 +801,13 @@ func TestDebugListRules(t *testing.T) {
 		})
 	}
 }
+
+func TestCheckIPRuleSupportsV6(t *testing.T) {
+	err := checkIPRuleSupportsV6()
+	if err != nil && os.Getuid() != 0 {
+		t.Skipf("skipping, error when not root: %v", err)
+	}
+	// Just log it. For interactive testing only.
+	// Some machines running our tests might not have IPv6.
+	t.Logf("Got: %v", err)
+}

wgengine/userspace.go

@@ -42,6 +42,7 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/deephash"
 	"tailscale.com/version"
 	"tailscale.com/wgengine/filter"
@@ -114,8 +115,8 @@ type userspaceEngine struct {
 	lastEngineSigFull   deephash.Sum // of full wireguard config
 	lastEngineSigTrim   deephash.Sum // of trimmed wireguard config
 	lastDNSConfig       *dns.Config
-	recvActivityAt      map[tailcfg.NodeKey]mono.Time
-	trimmedNodes        map[tailcfg.NodeKey]bool  // set of node keys of peers currently excluded from wireguard config
+	recvActivityAt      map[key.NodePublic]mono.Time
+	trimmedNodes        map[key.NodePublic]bool   // set of node keys of peers currently excluded from wireguard config
 	sentActivityAt      map[netaddr.IP]*mono.Time // value is accessed atomically
 	destIPActivityFuncs map[netaddr.IP]func()
 	statusBufioReader   *bufio.Reader // reusable for UAPI
@@ -127,7 +128,7 @@ type userspaceEngine struct {
 	netMap              *netmap.NetworkMap // or nil
 	closing             bool               // Close was called (even if we're still closing)
 	statusCallback      StatusCallback
-	peerSequence        []tailcfg.NodeKey
+	peerSequence        []key.NodePublic
 	endpoints           []tailcfg.Endpoint
 	pendOpen            map[flowtrack.Tuple]*pendingOpenFlow // see pendopen.go
 	networkMapCallbacks map[*someHandle]NetworkMapCallback
@@ -331,7 +332,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	closePool.add(e.magicConn)
 	e.magicConn.SetNetworkUp(e.linkMon.InterfaceState().AnyInterfaceUp())
 
-	tsTUNDev.SetDiscoKey(tailcfg.DiscoKeyFromDiscoPublic(e.magicConn.DiscoPublicKey()))
+	tsTUNDev.SetDiscoKey(e.magicConn.DiscoPublicKey())
 
 	if conf.RespondToPing {
 		e.tundev.PostFilterIn = echoRespondToAll
@@ -362,7 +363,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 
 	// wgdev takes ownership of tundev, will close it when closed.
 	e.logf("Creating wireguard device...")
-	e.wgdev = device.NewDevice(e.tundev, e.magicConn.Bind(), e.wgLogger.DeviceLogger)
+	e.wgdev = wgcfg.NewDevice(e.tundev, e.magicConn.Bind(), e.wgLogger.DeviceLogger)
 	closePool.addFunc(e.wgdev.Close)
 	closePool.addFunc(func() {
 		if err := e.magicConn.Close(); err != nil {
@@ -435,6 +436,7 @@ func echoRespondToAll(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
 // main ACL filter.
 func (e *userspaceEngine) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
 	if verdict := e.handleDNS(p, t); verdict == filter.Drop {
+		metricMagicDNSPacketIn.Add(1)
 		// local DNS handled the packet.
 		return filter.Drop
 	}
@@ -450,6 +452,7 @@ func (e *userspaceEngine) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper)
 			// notice that an outbound packet is actually destined for
 			// ourselves, and loop it back into macOS.
 			t.InjectInboundCopy(p.Buffer())
+			metricReflectToOS.Add(1)
 			return filter.Drop
 		}
 	}
@@ -554,7 +557,7 @@ func isTrimmablePeer(p *wgcfg.Peer, numPeers int) bool {
 // noteRecvActivity is called by magicsock when a packet has been
 // received for the peer with node key nk. Magicsock calls this no
 // more than every 10 seconds for a given peer.
-func (e *userspaceEngine) noteRecvActivity(nk tailcfg.NodeKey) {
+func (e *userspaceEngine) noteRecvActivity(nk key.NodePublic) {
 	e.wgLock.Lock()
 	defer e.wgLock.Unlock()
 
@@ -596,7 +599,7 @@ func (e *userspaceEngine) noteRecvActivity(nk tailcfg.NodeKey) {
 // has had a packet sent to or received from it since t.
 //
 // e.wgLock must be held.
-func (e *userspaceEngine) isActiveSinceLocked(nk tailcfg.NodeKey, ip netaddr.IP, t mono.Time) bool {
+func (e *userspaceEngine) isActiveSinceLocked(nk key.NodePublic, ip netaddr.IP, t mono.Time) bool {
 	if e.recvActivityAt[nk].After(t) {
 		return true
 	}
@@ -613,7 +616,7 @@ func (e *userspaceEngine) isActiveSinceLocked(nk tailcfg.NodeKey, ip netaddr.IP,
 // If discoChanged is nil or empty, this extra removal step isn't done.
 //
 // e.wgLock must be held.
-func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[tailcfg.NodeKey]bool) error {
+func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.NodePublic]bool) error {
 	if hook := e.testMaybeReconfigHook; hook != nil {
 		hook()
 		return nil
@@ -639,36 +642,35 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[tailcfg.
 	// their NodeKey and Tailscale IPs.  These are the ones we'll need
 	// to install tracking hooks for to watch their send/receive
 	// activity.
-	trackNodes := make([]tailcfg.NodeKey, 0, len(full.Peers))
+	trackNodes := make([]key.NodePublic, 0, len(full.Peers))
 	trackIPs := make([]netaddr.IP, 0, len(full.Peers))
 
-	trimmedNodes := map[tailcfg.NodeKey]bool{} // TODO: don't re-alloc this map each time
+	trimmedNodes := map[key.NodePublic]bool{} // TODO: don't re-alloc this map each time
 
 	needRemoveStep := false
 	for i := range full.Peers {
 		p := &full.Peers[i]
 		nk := p.PublicKey
-		tnk := tailcfg.NodeKeyFromNodePublic(nk)
 		if !isTrimmablePeer(p, len(full.Peers)) {
 			min.Peers = append(min.Peers, *p)
-			if discoChanged[tnk] {
+			if discoChanged[nk] {
 				needRemoveStep = true
 			}
 			continue
 		}
-		trackNodes = append(trackNodes, tnk)
+		trackNodes = append(trackNodes, nk)
 		recentlyActive := false
 		for _, cidr := range p.AllowedIPs {
 			trackIPs = append(trackIPs, cidr.IP())
-			recentlyActive = recentlyActive || e.isActiveSinceLocked(tnk, cidr.IP(), activeCutoff)
+			recentlyActive = recentlyActive || e.isActiveSinceLocked(nk, cidr.IP(), activeCutoff)
 		}
 		if recentlyActive {
 			min.Peers = append(min.Peers, *p)
-			if discoChanged[tnk] {
+			if discoChanged[nk] {
 				needRemoveStep = true
 			}
 		} else {
-			trimmedNodes[tnk] = true
+			trimmedNodes[nk] = true
 		}
 	}
 	e.lastNMinPeers = len(min.Peers)
@@ -687,7 +689,7 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[tailcfg.
 		minner.Peers = nil
 		numRemove := 0
 		for _, p := range min.Peers {
-			if discoChanged[tailcfg.NodeKeyFromNodePublic(p.PublicKey)] {
+			if discoChanged[p.PublicKey] {
 				numRemove++
 				continue
 			}
@@ -715,10 +717,10 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[tailcfg.
 // as given to wireguard-go.
 //
 // e.wgLock must be held.
-func (e *userspaceEngine) updateActivityMapsLocked(trackNodes []tailcfg.NodeKey, trackIPs []netaddr.IP) {
+func (e *userspaceEngine) updateActivityMapsLocked(trackNodes []key.NodePublic, trackIPs []netaddr.IP) {
 	// Generate the new map of which nodekeys we want to track
 	// receive times for.
-	mr := map[tailcfg.NodeKey]mono.Time{} // TODO: only recreate this if set of keys changed
+	mr := map[key.NodePublic]mono.Time{} // TODO: only recreate this if set of keys changed
 	for _, nk := range trackNodes {
 		// Preserve old times in the new map, but also
 		// populate map entries for new trackNodes values with
@@ -807,7 +809,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	e.mu.Lock()
 	e.peerSequence = e.peerSequence[:0]
 	for _, p := range cfg.Peers {
-		e.peerSequence = append(e.peerSequence, tailcfg.NodeKeyFromNodePublic(p.PublicKey))
+		e.peerSequence = append(e.peerSequence, p.PublicKey)
 		peerSet[p.PublicKey] = struct{}{}
 	}
 	e.mu.Unlock()
@@ -840,12 +842,12 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	// If so, we need to update the wireguard-go/device.Device in two phases:
 	// once without the node which has restarted, to clear its wireguard session key,
 	// and a second time with it.
-	discoChanged := make(map[tailcfg.NodeKey]bool)
+	discoChanged := make(map[key.NodePublic]bool)
 	{
-		prevEP := make(map[tailcfg.NodeKey]key.DiscoPublic)
+		prevEP := make(map[key.NodePublic]key.DiscoPublic)
 		for i := range e.lastCfgFull.Peers {
 			if p := &e.lastCfgFull.Peers[i]; !p.DiscoKey.IsZero() {
-				prevEP[tailcfg.NodeKeyFromNodePublic(p.PublicKey)] = p.DiscoKey
+				prevEP[p.PublicKey] = p.DiscoKey
 			}
 		}
 		for i := range cfg.Peers {
@@ -853,7 +855,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 			if p.DiscoKey.IsZero() {
 				continue
 			}
-			pub := tailcfg.NodeKeyFromNodePublic(p.PublicKey)
+			pub := p.PublicKey
 			if old, ok := prevEP[pub]; ok && old != p.DiscoKey {
 				discoChanged[pub] = true
 				e.logf("wgengine: Reconfig: %s changed from %q to %q", pub.ShortString(), old, p.DiscoKey)
@@ -978,7 +980,7 @@ func (e *userspaceEngine) getStatus() (*Status, error) {
 		errc <- err
 	}()
 
-	pp := make(map[tailcfg.NodeKey]ipnstate.PeerStatusLite)
+	pp := make(map[key.NodePublic]ipnstate.PeerStatusLite)
 	var p ipnstate.PeerStatusLite
 
 	var hst1, hst2, n int64
@@ -1014,7 +1016,7 @@ func (e *userspaceEngine) getStatus() (*Status, error) {
 			if !p.NodeKey.IsZero() {
 				pp[p.NodeKey] = p
 			}
-			p = ipnstate.PeerStatusLite{NodeKey: tailcfg.NodeKeyFromNodePublic(pk)}
+			p = ipnstate.PeerStatusLite{NodeKey: pk}
 		case "rx_bytes":
 			n, err = mem.ParseInt(v, 10, 64)
 			p.RxBytes = n
@@ -1231,8 +1233,8 @@ func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) {
 	}
 }
 
-func (e *userspaceEngine) DiscoPublicKey() tailcfg.DiscoKey {
-	return tailcfg.DiscoKeyFromDiscoPublic(e.magicConn.DiscoPublicKey())
+func (e *userspaceEngine) DiscoPublicKey() key.DiscoPublic {
+	return e.magicConn.DiscoPublicKey()
 }
 
 func (e *userspaceEngine) UpdateStatus(sb *ipnstate.StatusBuilder) {
@@ -1242,7 +1244,7 @@ func (e *userspaceEngine) UpdateStatus(sb *ipnstate.StatusBuilder) {
 		return
 	}
 	for _, ps := range st.Peers {
-		sb.AddPeer(key.NodePublicFromRaw32(mem.B(ps.NodeKey[:])), &ipnstate.PeerStatus{
+		sb.AddPeer(ps.NodeKey, &ipnstate.PeerStatus{
 			RxBytes:       int64(ps.RxBytes),
 			TxBytes:       int64(ps.TxBytes),
 			LastHandshake: ps.LastHandshake,
@@ -1456,7 +1458,7 @@ func (e *userspaceEngine) peerForIP(ip netaddr.IP) (n *tailcfg.Node, isSelf bool
 
 	// TODO(bradfitz): this is O(n peers). Add ART to netaddr?
 	var best netaddr.IPPrefix
-	var bestKey tailcfg.NodeKey
+	var bestKey key.NodePublic
 	for _, p := range e.lastCfgFull.Peers {
 		for _, cidr := range p.AllowedIPs {
 			if !cidr.Contains(ip) {
@@ -1464,7 +1466,7 @@ func (e *userspaceEngine) peerForIP(ip netaddr.IP) (n *tailcfg.Node, isSelf bool
 			}
 			if best.IsZero() || cidr.Bits() > best.Bits() {
 				best = cidr
-				bestKey = tailcfg.NodeKeyFromNodePublic(p.PublicKey)
+				bestKey = p.PublicKey
 			}
 		}
 	}
@@ -1555,3 +1557,8 @@ func (ls fwdDNSLinkSelector) PickLink(ip netaddr.IP) (linkName string) {
 	}
 	return ""
 }
+
+var (
+	metricMagicDNSPacketIn = clientmetric.NewGauge("magicdns_packet_in") // for 100.100.100.100
+	metricReflectToOS      = clientmetric.NewGauge("packet_reflect_to_os")
+)

wgengine/userspace_test.go

@@ -37,15 +37,15 @@ func TestNoteReceiveActivity(t *testing.T) {
 	}
 	e := &userspaceEngine{
 		timeNow:               func() mono.Time { return now },
-		recvActivityAt:        map[tailcfg.NodeKey]mono.Time{},
+		recvActivityAt:        map[key.NodePublic]mono.Time{},
 		logf:                  logBuf.Logf,
 		tundev:                new(tstun.Wrapper),
 		testMaybeReconfigHook: func() { confc <- true },
-		trimmedNodes:          map[tailcfg.NodeKey]bool{},
+		trimmedNodes:          map[key.NodePublic]bool{},
 	}
 	ra := e.recvActivityAt
 
-	nk := tailcfg.NodeKeyFromNodePublic(key.NewNode().Public())
+	nk := key.NewNode().Public()
 
 	// Activity on an untracked key should do nothing.
 	e.noteRecvActivity(nk)
@@ -125,14 +125,14 @@ func TestUserspaceEngineReconfig(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		wantRecvAt := map[tailcfg.NodeKey]mono.Time{
+		wantRecvAt := map[key.NodePublic]mono.Time{
 			nkFromHex(nodeHex): 0,
 		}
 		if got := ue.recvActivityAt; !reflect.DeepEqual(got, wantRecvAt) {
 			t.Errorf("wrong recvActivityAt\n got: %v\nwant: %v\n", got, wantRecvAt)
 		}
 
-		wantTrimmedNodes := map[tailcfg.NodeKey]bool{
+		wantTrimmedNodes := map[key.NodePublic]bool{
 			nkFromHex(nodeHex): true,
 		}
 		if got := ue.trimmedNodes; !reflect.DeepEqual(got, wantTrimmedNodes) {
@@ -209,7 +209,7 @@ func TestUserspaceEnginePortReconfig(t *testing.T) {
 	}
 }
 
-func nkFromHex(hex string) tailcfg.NodeKey {
+func nkFromHex(hex string) key.NodePublic {
 	if len(hex) != 64 {
 		panic(fmt.Sprintf("%q is len %d; want 64", hex, len(hex)))
 	}
@@ -217,7 +217,7 @@ func nkFromHex(hex string) tailcfg.NodeKey {
 	if err != nil {
 		panic(fmt.Sprintf("%q is not hex: %v", hex, err))
 	}
-	return tailcfg.NodeKeyFromNodePublic(k)
+	return k
 }
 
 // an experiment to see if genLocalAddrFunc was worth it. As of Go

wgengine/watchdog.go

@@ -17,6 +17,7 @@ import (
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/magicsock"
@@ -112,7 +113,7 @@ func (e *watchdogEngine) AddNetworkMapCallback(callback NetworkMapCallback) func
 	e.watchdog("AddNetworkMapCallback", func() { fn = e.wrap.AddNetworkMapCallback(callback) })
 	return func() { e.watchdog("RemoveNetworkMapCallback", fn) }
 }
-func (e *watchdogEngine) DiscoPublicKey() (k tailcfg.DiscoKey) {
+func (e *watchdogEngine) DiscoPublicKey() (k key.DiscoPublic) {
 	e.watchdog("DiscoPublicKey", func() { k = e.wrap.DiscoPublicKey() })
 	return k
 }

wgengine/wgcfg/device.go

@@ -8,10 +8,20 @@ import (
 	"io"
 	"sort"
 
+	"golang.zx2c4.com/wireguard/conn"
 	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/multierr"
 )
 
+// NewDevice returns a wireguard-go Device configured for Tailscale use.
+func NewDevice(tunDev tun.Device, bind conn.Bind, logger *device.Logger) *device.Device {
+	ret := device.NewDevice(tunDev, bind, logger)
+	ret.DisableSomeRoamingForBrokenMobileSemantics()
+	return ret
+}
+
 func DeviceConfig(d *device.Device) (*Config, error) {
 	r, w := io.Pipe()
 	errc := make(chan error, 1)
@@ -19,12 +29,10 @@ func DeviceConfig(d *device.Device) (*Config, error) {
 		errc <- d.IpcGetOperation(w)
 		w.Close()
 	}()
-	cfg, err := FromUAPI(r)
-	// Prefer errors from IpcGetOperation.
-	if setErr := <-errc; setErr != nil {
-		return nil, setErr
-	}
-	// Check FromUAPI error.
+	cfg, fromErr := FromUAPI(r)
+	r.Close()
+	getErr := <-errc
+	err := multierr.New(getErr, fromErr)
 	if err != nil {
 		return nil, err
 	}
@@ -51,14 +59,11 @@ func ReconfigDevice(d *device.Device, cfg *Config, logf logger.Logf) (err error)
 	errc := make(chan error, 1)
 	go func() {
 		errc <- d.IpcSetOperation(r)
-		w.Close()
+		r.Close()
 	}()
 
-	err = cfg.ToUAPI(w, prev)
+	toErr := cfg.ToUAPI(w, prev)
 	w.Close()
-	// Prefer errors from IpcSetOperation.
-	if setErr := <-errc; setErr != nil {
-		return setErr
-	}
-	return err // err (if any) from cfg.ToUAPI
+	setErr := <-errc
+	return multierr.New(setErr, toErr)
 }

wgengine/wgcfg/device_test.go

@@ -55,8 +55,8 @@ func TestDeviceConfig(t *testing.T) {
 		}},
 	}
 
-	device1 := device.NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "device1"))
-	device2 := device.NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "device2"))
+	device1 := NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "device1"))
+	device2 := NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "device2"))
 	defer device1.Close()
 	defer device2.Close()
 

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -10,11 +10,9 @@ import (
 	"fmt"
 	"strings"
 
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
-	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine/wgcfg"
@@ -73,8 +71,8 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 			continue
 		}
 		cfg.Peers = append(cfg.Peers, wgcfg.Peer{
-			PublicKey: key.NodePublicFromRaw32(mem.B(peer.Key[:])),
-			DiscoKey:  key.DiscoPublicFromRaw32(mem.B(peer.DiscoKey[:])),
+			PublicKey: peer.Key,
+			DiscoKey:  peer.DiscoKey,
 		})
 		cpeer := &cfg.Peers[len(cfg.Peers)-1]
 		if peer.KeepAlive {

wgengine/wgengine.go

@@ -11,6 +11,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/dns"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/monitor"
@@ -127,7 +128,7 @@ type Engine interface {
 
 	// DiscoPublicKey gets the public key used for path discovery
 	// messages.
-	DiscoPublicKey() tailcfg.DiscoKey
+	DiscoPublicKey() key.DiscoPublic
 
 	// UpdateStatus populates the network state using the provided
 	// status builder.