cmd/tailtlsproxy: introduce HTTPS proxy bridge

This allows you to bridge existing HTTP/S services into your tailnet
using the Let's Encrypt[1] functionality. This will allow you to run
multiple services on the same computer and still have them point at
different target HTTP services.

This also includes an example systemd template unit that lets you easily
set up multiple instances of tailtlsproxy on the same machine.

[1]: https://tailscale.com/blog/tls-certs/

Signed-off-by: Xe <xe@tailscale.com>

.github/workflows/linux.yml

@@ -28,6 +28,15 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
+    - name: Get QEMU
+      run: |
+        # The qemu in Ubuntu 20.04 (Focal) is too old; we need 5.x something
+        # to run Go binaries. 5.2.0 (Debian bullseye) empirically works, and
+        # use this PPA which brings in a modern qemu.
+        sudo add-apt-repository -y ppa:jacob/virtualisation
+        sudo apt-get -y update
+        sudo apt-get -y install qemu-user
+
     - name: Run tests on linux
       run: go test -bench=. -benchtime=1x ./...
 

Dockerfile

@@ -48,8 +48,9 @@ ARG VERSION_SHORT=""
 ENV VERSION_SHORT=$VERSION_SHORT
 ARG VERSION_GIT_HASH=""
 ENV VERSION_GIT_HASH=$VERSION_GIT_HASH
+ARG TARGETARCH
 
-RUN go install -tags=xversion -ldflags="\
+RUN GOARCH=$TARGETARCH go install -tags=xversion -ldflags="\
       -X tailscale.com/version.Long=$VERSION_LONG \
       -X tailscale.com/version.Short=$VERSION_SHORT \
       -X tailscale.com/version.GitCommit=$VERSION_GIT_HASH" \

Makefile

@@ -1,3 +1,5 @@
+IMAGE_REPO ?= tailscale/tailscale
+
 usage:
 	echo "See Makefile"
 
@@ -21,6 +23,10 @@ build386:
 buildlinuxarm:
 	GOOS=linux GOARCH=arm go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled
 
+
+buildmultiarchimage:
+	docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t ${IMAGE_REPO}:latest --push -f Dockerfile .
+
 check: staticcheck vet depaware buildwindows build386 buildlinuxarm
 
 staticcheck:

VERSION.txt

@@ -1 +1 @@
-1.17.0
+1.19.0

client/tailscale/tailscale.go

@@ -56,7 +56,7 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 			return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(TailscaledSocket, 41112)
+	return safesocket.Connect(TailscaledSocket, safesocket.WindowsLocalPort)
 }
 
 var (
@@ -196,6 +196,12 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return get200(ctx, "/localapi/v0/goroutines")
 }
 
+// DaemonMetrics returns the Tailscale daemon's metrics in
+// the Prometheus text exposition format.
+func DaemonMetrics(ctx context.Context) ([]byte, error) {
+	return get200(ctx, "/localapi/v0/metrics")
+}
+
 // Profile returns a pprof profile of the Tailscale daemon.
 func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string

cmd/derper/derper.go

@@ -141,7 +141,7 @@ func main() {
 
 	cfg := loadConfig()
 
-	serveTLS := tsweb.IsProd443(*addr)
+	serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual"
 
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)

cmd/tailscale/cli/cli.go

@@ -191,7 +191,7 @@ var rootArgs struct {
 var gotSignal syncs.AtomicBool
 
 func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	c, err := safesocket.Connect(rootArgs.socket, safesocket.WindowsLocalPort)
 	if err != nil {
 		if runtime.GOOS != "windows" && rootArgs.socket == "" {
 			fatalf("--socket cannot be empty")

cmd/tailscale/cli/debug.go

@@ -5,6 +5,8 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -14,7 +16,9 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -24,39 +28,76 @@ import (
 )
 
 var debugCmd = &ffcli.Command{
-	Name: "debug",
-	Exec: runDebug,
+	Name:     "debug",
+	Exec:     runDebug,
+	LongHelp: `"tailscale debug" contains misc debug facilities; it is not a stable interface.`,
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("debug")
-		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
-		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
-		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
-		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
-		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
-		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
-		fs.BoolVar(&debugArgs.env, "env", false, "dump environment")
-		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file; - for stdout")
 		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout")
 		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs
 	})(),
+	Subcommands: []*ffcli.Command{
+		{
+			Name:      "derp-map",
+			Exec:      runDERPMap,
+			ShortHelp: "print DERP map",
+		},
+		{
+			Name:      "daemon-goroutines",
+			Exec:      runDaemonGoroutines,
+			ShortHelp: "print tailscaled's goroutines",
+		},
+		{
+			Name:      "metrics",
+			Exec:      runDaemonMetrics,
+			ShortHelp: "print tailscaled's metrics",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("metrics")
+				fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values")
+				return fs
+			})(),
+		},
+		{
+			Name:      "env",
+			Exec:      runEnv,
+			ShortHelp: "print cmd/tailscale environment",
+		},
+		{
+			Name:      "local-creds",
+			Exec:      runLocalCreds,
+			ShortHelp: "print how to access Tailscale local API",
+		},
+		{
+			Name:      "prefs",
+			Exec:      runPrefs,
+			ShortHelp: "print prefs",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("prefs")
+				fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output")
+				return fs
+			})(),
+		},
+		{
+			Name:      "watch-ipn",
+			Exec:      runWatchIPN,
+			ShortHelp: "subscribe to IPN message bus",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("watch-ipn")
+				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
+				return fs
+			})(),
+		},
+	},
 }
 
 var debugArgs struct {
-	env        bool
-	localCreds bool
-	goroutines bool
-	ipn        bool
-	netMap     bool
-	derpMap    bool
-	file       string
-	prefs      bool
-	pretty     bool
-	cpuSec     int
-	cpuFile    string
-	memFile    string
+	file    string
+	cpuSec  int
+	cpuFile string
+	memFile string
 }
 
 func writeProfile(dst string, v []byte) error {
@@ -81,26 +122,9 @@ func runDebug(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("unknown arguments")
 	}
-	if debugArgs.env {
-		for _, e := range os.Environ() {
-			outln(e)
-		}
-		return nil
-	}
-	if debugArgs.localCreds {
-		port, token, err := safesocket.LocalTCPPortAndToken()
-		if err == nil {
-			printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
-			return nil
-		}
-		if runtime.GOOS == "windows" {
-			printf("curl http://localhost:41112/localapi/v0/status\n")
-			return nil
-		}
-		printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
-		return nil
-	}
+	var usedFlag bool
 	if out := debugArgs.cpuFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec)
 		if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil {
 			return err
@@ -112,6 +136,7 @@ func runDebug(ctx context.Context, args []string) error {
 		}
 	}
 	if out := debugArgs.memFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing memory profile ...")
 		if v, err := tailscale.Profile(ctx, "heap", 0); err != nil {
 			return err
@@ -122,55 +147,8 @@ func runDebug(ctx context.Context, args []string) error {
 			log.Printf("Memory profile written to %s", outName(out))
 		}
 	}
-	if debugArgs.prefs {
-		prefs, err := tailscale.GetPrefs(ctx)
-		if err != nil {
-			return err
-		}
-		if debugArgs.pretty {
-			outln(prefs.Pretty())
-		} else {
-			j, _ := json.MarshalIndent(prefs, "", "\t")
-			outln(string(j))
-		}
-		return nil
-	}
-	if debugArgs.goroutines {
-		goroutines, err := tailscale.Goroutines(ctx)
-		if err != nil {
-			return err
-		}
-		Stdout.Write(goroutines)
-		return nil
-	}
-	if debugArgs.derpMap {
-		dm, err := tailscale.CurrentDERPMap(ctx)
-		if err != nil {
-			return fmt.Errorf(
-				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
-			)
-		}
-		enc := json.NewEncoder(Stdout)
-		enc.SetIndent("", "\t")
-		enc.Encode(dm)
-		return nil
-	}
-	if debugArgs.ipn {
-		c, bc, ctx, cancel := connect(ctx)
-		defer cancel()
-
-		bc.SetNotifyCallback(func(n ipn.Notify) {
-			if !debugArgs.netMap {
-				n.NetMap = nil
-			}
-			j, _ := json.MarshalIndent(n, "", "\t")
-			printf("%s\n", j)
-		})
-		bc.RequestEngineStatus()
-		pump(ctx, bc, c)
-		return errors.New("exit")
-	}
 	if debugArgs.file != "" {
+		usedFlag = true // TODO(bradfitz): add "file" subcommand
 		if debugArgs.file == "get" {
 			wfs, err := tailscale.WaitingFiles(ctx)
 			if err != nil {
@@ -193,5 +171,148 @@ func runDebug(ctx context.Context, args []string) error {
 		io.Copy(Stdout, rc)
 		return nil
 	}
+	if usedFlag {
+		// TODO(bradfitz): delete this path when all debug flags are migrated
+		// to subcommands.
+		return nil
+	}
+	return errors.New("see 'tailscale debug --help")
+}
+
+func runLocalCreds(ctx context.Context, args []string) error {
+	port, token, err := safesocket.LocalTCPPortAndToken()
+	if err == nil {
+		printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
+		return nil
+	}
+	if runtime.GOOS == "windows" {
+		printf("curl http://localhost:%v/localapi/v0/status\n", safesocket.WindowsLocalPort)
+		return nil
+	}
+	printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
 	return nil
 }
+
+var prefsArgs struct {
+	pretty bool
+}
+
+func runPrefs(ctx context.Context, args []string) error {
+	prefs, err := tailscale.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+	if prefsArgs.pretty {
+		outln(prefs.Pretty())
+	} else {
+		j, _ := json.MarshalIndent(prefs, "", "\t")
+		outln(string(j))
+	}
+	return nil
+}
+
+var watchIPNArgs struct {
+	netmap bool
+}
+
+func runWatchIPN(ctx context.Context, args []string) error {
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if !watchIPNArgs.netmap {
+			n.NetMap = nil
+		}
+		j, _ := json.MarshalIndent(n, "", "\t")
+		printf("%s\n", j)
+	})
+	bc.RequestEngineStatus()
+	pump(ctx, bc, c)
+	return errors.New("exit")
+}
+
+func runDERPMap(ctx context.Context, args []string) error {
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+		)
+	}
+	enc := json.NewEncoder(Stdout)
+	enc.SetIndent("", "\t")
+	enc.Encode(dm)
+	return nil
+}
+
+func runEnv(ctx context.Context, args []string) error {
+	for _, e := range os.Environ() {
+		outln(e)
+	}
+	return nil
+}
+
+func runDaemonGoroutines(ctx context.Context, args []string) error {
+	goroutines, err := tailscale.Goroutines(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(goroutines)
+	return nil
+}
+
+var metricsArgs struct {
+	watch bool
+}
+
+func runDaemonMetrics(ctx context.Context, args []string) error {
+	last := map[string]int64{}
+	for {
+		out, err := tailscale.DaemonMetrics(ctx)
+		if err != nil {
+			return err
+		}
+		if !metricsArgs.watch {
+			Stdout.Write(out)
+			return nil
+		}
+		bs := bufio.NewScanner(bytes.NewReader(out))
+		type change struct {
+			name     string
+			from, to int64
+		}
+		var changes []change
+		var maxNameLen int
+		for bs.Scan() {
+			line := bytes.TrimSpace(bs.Bytes())
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			f := strings.Fields(string(line))
+			if len(f) != 2 {
+				continue
+			}
+			name := f[0]
+			n, _ := strconv.ParseInt(f[1], 10, 64)
+			prev, ok := last[name]
+			if ok && prev == n {
+				continue
+			}
+			last[name] = n
+			if !ok {
+				continue
+			}
+			changes = append(changes, change{name, prev, n})
+			if len(name) > maxNameLen {
+				maxNameLen = len(name)
+			}
+		}
+		if len(changes) > 0 {
+			format := fmt.Sprintf("%%-%ds %%+5d => %%v\n", maxNameLen)
+			for _, c := range changes {
+				fmt.Fprintf(Stdout, format, c.name, c.to-c.from, c.to)
+			}
+			io.WriteString(Stdout, "\n")
+		}
+		time.Sleep(time.Second)
+	}
+}

cmd/tailscale/depaware.txt

@@ -72,6 +72,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/structs                                  from tailscale.com/ipn+
+        tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli

cmd/tailscaled/depaware.txt

@@ -59,7 +59,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-        github.com/go-multierror/multierror                          from tailscale.com/cmd/tailscaled+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
@@ -92,27 +91,27 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/wgengine/router
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink                               from tailscale.com/wgengine/router
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/vishvananda/netlink
-   L    github.com/vishvananda/netns                                 from github.com/vishvananda/netlink+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+   W 💣 golang.zx2c4.com/wintun                                      from golang.zx2c4.com/wireguard/tun
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
      💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
      💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
-   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+   W 💣 golang.zx2c4.com/wireguard/ipc/namedpipe                     from golang.zx2c4.com/wireguard/ipc
         golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
         golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device
      💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
-   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/cmd/tailscaled+
         inet.af/netaddr                                              from inet.af/wf+
         inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
@@ -176,14 +175,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/log/filelogger                                 from tailscale.com/ipn/ipnserver
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled
-        tailscale.com/logtail                                        from tailscale.com/logpolicy
+        tailscale.com/logtail                                        from tailscale.com/logpolicy+
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
      💣 tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/net/dns/resolver                               from tailscale.com/net/dns+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
-        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
      💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock
@@ -222,12 +221,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/clientmetric                              from tailscale.com/ipn/localapi+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/dns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy

cmd/tailscaled/tailscaled.go

@@ -20,6 +20,7 @@ import (
 	"net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
@@ -27,17 +28,20 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/socks5/tssocks"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -78,6 +82,7 @@ var args struct {
 	debug          string
 	port           uint16
 	statepath      string
+	statedir       string
 	socketpath     string
 	birdSocketPath string
 	verbose        int
@@ -114,7 +119,8 @@ func main() {
 	flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM")
+	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "absolute path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM. If empty and --statedir is provided, the default is <statedir>/tailscaled.state")
+	flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -202,6 +208,16 @@ func trySynologyMigration(p string) error {
 	return nil
 }
 
+func statePathOrDefault() string {
+	if args.statepath != "" {
+		return args.statepath
+	}
+	if args.statedir != "" {
+		return filepath.Join(args.statedir, "tailscaled.state")
+	}
+	return ""
+}
+
 func ipnServerOpts() (o ipnserver.Options) {
 	// Allow changing the OS-specific IPN behavior for tests
 	// so we can e.g. test Windows-specific behaviors on Linux.
@@ -210,9 +226,15 @@ func ipnServerOpts() (o ipnserver.Options) {
 		goos = runtime.GOOS
 	}
 
-	o.Port = 41112
-	o.StatePath = args.statepath
-	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+	o.VarRoot = args.statedir
+
+	// If an absolute --state is provided but not --statedir, try to derive
+	// a state directory.
+	if o.VarRoot == "" && filepath.IsAbs(args.statepath) {
+		if dir := filepath.Dir(args.statepath); strings.EqualFold(filepath.Base(dir), "tailscale") {
+			o.VarRoot = dir
+		}
+	}
 
 	switch goos {
 	default:
@@ -227,7 +249,7 @@ func ipnServerOpts() (o ipnserver.Options) {
 func run() error {
 	var err error
 
-	pol := logpolicy.New("tailnode.log.tailscale.io")
+	pol := logpolicy.New(logtail.CollectionNode)
 	pol.SetVerbosityLevel(args.verbose)
 	defer func() {
 		// Finish uploading logs after closing everything else.
@@ -261,10 +283,10 @@ func run() error {
 		return nil
 	}
 
-	if args.statepath == "" {
-		log.Fatalf("--state is required")
+	if args.statepath == "" && args.statedir == "" {
+		log.Fatalf("--statedir (or at least --state) is required")
 	}
-	if err := trySynologyMigration(args.statepath); err != nil {
+	if err := trySynologyMigration(statePathOrDefault()); err != nil {
 		log.Printf("error in synology migration: %v", err)
 	}
 
@@ -289,10 +311,14 @@ func run() error {
 		return err
 	}
 
-	var ns *netstack.Impl
-	if useNetstack || wrapNetstack {
-		onlySubnets := wrapNetstack && !useNetstack
-		ns = mustStartNetstack(logf, e, onlySubnets)
+	ns, err := newNetstack(logf, e)
+	if err != nil {
+		return fmt.Errorf("newNetstack: %w", err)
+	}
+	ns.ProcessLocalIPs = useNetstack
+	ns.ProcessSubnets = useNetstack || wrapNetstack
+	if err := ns.Start(); err != nil {
+		log.Fatalf("failed to start netstack: %v", err)
 	}
 
 	if socksListener != nil || httpProxyListener != nil {
@@ -332,8 +358,27 @@ func run() error {
 	}()
 
 	opts := ipnServerOpts()
-	opts.DebugMux = debugMux
-	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+	srv, err := ipnserver.New(logf, pol.PublicID.String(), store, e, nil, opts)
+	if err != nil {
+		logf("ipnserver.New: %v", err)
+		return err
+	}
+
+	if debugMux != nil {
+		debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus)
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = srv.Run(ctx, ln)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
 		logf("ipnserver.Run: %v", err)
@@ -357,7 +402,7 @@ func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, us
 		logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err)
 		errs = append(errs, err)
 	}
-	return nil, false, multierror.New(errs)
+	return nil, false, multierr.New(errs...)
 }
 
 var wrapNetstack = shouldWrapNetstack()
@@ -435,6 +480,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 
 func newDebugMux() *http.ServeMux {
 	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/metrics", servePrometheusMetrics)
 	mux.HandleFunc("/debug/pprof/", pprof.Index)
 	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
@@ -443,6 +489,11 @@ func newDebugMux() *http.ServeMux {
 	return mux
 }
 
+func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 func runDebugServer(mux *http.ServeMux, addr string) {
 	srv := &http.Server{
 		Addr:    addr,
@@ -453,19 +504,12 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 	}
 }
 
-func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) *netstack.Impl {
+func newNetstack(logf logger.Logf, e wgengine.Engine) (*netstack.Impl, error) {
 	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
 	if !ok {
-		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+		return nil, fmt.Errorf("%T is not a wgengine.InternalsGetter", e)
 	}
-	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
-	if err != nil {
-		log.Fatalf("netstack.Create: %v", err)
-	}
-	if err := ns.Start(); err != nil {
-		log.Fatalf("failed to start netstack: %v", err)
-	}
-	return ns
+	return netstack.Create(logf, tunDev, e, magicConn)
 }
 
 func mustStartTCPListener(name, addr string) net.Listener {

cmd/tailscaled/tailscaled_windows.go

@@ -33,6 +33,7 @@ import (
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
@@ -74,7 +75,11 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	go func() {
 		defer close(doneCh)
 		args := []string{"/subproc", service.Policy.PublicID.String()}
-		ipnserver.BabysitProc(ctx, args, log.Printf)
+		// Make a logger without a date prefix, as filelogger
+		// and logtail both already add their own. All we really want
+		// from the log package is the automatic newline.
+		logger := log.New(os.Stderr, "", 0)
+		ipnserver.BabysitProc(ctx, args, logger.Printf)
 	}()
 
 	changes <- svc.Status{State: svc.Running, Accepts: svcAccepts}
@@ -202,9 +207,14 @@ func startIPNServer(ctx context.Context, logid string) error {
 			dev.Close()
 			return nil, fmt.Errorf("engine: %w", err)
 		}
-		onlySubnets := true
-		if wrapNetstack {
-			mustStartNetstack(logf, eng, onlySubnets)
+		ns, err := newNetstack(logf, eng)
+		if err != nil {
+			return nil, fmt.Errorf("newNetstack: %w", err)
+		}
+		ns.ProcessLocalIPs = false
+		ns.ProcessSubnets = wrapNetstack
+		if err := ns.Start(); err != nil {
+			return nil, fmt.Errorf("failed to start netstack: %w", err)
 		}
 		return wgengine.NewWatchdog(eng), nil
 	}
@@ -266,7 +276,18 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = ipnserver.Run(ctx, logf, ln, store, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tailtlsproxy/main.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/tsnet"
+)
+
+func envOr(name, def string) string {
+	if val, ok := os.LookupEnv(name); ok {
+		return val
+	}
+
+	return def
+}
+
+var (
+	bind     = flag.String("bind", ":443", "TCP hostport to bind to for TLS http traffic")
+	hostname = flag.String("hostname", envOr("HOSTNAME", "toolname"), "hostname to register on tailnet (can be set with $HOSTNAME)")
+	auth     = flag.Bool("auth", false, "if set, authenticate with tailscale (enables verbose logging)")
+	v        = flag.Bool("v", false, "if set, enable verbose tailscale logs")
+	to       = flag.String("to", envOr("TO", "http://127.0.0.1:3030"), "HTTP/S url to reverse proxy to (can be set with $TO)")
+)
+
+func main() {
+	os.Setenv("TAILSCALE_USE_WIP_CODE", "true")
+	flag.Parse()
+
+	srv := tsnet.Server{
+		Hostname: *hostname,
+	}
+
+	if *auth || *v {
+		srv.Logf = log.Printf
+	} else {
+		srv.Logf = func(string, ...interface{}) {}
+	}
+
+	if *auth {
+		os.Setenv("TS_LOGIN", "1")
+	}
+
+	ln, err := srv.Listen("tcp", *bind)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	u, err := url.Parse(*to)
+	if err != nil {
+		log.Fatalf("%s wasn't a valid URL: %v", *to, err)
+	}
+
+	h := httputil.NewSingleHostReverseProxy(u)
+
+	ln = tls.NewListener(ln, &tls.Config{
+		GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			c, err := tailscale.GetCertificate(chi)
+			if err != nil {
+				log.Println(err)
+			}
+			return c, err
+		},
+	})
+
+	s := &http.Server{
+		IdleTimeout: 5 * time.Minute,
+		Addr:        *bind,
+		Handler:     h,
+	}
+
+	log.Printf("listening for https on https://%s.your.tailcert.domain and forwarding to %s", *hostname, *to)
+
+	log.Fatal(s.Serve(ln))
+}

cmd/tailtlsproxy/tailtlsproxy-test.defaults

@@ -0,0 +1,2 @@
+HOSTNAME=test
+TO=http://127.0.0.1:3030

cmd/tailtlsproxy/tailtlsproxy@.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Tailscale TLS Proxy bridge for %i
+After=network.target
+
+[Service]
+Environment=HOME=/var/lib/private/tailtlsproxy-%i
+EnvironmentFile=/etc/default/tailtlsproxy-%i
+ExecStart=/usr/bin/tailtlsproxy
+Restart=on-failure
+RuntimeDirectory=tailtlsproxy-%i
+RuntimeDirectoryMode=0755
+StateDirectory=tailtlsproxy-%i
+StateDirectoryMode=0700
+CacheDirectory=tailtlsproxy-%i
+CacheDirectoryMode=0750
+DynamicUser=yes
+
+[Install]
+WantedBy=multi-user.target

control/controlclient/client.go

@@ -79,3 +79,9 @@ type Client interface {
 	// requesting a DNS record be created or updated.
 	SetDNS(context.Context, *tailcfg.SetDNSRequest) error
 }
+
+// UserVisibleError is an error that should be shown to users.
+type UserVisibleError string
+
+func (e UserVisibleError) Error() string            { return string(e) }
+func (e UserVisibleError) UserVisibleError() string { return string(e) }

control/controlclient/direct.go

@@ -46,6 +46,7 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/systemd"
 	"tailscale.com/wgengine/monitor"
 )
@@ -60,7 +61,7 @@ type Direct struct {
 	keepAlive              bool
 	logf                   logger.Logf
 	linkMon                *monitor.Mon // or nil
-	discoPubKey            tailcfg.DiscoKey
+	discoPubKey            key.DiscoPublic
 	getMachinePrivKey      func() (key.MachinePrivate, error)
 	debugFlags             []string
 	keepSharerAndUserSplit bool
@@ -88,7 +89,7 @@ type Options struct {
 	AuthKey              string                             // optional node auth key for auto registration
 	TimeNow              func() time.Time                   // time.Now implementation used by Client
 	Hostinfo             *tailcfg.Hostinfo                  // non-nil passes ownership, nil means to use default using os.Hostname, etc
-	DiscoPublicKey       tailcfg.DiscoKey
+	DiscoPublicKey       key.DiscoPublic
 	NewDecompressor      func() (Decompressor, error)
 	KeepAlive            bool
 	Logf                 logger.Logf
@@ -146,13 +147,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	}
 
 	httpc := opts.HTTPTestClient
+	if httpc == nil && runtime.GOOS == "js" {
+		// In js/wasm, net/http.Transport (as of Go 1.18) will
+		// only use the browser's Fetch API if you're using
+		// the DefaultClient (or a client without dial hooks
+		// etc set).
+		httpc = http.DefaultClient
+	}
 	if httpc == nil {
 		dnsCache := &dnscache.Resolver{
 			Forward:          dnscache.Get().Forward, // use default cache's forwarder
 			UseLastGood:      true,
 			LookupIPFallback: dnsfallback.Lookup,
 		}
-		dialer := netns.NewDialer()
+		dialer := netns.NewDialer(opts.Logf)
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
@@ -284,8 +292,8 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -357,9 +365,9 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	now := time.Now().Round(time.Second)
 	request := tailcfg.RegisterRequest{
 		Version:    1,
-		OldNodeKey: tailcfg.NodeKeyFromNodePublic(oldNodeKey),
-		NodeKey:    tailcfg.NodeKeyFromNodePublic(tryingNewKey.Public()),
-		Hostinfo:   hostinfo,
+		OldNodeKey: oldNodeKey,
+		NodeKey:    tryingNewKey.Public(),
+		Hostinfo:   hi,
 		Followup:   opt.URL,
 		Timestamp:  &now,
 		Ephemeral:  (opt.Flags & LoginEphemeral) != 0,
@@ -431,7 +439,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
 
 	if resp.Error != "" {
-		return false, "", errors.New(resp.Error)
+		return false, "", UserVisibleError(resp.Error)
 	}
 	if resp.NodeKeyExpired {
 		if regen {
@@ -551,12 +559,21 @@ const pollTimeout = 120 * time.Second
 
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	metricMapRequests.Add(1)
+	metricMapRequestsActive.Add(1)
+	defer metricMapRequestsActive.Add(-1)
+	if maxPolls == -1 {
+		metricMapRequestsPoll.Add(1)
+	} else {
+		metricMapRequestsLite.Add(1)
+	}
+
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	localPort := c.localPort
 	var epStrs []string
 	var epTypes []tailcfg.EndpointType
@@ -595,18 +612,18 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	request := &tailcfg.MapRequest{
 		Version:       tailcfg.CurrentMapRequestVersion,
 		KeepAlive:     c.keepAlive,
-		NodeKey:       tailcfg.NodeKeyFromNodePublic(persist.PrivateNodeKey.Public()),
+		NodeKey:       persist.PrivateNodeKey.Public(),
 		DiscoKey:      c.discoPubKey,
 		Endpoints:     epStrs,
 		EndpointTypes: epTypes,
 		Stream:        allowStream,
-		Hostinfo:      hostinfo,
+		Hostinfo:      hi,
 		DebugFlags:    c.debugFlags,
 		OmitPeers:     cb == nil,
 	}
 	var extraDebugFlags []string
-	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
-		ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
+	if hi != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
+		ipForwardingBroken(hi.RoutableIPs, c.linkMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
 	if health.RouterHealth() != nil {
@@ -615,6 +632,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	if health.NetworkCategoryHealth() != nil {
 		extraDebugFlags = append(extraDebugFlags, "warn-network-category-unhealthy")
 	}
+	if hostinfo.DisabledEtcAptSource() {
+		extraDebugFlags = append(extraDebugFlags, "warn-etc-apt-source-disabled")
+	}
 	if len(extraDebugFlags) > 0 {
 		old := request.DebugFlags
 		request.DebugFlags = append(old[:len(old):len(old)], extraDebugFlags...)
@@ -737,11 +757,14 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return err
 		}
 
+		metricMapResponseMessages.Add(1)
+
 		if allowStream {
 			health.GotStreamedMapResponse()
 		}
 
 		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+			metricMapResponsePings.Add(1)
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -758,13 +781,23 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return ctx.Err()
 		}
 		if resp.KeepAlive {
+			metricMapResponseKeepAlives.Add(1)
 			continue
 		}
 
+		metricMapResponseMap.Add(1)
+		if i > 0 {
+			metricMapResponseMapDelta.Add(1)
+		}
+
 		hasDebug := resp.Debug != nil
 		// being conservative here, if Debug not present set to False
 		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
 		if hasDebug {
+			if code := resp.Debug.Exit; code != nil {
+				c.logf("exiting process with status %v per controlplane", *code)
+				os.Exit(*code)
+			}
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}
@@ -1167,7 +1200,13 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 
 // SetDNS sends the SetDNSRequest request to the control plane server,
 // requesting a DNS record be created or updated.
-func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) (err error) {
+	metricSetDNS.Add(1)
+	defer func() {
+		if err != nil {
+			metricSetDNSError.Add(1)
+		}
+	}()
 	c.mu.Lock()
 	serverKey := c.serverKey
 	c.mu.Unlock()
@@ -1267,3 +1306,20 @@ func postPingResult(now time.Time, logf logger.Logf, c *http.Client, pr *tailcfg
 	}
 	return nil
 }
+
+var (
+	metricMapRequestsActive = clientmetric.NewGauge("controlclient_map_requests_active")
+
+	metricMapRequests     = clientmetric.NewCounter("controlclient_map_requests")
+	metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite")
+	metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll")
+
+	metricMapResponseMessages   = clientmetric.NewCounter("controlclient_map_response_message") // any message type
+	metricMapResponsePings      = clientmetric.NewCounter("controlclient_map_response_ping")
+	metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive")
+	metricMapResponseMap        = clientmetric.NewCounter("controlclient_map_response_map")       // any non-keepalive map response
+	metricMapResponseMapDelta   = clientmetric.NewCounter("controlclient_map_response_map_delta") // 2nd+ non-keepalive map response
+
+	metricSetDNS      = clientmetric.NewCounter("controlclient_setdns")
+	metricSetDNSError = clientmetric.NewCounter("controlclient_setdns_error")
+)

control/controlclient/map.go

@@ -110,7 +110,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 	}
 
 	nm := &netmap.NetworkMap{
-		NodeKey:         tailcfg.NodeKeyFromNodePublic(ms.privateNodeKey.Public()),
+		NodeKey:         ms.privateNodeKey.Public(),
 		PrivateKey:      ms.privateNodeKey,
 		MachineKey:      ms.machinePubKey,
 		Peers:           resp.Peers,

control/noise/conn.go

@@ -0,0 +1,359 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package noise implements the base transport of the Tailscale 2021
+// control protocol.
+//
+// The base transport implements Noise IK, instantiated with
+// Curve25519, ChaCha20Poly1305 and BLAKE2s.
+package noise
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"tailscale.com/types/key"
+)
+
+const (
+	// maxMessageSize is the maximum size of a protocol frame on the
+	// wire, including header and payload.
+	maxMessageSize = 4096
+	// maxCiphertextSize is the maximum amount of ciphertext bytes
+	// that one protocol frame can carry, after framing.
+	maxCiphertextSize = maxMessageSize - 3
+	// maxPlaintextSize is the maximum amount of plaintext bytes that
+	// one protocol frame can carry, after encryption and framing.
+	maxPlaintextSize = maxCiphertextSize - chp.Overhead
+)
+
+// A Conn is a secured Noise connection. It implements the net.Conn
+// interface, with the unusual trait that any write error (including a
+// SetWriteDeadline induced i/o timeout) causes all future writes to
+// fail.
+type Conn struct {
+	conn          net.Conn
+	version       uint16
+	peer          key.MachinePublic
+	handshakeHash [blake2s.Size]byte
+	rx            rxState
+	tx            txState
+}
+
+// rxState is all the Conn state that Read uses.
+type rxState struct {
+	sync.Mutex
+	cipher    cipher.AEAD
+	nonce     nonce
+	buf       [maxMessageSize]byte
+	n         int    // number of valid bytes in buf
+	next      int    // offset of next undecrypted packet
+	plaintext []byte // slice into buf of decrypted bytes
+}
+
+// txState is all the Conn state that Write uses.
+type txState struct {
+	sync.Mutex
+	cipher cipher.AEAD
+	nonce  nonce
+	buf    [maxMessageSize]byte
+	err    error // records the first partial write error for all future calls
+}
+
+// ProtocolVersion returns the protocol version that was used to
+// establish this Conn.
+func (c *Conn) ProtocolVersion() int {
+	return int(c.version)
+}
+
+// HandshakeHash returns the Noise handshake hash for the connection,
+// which can be used to bind other messages to this connection
+// (i.e. to ensure that the message wasn't replayed from a different
+// connection).
+func (c *Conn) HandshakeHash() [blake2s.Size]byte {
+	return c.handshakeHash
+}
+
+// Peer returns the peer's long-term public key.
+func (c *Conn) Peer() key.MachinePublic {
+	return c.peer
+}
+
+// readNLocked reads into c.rx.buf until buf contains at least total
+// bytes. Returns a slice of the total bytes in rxBuf, or an
+// error if fewer than total bytes are available.
+func (c *Conn) readNLocked(total int) ([]byte, error) {
+	if total > maxMessageSize {
+		return nil, errReadTooBig{total}
+	}
+	for {
+		if total <= c.rx.n {
+			return c.rx.buf[:total], nil
+		}
+
+		n, err := c.conn.Read(c.rx.buf[c.rx.n:])
+		c.rx.n += n
+		if err != nil {
+			return nil, err
+		}
+	}
+}
+
+// decryptLocked decrypts msg (which is header+ciphertext) in-place
+// and sets c.rx.plaintext to the decrypted bytes.
+func (c *Conn) decryptLocked(msg []byte) (err error) {
+	if msgType := msg[0]; msgType != msgTypeRecord {
+		return fmt.Errorf("received message with unexpected type %d, want %d", msgType, msgTypeRecord)
+	}
+	// We don't check the length field here, because the caller
+	// already did in order to figure out how big the msg slice should
+	// be.
+	ciphertext := msg[headerLen:]
+
+	if !c.rx.nonce.Valid() {
+		return errCipherExhausted{}
+	}
+
+	c.rx.plaintext, err = c.rx.cipher.Open(ciphertext[:0], c.rx.nonce[:], ciphertext, nil)
+	c.rx.nonce.Increment()
+
+	if err != nil {
+		// Once a decryption has failed, our Conn is no longer
+		// synchronized with our peer. Nuke the cipher state to be
+		// safe, so that no further decryptions are attempted. Future
+		// read attempts will return net.ErrClosed.
+		c.rx.cipher = nil
+	}
+	return err
+}
+
+// encryptLocked encrypts plaintext into c.tx.buf (including the
+// packet header) and returns a slice of the ciphertext, or an error
+// if the cipher is exhausted (i.e. can no longer be used safely).
+func (c *Conn) encryptLocked(plaintext []byte) ([]byte, error) {
+	if !c.tx.nonce.Valid() {
+		// Received 2^64-1 messages on this cipher state. Connection
+		// is no longer usable.
+		return nil, errCipherExhausted{}
+	}
+
+	c.tx.buf[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(c.tx.buf[1:headerLen], uint16(len(plaintext)+chp.Overhead))
+	ret := c.tx.cipher.Seal(c.tx.buf[:headerLen], c.tx.nonce[:], plaintext, nil)
+	c.tx.nonce.Increment()
+
+	return ret, nil
+}
+
+// wholeMessageLocked returns a slice of one whole Noise transport
+// message from c.rx.buf, if one whole message is available, and
+// advances the read state to the next Noise message in the
+// buffer. Returns nil without advancing read state if there isn't one
+// whole message in c.rx.buf.
+func (c *Conn) wholeMessageLocked() []byte {
+	available := c.rx.n - c.rx.next
+	if available < headerLen {
+		return nil
+	}
+	bs := c.rx.buf[c.rx.next:c.rx.n]
+	totalSize := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	if len(bs) < totalSize {
+		return nil
+	}
+	c.rx.next += totalSize
+	return bs[:totalSize]
+}
+
+// decryptOneLocked decrypts one Noise transport message, reading from
+// c.conn as needed, and sets c.rx.plaintext to point to the decrypted
+// bytes. c.rx.plaintext is only valid if err == nil.
+func (c *Conn) decryptOneLocked() error {
+	c.rx.plaintext = nil
+
+	// Fast path: do we have one whole ciphertext frame buffered
+	// already?
+	if bs := c.wholeMessageLocked(); bs != nil {
+		return c.decryptLocked(bs)
+	}
+
+	if c.rx.next != 0 {
+		// To simplify the read logic, move the remainder of the
+		// buffered bytes back to the head of the buffer, so we can
+		// grow it without worrying about wraparound.
+		c.rx.n = copy(c.rx.buf[:], c.rx.buf[c.rx.next:c.rx.n])
+		c.rx.next = 0
+	}
+
+	bs, err := c.readNLocked(headerLen)
+	if err != nil {
+		return err
+	}
+	// The rest of the header (besides the length field) gets verified
+	// in decryptLocked, not here.
+	messageLen := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	bs, err = c.readNLocked(messageLen)
+	if err != nil {
+		return err
+	}
+
+	c.rx.next = len(bs)
+
+	return c.decryptLocked(bs)
+}
+
+// Read implements io.Reader.
+func (c *Conn) Read(bs []byte) (int, error) {
+	c.rx.Lock()
+	defer c.rx.Unlock()
+
+	if c.rx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+	// If no plaintext is buffered, decrypt incoming frames until we
+	// have some plaintext. Zero-byte Noise frames are allowed in this
+	// protocol, which is why we have to loop here rather than decrypt
+	// a single additional frame.
+	for len(c.rx.plaintext) == 0 {
+		if err := c.decryptOneLocked(); err != nil {
+			return 0, err
+		}
+	}
+	n := copy(bs, c.rx.plaintext)
+	c.rx.plaintext = c.rx.plaintext[n:]
+	return n, nil
+}
+
+// Write implements io.Writer.
+func (c *Conn) Write(bs []byte) (n int, err error) {
+	c.tx.Lock()
+	defer c.tx.Unlock()
+
+	if c.tx.err != nil {
+		return 0, c.tx.err
+	}
+	defer func() {
+		if err != nil {
+			// All write errors are fatal for this conn, so clear the
+			// cipher state whenever an error happens.
+			c.tx.cipher = nil
+		}
+		if c.tx.err == nil {
+			// Only set c.tx.err if not nil so that we can return one
+			// error on the first failure, and a different one for
+			// subsequent calls. See the error handling around Write
+			// below for why.
+			c.tx.err = err
+		}
+	}()
+
+	if c.tx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+
+	var sent int
+	for len(bs) > 0 {
+		toSend := bs
+		if len(toSend) > maxPlaintextSize {
+			toSend = bs[:maxPlaintextSize]
+		}
+		bs = bs[len(toSend):]
+
+		ciphertext, err := c.encryptLocked(toSend)
+		if err != nil {
+			return 0, err
+		}
+
+		n, err := c.conn.Write(ciphertext)
+		sent += n
+		if err != nil {
+			// Return the raw error on the Write that actually
+			// failed. For future writes, return that error wrapped in
+			// a desync error.
+			c.tx.err = errPartialWrite{err}
+			return sent, err
+		}
+	}
+	return sent, nil
+}
+
+// Close implements io.Closer.
+func (c *Conn) Close() error {
+	closeErr := c.conn.Close() // unblocks any waiting reads or writes
+
+	// Remove references to live cipher state. Strictly speaking this
+	// is unnecessary, but we want to try and hand the active cipher
+	// state to the garbage collector promptly, to preserve perfect
+	// forward secrecy as much as we can.
+	c.rx.Lock()
+	c.rx.cipher = nil
+	c.rx.Unlock()
+	c.tx.Lock()
+	c.tx.cipher = nil
+	c.tx.Unlock()
+	return closeErr
+}
+
+func (c *Conn) LocalAddr() net.Addr                { return c.conn.LocalAddr() }
+func (c *Conn) RemoteAddr() net.Addr               { return c.conn.RemoteAddr() }
+func (c *Conn) SetDeadline(t time.Time) error      { return c.conn.SetDeadline(t) }
+func (c *Conn) SetReadDeadline(t time.Time) error  { return c.conn.SetReadDeadline(t) }
+func (c *Conn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) }
+
+// errCipherExhausted is the error returned when we run out of nonces
+// on a cipher.
+type errCipherExhausted struct{}
+
+func (errCipherExhausted) Error() string {
+	return "cipher exhausted, no more nonces available for current key"
+}
+func (errCipherExhausted) Timeout() bool   { return false }
+func (errCipherExhausted) Temporary() bool { return false }
+
+// errPartialWrite is the error returned when the cipher state has
+// become unusable due to a past partial write.
+type errPartialWrite struct {
+	err error
+}
+
+func (e errPartialWrite) Error() string {
+	return fmt.Sprintf("cipher state desynchronized due to partial write (%v)", e.err)
+}
+func (e errPartialWrite) Unwrap() error   { return e.err }
+func (e errPartialWrite) Temporary() bool { return false }
+func (e errPartialWrite) Timeout() bool   { return false }
+
+// errReadTooBig is the error returned when the peer sent an
+// unacceptably large Noise frame.
+type errReadTooBig struct {
+	requested int
+}
+
+func (e errReadTooBig) Error() string {
+	return fmt.Sprintf("requested read of %d bytes exceeds max allowed Noise frame size", e.requested)
+}
+func (e errReadTooBig) Temporary() bool {
+	// permanent error because this error only occurs when our peer
+	// sends us a frame so large we're unwilling to ever decode it.
+	return false
+}
+func (e errReadTooBig) Timeout() bool { return false }
+
+type nonce [chp.NonceSize]byte
+
+func (n *nonce) Valid() bool {
+	return binary.BigEndian.Uint32(n[:4]) == 0 && binary.BigEndian.Uint64(n[4:]) != invalidNonce
+}
+
+func (n *nonce) Increment() {
+	if !n.Valid() {
+		panic("increment of invalid nonce")
+	}
+	binary.BigEndian.PutUint64(n[4:], 1+binary.BigEndian.Uint64(n[4:]))
+}

control/noise/conn_test.go

@@ -0,0 +1,339 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"testing/iotest"
+
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/net/nettest"
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestMessageSize(t *testing.T) {
+	// This test is a regression guard against someone looking at
+	// maxCiphertextSize, going "huh, we could be more efficient if it
+	// were larger, and accidentally violating the Noise spec. Do not
+	// change this max value, it's a deliberate limitation of the
+	// cryptographic protocol we use (see Section 3 "Message Format"
+	// of the Noise spec).
+	const max = 65535
+	if maxCiphertextSize > max {
+		t.Fatalf("max ciphertext size is %d, which is larger than the maximum noise message size %d", maxCiphertextSize, max)
+	}
+}
+
+func TestConnBasic(t *testing.T) {
+	client, server := pair(t)
+
+	sb := sinkReads(server)
+
+	want := "test"
+	if _, err := io.WriteString(client, want); err != nil {
+		t.Fatalf("client write failed: %v", err)
+	}
+	client.Close()
+
+	if got := sb.String(4); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatal("client close wasn't seen by server")
+	}
+	if sb.Total() != 4 {
+		t.Fatalf("wrong amount of bytes received: got %d, want 4", sb.Total())
+	}
+}
+
+// bufferedWriteConn wraps a net.Conn and gives control over how
+// Writes get batched out.
+type bufferedWriteConn struct {
+	net.Conn
+	w           *bufio.Writer
+	manualFlush bool
+}
+
+func (c *bufferedWriteConn) Write(bs []byte) (int, error) {
+	n, err := c.w.Write(bs)
+	if err == nil && !c.manualFlush {
+		err = c.w.Flush()
+	}
+	return n, err
+}
+
+// TestFastPath exercises the Read codepath that can receive multiple
+// Noise frames at once and decode each in turn without making another
+// syscall.
+func TestFastPath(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	b := &bufferedWriteConn{s1, bufio.NewWriterSize(s1, 10000), false}
+	client, server := pairWithConns(t, b, s2)
+
+	b.manualFlush = true
+
+	sb := sinkReads(server)
+
+	const packets = 10
+	s := "test"
+	for i := 0; i < packets; i++ {
+		// Many separate writes, to force separate Noise frames that
+		// all get buffered up and then all sent as a single slice to
+		// the server.
+		if _, err := io.WriteString(client, s); err != nil {
+			t.Fatalf("client write1 failed: %v", err)
+		}
+	}
+	if err := b.w.Flush(); err != nil {
+		t.Fatalf("client flush failed: %v", err)
+	}
+	client.Close()
+
+	want := strings.Repeat(s, packets)
+	if got := sb.String(len(want)); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatalf("client close wasn't seen by server")
+	}
+}
+
+// Writes things larger than a single Noise frame, to check the
+// chunking on the encoder and decoder.
+func TestBigData(t *testing.T) {
+	client, server := pair(t)
+
+	serverReads := sinkReads(server)
+	clientReads := sinkReads(client)
+
+	const sz = 15 * 1024 // 15KiB
+	clientStr := strings.Repeat("abcde", sz/5)
+	serverStr := strings.Repeat("fghij", sz/5*2)
+
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+	if _, err := io.WriteString(server, serverStr); err != nil {
+		t.Fatalf("writing server>client: %v", err)
+	}
+
+	if serverGot := serverReads.String(sz); serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+	if clientGot := clientReads.String(2 * sz); clientGot != serverStr {
+		t.Error("client didn't receive what server sent")
+	}
+
+	getNonce := func(n [chp.NonceSize]byte) uint64 {
+		if binary.BigEndian.Uint32(n[:4]) != 0 {
+			panic("unexpected nonce")
+		}
+		return binary.BigEndian.Uint64(n[4:])
+	}
+
+	// Reach into the Conns and verify the cipher nonces advanced as
+	// expected.
+	if getNonce(client.tx.nonce) != getNonce(server.rx.nonce) {
+		t.Error("desynchronized client tx nonce")
+	}
+	if getNonce(server.tx.nonce) != getNonce(client.rx.nonce) {
+		t.Error("desynchronized server tx nonce")
+	}
+	if n := getNonce(client.tx.nonce); n != 4 {
+		t.Errorf("wrong client tx nonce, got %d want 4", n)
+	}
+	if n := getNonce(server.tx.nonce); n != 8 {
+		t.Errorf("wrong client tx nonce, got %d want 8", n)
+	}
+}
+
+// readerConn wraps a net.Conn and routes its Reads through a separate
+// io.Reader.
+type readerConn struct {
+	net.Conn
+	r io.Reader
+}
+
+func (c readerConn) Read(bs []byte) (int, error) { return c.r.Read(bs) }
+
+// Check that the receiver can handle not being able to read an entire
+// frame in a single syscall.
+func TestDataTrickle(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	client, server := pairWithConns(t, s1, readerConn{s2, iotest.OneByteReader(s2)})
+	serverReads := sinkReads(server)
+
+	const sz = 10000
+	clientStr := strings.Repeat("abcde", sz/5)
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+
+	serverGot := serverReads.String(sz)
+	if serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+}
+
+func TestConnStd(t *testing.T) {
+	// You can run this test manually, and noise.Conn should pass all
+	// of them except for TestConn/PastTimeout,
+	// TestConn/FutureTimeout, TestConn/ConcurrentMethods, because
+	// those tests assume that write errors are recoverable, and
+	// they're not on our Conn due to cipher security.
+	t.Skip("not all tests can pass on this Conn, see https://github.com/golang/go/issues/46977")
+	nettest.TestConn(t, func() (c1 net.Conn, c2 net.Conn, stop func(), err error) {
+		s1, s2 := tsnettest.NewConn("noise", 4096)
+		controlKey := key.NewMachine()
+		machineKey := key.NewMachine()
+		serverErr := make(chan error, 1)
+		go func() {
+			var err error
+			c2, err = Server(context.Background(), s2, controlKey)
+			serverErr <- err
+		}()
+		c1, err = Client(context.Background(), s1, machineKey, controlKey.Public())
+		if err != nil {
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting client: %w", err)
+		}
+		if err := <-serverErr; err != nil {
+			c1.Close()
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting server: %w", err)
+		}
+		return c1, c2, func() {
+			c1.Close()
+			c2.Close()
+		}, nil
+	})
+}
+
+// mkConns creates synthetic Noise Conns wrapping the given net.Conns.
+// This function is for testing just the Conn transport logic without
+// having to muck about with Noise handshakes.
+func mkConns(s1, s2 net.Conn) (*Conn, *Conn) {
+	var k1, k2 [chp.KeySize]byte
+	if _, err := rand.Read(k1[:]); err != nil {
+		panic(err)
+	}
+	if _, err := rand.Read(k2[:]); err != nil {
+		panic(err)
+	}
+
+	ret1 := &Conn{
+		conn: s1,
+		tx:   txState{cipher: newCHP(k1)},
+		rx:   rxState{cipher: newCHP(k2)},
+	}
+	ret2 := &Conn{
+		conn: s2,
+		tx:   txState{cipher: newCHP(k2)},
+		rx:   rxState{cipher: newCHP(k1)},
+	}
+
+	return ret1, ret2
+}
+
+type readSink struct {
+	r io.Reader
+
+	cond *sync.Cond
+	sync.Mutex
+	bs  bytes.Buffer
+	err error
+}
+
+func sinkReads(r io.Reader) *readSink {
+	ret := &readSink{
+		r: r,
+	}
+	ret.cond = sync.NewCond(&ret.Mutex)
+	go func() {
+		var buf [4096]byte
+		for {
+			n, err := r.Read(buf[:])
+			ret.Lock()
+			ret.bs.Write(buf[:n])
+			if err != nil {
+				ret.err = err
+			}
+			ret.cond.Broadcast()
+			ret.Unlock()
+			if err != nil {
+				return
+			}
+		}
+	}()
+	return ret
+}
+
+func (s *readSink) String(total int) string {
+	s.Lock()
+	defer s.Unlock()
+	for s.bs.Len() < total && s.err == nil {
+		s.cond.Wait()
+	}
+	if s.err != nil {
+		total = s.bs.Len()
+	}
+	return string(s.bs.Bytes()[:total])
+}
+
+func (s *readSink) Error() error {
+	s.Lock()
+	defer s.Unlock()
+	for s.err == nil {
+		s.cond.Wait()
+	}
+	return s.err
+}
+
+func (s *readSink) Total() int {
+	s.Lock()
+	defer s.Unlock()
+	return s.bs.Len()
+}
+
+func pairWithConns(t *testing.T, clientConn, serverConn net.Conn) (*Conn, *Conn) {
+	var (
+		controlKey = key.NewMachine()
+		machineKey = key.NewMachine()
+		server     *Conn
+		serverErr  = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, controlKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, machineKey, controlKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+	return client, server
+}
+
+func pair(t *testing.T) (*Conn, *Conn) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	return pairWithConns(t, s1, s2)
+}

control/noise/handshake.go

@@ -0,0 +1,443 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	"go4.org/mem"
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+	"tailscale.com/types/key"
+)
+
+const (
+	// protocolName is the name of the specific instantiation of Noise
+	// that the control protocol uses. This string's value is fixed by
+	// the Noise spec, and shouldn't be changed unless we're updating
+	// the control protocol to use a different Noise instance.
+	protocolName = "Noise_IK_25519_ChaChaPoly_BLAKE2s"
+	// protocolVersion is the version of the control protocol that
+	// Client will use when initiating a handshake.
+	protocolVersion uint16 = 1
+	// protocolVersionPrefix is the name portion of the protocol
+	// name+version string that gets mixed into the handshake as a
+	// prologue.
+	//
+	// This mixing verifies that both clients agree that they're
+	// executing the control protocol at a specific version that
+	// matches the advertised version in the cleartext packet header.
+	protocolVersionPrefix = "Tailscale Control Protocol v"
+	invalidNonce          = ^uint64(0)
+)
+
+func protocolVersionPrologue(version uint16) []byte {
+	ret := make([]byte, 0, len(protocolVersionPrefix)+5) // 5 bytes is enough to encode all possible version numbers.
+	ret = append(ret, protocolVersionPrefix...)
+	return strconv.AppendUint(ret, uint64(version), 10)
+}
+
+// Client initiates a control client handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process. Any preexisting Conn deadline is removed.
+func Client(ctx context.Context, conn net.Conn, machineKey key.MachinePrivate, controlKey key.MachinePublic) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	// prologue
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	s.MixHash(controlKey.UntypedBytes())
+
+	// -> e, es, s, ss
+	init := mkInitiationMessage()
+	machineEphemeral := key.NewMachine()
+	machineEphemeralPub := machineEphemeral.Public()
+	copy(init.EphemeralPub(), machineEphemeralPub.UntypedBytes())
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(machineEphemeral, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	machineKeyPub := machineKey.Public()
+	s.EncryptAndHash(cipher, init.MachinePub(), machineKeyPub.UntypedBytes())
+	cipher, err = s.MixDH(machineKey, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	s.EncryptAndHash(cipher, init.Tag(), nil) // empty message payload
+
+	if _, err := conn.Write(init[:]); err != nil {
+		return nil, fmt.Errorf("writing initiation: %w", err)
+	}
+
+	// Read in the payload and look for errors/protocol violations from the server.
+	var resp responseMessage
+	if _, err := io.ReadFull(conn, resp.Header()); err != nil {
+		return nil, fmt.Errorf("reading response header: %w", err)
+	}
+	if resp.Type() != msgTypeResponse {
+		if resp.Type() != msgTypeError {
+			return nil, fmt.Errorf("unexpected response message type %d", resp.Type())
+		}
+		msg := make([]byte, resp.Length())
+		if _, err := io.ReadFull(conn, msg); err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("server error: %q", msg)
+	}
+	if resp.Length() != len(resp.Payload()) {
+		return nil, fmt.Errorf("wrong length %d received for handshake response", resp.Length())
+	}
+	if _, err := io.ReadFull(conn, resp.Payload()); err != nil {
+		return nil, err
+	}
+
+	// <- e, ee, se
+	controlEphemeralPub := key.MachinePublicFromRaw32(mem.B(resp.EphemeralPub()))
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err = s.MixDH(machineEphemeral, controlEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(machineKey, controlEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, resp.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting payload: %w", err)
+	}
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          controlKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c1,
+		},
+		rx: rxState{
+			cipher: c2,
+		},
+	}
+	return c, nil
+}
+
+// Server initiates a control server handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process.
+func Server(ctx context.Context, conn net.Conn, controlKey key.MachinePrivate) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	// Deliberately does not support formatting, so that we don't echo
+	// attacker-controlled input back to them.
+	sendErr := func(msg string) error {
+		if len(msg) >= 1<<16 {
+			msg = msg[:1<<16]
+		}
+		var hdr [headerLen]byte
+		hdr[0] = msgTypeError
+		binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg)))
+		if _, err := conn.Write(hdr[:]); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		if _, err := io.WriteString(conn, msg); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		return fmt.Errorf("refused client handshake: %q", msg)
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	var init initiationMessage
+	if _, err := io.ReadFull(conn, init.Header()); err != nil {
+		return nil, err
+	}
+	if init.Version() != protocolVersion {
+		return nil, sendErr("unsupported protocol version")
+	}
+	if init.Type() != msgTypeInitiation {
+		return nil, sendErr("unexpected handshake message type")
+	}
+	if init.Length() != len(init.Payload()) {
+		return nil, sendErr("wrong handshake initiation length")
+	}
+	if _, err := io.ReadFull(conn, init.Payload()); err != nil {
+		return nil, err
+	}
+
+	// prologue. Can only do this once we at least think the client is
+	// handshaking using a supported version.
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	controlKeyPub := controlKey.Public()
+	s.MixHash(controlKeyPub.UntypedBytes())
+
+	// -> e, es, s, ss
+	machineEphemeralPub := key.MachinePublicFromRaw32(mem.B(init.EphemeralPub()))
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(controlKey, machineEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	var machineKeyBytes [32]byte
+	if err := s.DecryptAndHash(cipher, machineKeyBytes[:], init.MachinePub()); err != nil {
+		return nil, fmt.Errorf("decrypting machine key: %w", err)
+	}
+	machineKey := key.MachinePublicFromRaw32(mem.B(machineKeyBytes[:]))
+	cipher, err = s.MixDH(controlKey, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, init.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting initiation tag: %w", err)
+	}
+
+	// <- e, ee, se
+	resp := mkResponseMessage()
+	controlEphemeral := key.NewMachine()
+	controlEphemeralPub := controlEphemeral.Public()
+	copy(resp.EphemeralPub(), controlEphemeralPub.UntypedBytes())
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err := s.MixDH(controlEphemeral, machineEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(controlEphemeral, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	s.EncryptAndHash(cipher, resp.Tag(), nil) // empty message payload
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	if _, err := conn.Write(resp[:]); err != nil {
+		return nil, err
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          machineKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c2,
+		},
+		rx: rxState{
+			cipher: c1,
+		},
+	}
+	return c, nil
+}
+
+// symmetricState contains the state of an in-flight handshake.
+type symmetricState struct {
+	finished bool
+
+	h  [blake2s.Size]byte // hash of currently-processed handshake state
+	ck [blake2s.Size]byte // chaining key used to construct session keys at the end of the handshake
+}
+
+func (s *symmetricState) checkFinished() {
+	if s.finished {
+		panic("attempted to use symmetricState after Split was called")
+	}
+}
+
+// Initialize sets s to the initial handshake state, prior to
+// processing any handshake messages.
+func (s *symmetricState) Initialize() {
+	s.checkFinished()
+	s.h = blake2s.Sum256([]byte(protocolName))
+	s.ck = s.h
+}
+
+// MixHash updates s.h to be BLAKE2s(s.h || data), where || is
+// concatenation.
+func (s *symmetricState) MixHash(data []byte) {
+	s.checkFinished()
+	h := newBLAKE2s()
+	h.Write(s.h[:])
+	h.Write(data)
+	h.Sum(s.h[:0])
+}
+
+// MixDH updates s.ck with the result of X25519(priv, pub) and returns
+// a singleUseCHP that can be used to encrypt or decrypt handshake
+// data.
+//
+// MixDH corresponds to MixKey(X25519(...))) in the spec. Implementing
+// it as a single function allows for strongly-typed arguments that
+// reduce the risk of error in the caller (e.g. invoking X25519 with
+// two private keys, or two public keys), and thus producing the wrong
+// calculation.
+func (s *symmetricState) MixDH(priv key.MachinePrivate, pub key.MachinePublic) (*singleUseCHP, error) {
+	s.checkFinished()
+	keyData, err := curve25519.X25519(priv.UntypedBytes(), pub.UntypedBytes())
+	if err != nil {
+		return nil, fmt.Errorf("computing X25519: %w", err)
+	}
+
+	r := hkdf.New(newBLAKE2s, keyData, s.ck[:], nil)
+	if _, err := io.ReadFull(r, s.ck[:]); err != nil {
+		return nil, fmt.Errorf("extracting ck: %w", err)
+	}
+	var k [chp.KeySize]byte
+	if _, err := io.ReadFull(r, k[:]); err != nil {
+		return nil, fmt.Errorf("extracting k: %w", err)
+	}
+	return newSingleUseCHP(k), nil
+}
+
+// EncryptAndHash encrypts plaintext into ciphertext (which must be
+// the correct size to hold the encrypted plaintext) using cipher,
+// mixes the ciphertext into s.h, and returns the ciphertext.
+func (s *symmetricState) EncryptAndHash(cipher *singleUseCHP, ciphertext, plaintext []byte) {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		panic("ciphertext is wrong size for given plaintext")
+	}
+	ret := cipher.Seal(ciphertext[:0], plaintext, s.h[:])
+	s.MixHash(ret)
+}
+
+// DecryptAndHash decrypts the given ciphertext into plaintext (which
+// must be the correct size to hold the decrypted ciphertext) using
+// cipher. If decryption is successful, it mixes the ciphertext into
+// s.h.
+func (s *symmetricState) DecryptAndHash(cipher *singleUseCHP, plaintext, ciphertext []byte) error {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		return errors.New("plaintext is wrong size for given ciphertext")
+	}
+	if _, err := cipher.Open(plaintext[:0], ciphertext, s.h[:]); err != nil {
+		return err
+	}
+	s.MixHash(ciphertext)
+	return nil
+}
+
+// Split returns two ChaCha20Poly1305 ciphers with keys derived from
+// the current handshake state. Methods on s cannot be used again
+// after calling Split.
+func (s *symmetricState) Split() (c1, c2 cipher.AEAD, err error) {
+	s.finished = true
+
+	var k1, k2 [chp.KeySize]byte
+	r := hkdf.New(newBLAKE2s, nil, s.ck[:], nil)
+	if _, err := io.ReadFull(r, k1[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k1: %w", err)
+	}
+	if _, err := io.ReadFull(r, k2[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k2: %w", err)
+	}
+	c1, err = chp.New(k1[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c1: %w", err)
+	}
+	c2, err = chp.New(k2[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c2: %w", err)
+	}
+	return c1, c2, nil
+}
+
+// newBLAKE2s returns a hash.Hash implementing BLAKE2s, or panics on
+// error.
+func newBLAKE2s() hash.Hash {
+	h, err := blake2s.New256(nil)
+	if err != nil {
+		// Should never happen, errors only happen when using BLAKE2s
+		// in MAC mode with a key.
+		panic(err)
+	}
+	return h
+}
+
+// newCHP returns a cipher.AEAD implementing ChaCha20Poly1305, or
+// panics on error.
+func newCHP(key [chp.KeySize]byte) cipher.AEAD {
+	aead, err := chp.New(key[:])
+	if err != nil {
+		// Can only happen if we passed a key of the wrong length. The
+		// function signature prevents that.
+		panic(err)
+	}
+	return aead
+}
+
+// singleUseCHP is an instance of ChaCha20Poly1305 that can be used
+// only once, either for encrypting or decrypting, but not both. The
+// chosen operation is always executed with an all-zeros
+// nonce. Subsequent calls to either Seal or Open panic.
+type singleUseCHP struct {
+	c cipher.AEAD
+}
+
+func newSingleUseCHP(key [chp.KeySize]byte) *singleUseCHP {
+	return &singleUseCHP{newCHP(key)}
+}
+
+func (c *singleUseCHP) Seal(dst, plaintext, additionalData []byte) []byte {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Seal(dst, nonce[:], plaintext, additionalData)
+}
+
+func (c *singleUseCHP) Open(dst, ciphertext, additionalData []byte) ([]byte, error) {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Open(dst, nonce[:], ciphertext, additionalData)
+}

control/noise/handshake_test.go

@@ -0,0 +1,296 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestHandshake(t *testing.T) {
+	var (
+		clientConn, serverConn = tsnettest.NewConn("noise", 128000)
+		serverKey              = key.NewMachine()
+		clientKey              = key.NewMachine()
+		server                 *Conn
+		serverErr              = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, serverKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+
+	if client.HandshakeHash() != server.HandshakeHash() {
+		t.Fatal("client and server disagree on handshake hash")
+	}
+
+	if client.ProtocolVersion() != int(protocolVersion) {
+		t.Fatalf("client reporting wrong protocol version %d, want %d", client.ProtocolVersion(), protocolVersion)
+	}
+	if client.ProtocolVersion() != server.ProtocolVersion() {
+		t.Fatalf("peers disagree on protocol version, client=%d server=%d", client.ProtocolVersion(), server.ProtocolVersion())
+	}
+	if client.Peer() != serverKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+	if server.Peer() != clientKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+}
+
+// Check that handshaking repeatedly with the same long-term keys
+// result in different handshake hashes and wire traffic.
+func TestNoReuse(t *testing.T) {
+	var (
+		hashes           = map[[32]byte]bool{}
+		clientHandshakes = map[[96]byte]bool{}
+		serverHandshakes = map[[48]byte]bool{}
+		packets          = map[[32]byte]bool{}
+	)
+	for i := 0; i < 10; i++ {
+		var (
+			clientRaw, serverRaw = tsnettest.NewConn("noise", 128000)
+			clientBuf, serverBuf bytes.Buffer
+			clientConn           = &readerConn{clientRaw, io.TeeReader(clientRaw, &clientBuf)}
+			serverConn           = &readerConn{serverRaw, io.TeeReader(serverRaw, &serverBuf)}
+			serverKey            = key.NewMachine()
+			clientKey            = key.NewMachine()
+			server               *Conn
+			serverErr            = make(chan error, 1)
+		)
+		go func() {
+			var err error
+			server, err = Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client connection failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed: %v", err)
+		}
+
+		var clientHS [96]byte
+		copy(clientHS[:], serverBuf.Bytes())
+		if clientHandshakes[clientHS] {
+			t.Fatal("client handshake seen twice")
+		}
+		clientHandshakes[clientHS] = true
+
+		var serverHS [48]byte
+		copy(serverHS[:], clientBuf.Bytes())
+		if serverHandshakes[serverHS] {
+			t.Fatal("server handshake seen twice")
+		}
+		serverHandshakes[serverHS] = true
+
+		clientBuf.Reset()
+		serverBuf.Reset()
+		cb := sinkReads(client)
+		sb := sinkReads(server)
+
+		if hashes[client.HandshakeHash()] {
+			t.Fatalf("handshake hash %v seen twice", client.HandshakeHash())
+		}
+		hashes[client.HandshakeHash()] = true
+
+		// Sending 14 bytes turns into 32 bytes on the wire (+16 for
+		// the chacha20poly1305 overhead, +2 length header)
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if _, err := io.WriteString(server, strings.Repeat("b", 14)); err != nil {
+			t.Fatalf("server>client write failed: %v", err)
+		}
+
+		// Wait for the bytes to be read, so we know they've traveled end to end
+		cb.String(14)
+		sb.String(14)
+
+		var clientWire, serverWire [32]byte
+		copy(clientWire[:], clientBuf.Bytes())
+		copy(serverWire[:], serverBuf.Bytes())
+
+		if packets[clientWire] {
+			t.Fatalf("client wire traffic seen twice")
+		}
+		packets[clientWire] = true
+		if packets[serverWire] {
+			t.Fatalf("server wire traffic seen twice")
+		}
+		packets[serverWire] = true
+	}
+}
+
+// tamperReader wraps a reader and mutates the Nth byte.
+type tamperReader struct {
+	r     io.Reader
+	n     int
+	total int
+}
+
+func (r *tamperReader) Read(bs []byte) (int, error) {
+	n, err := r.r.Read(bs)
+	if off := r.n - r.total; off >= 0 && off < n {
+		bs[off] += 1
+	}
+	r.total += n
+	return n, err
+}
+
+func TestTampering(t *testing.T) {
+	// Tamper with every byte of the client initiation message.
+	for i := 0; i < 101; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			// If the server failed, we have to close the Conn to
+			// unblock the client.
+			if err != nil {
+				serverConn.Close()
+			}
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatalf("server connection succeeded despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the server response message.
+	for i := 0; i < 51; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed despite no tampering: %v", err)
+		}
+	}
+
+	// Tamper with every byte of the first server>client transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, 51 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			_, err = io.WriteString(server, strings.Repeat("a", 14))
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		// The client needs a timeout if the tampering is hitting the length header.
+		if i == 1 || i == 2 {
+			client.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+		}
+
+		var bs [100]byte
+		n, err := client.Read(bs[:])
+		if err == nil {
+			t.Fatal("read succeeded despite tampering")
+		}
+		if n != 0 {
+			t.Fatal("conn yielded some bytes despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the first client>server transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, 101 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			var bs [100]byte
+			// The server needs a timeout if the tampering is hitting the length header.
+			if i == 1 || i == 2 {
+				server.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+			}
+			n, err := server.Read(bs[:])
+			if n != 0 {
+				panic("server got bytes despite tampering")
+			} else {
+				serverErr <- err
+			}
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatal("server successfully received bytes despite tampering")
+		}
+	}
+}

control/noise/interop_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"net"
+	"testing"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+// Can a reference Noise IK client talk to our server?
+func TestInteropClient(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		serverErr   = make(chan error, 2)
+		serverBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		server, err := Server(context.Background(), s2, controlKey)
+		serverErr <- err
+		if err != nil {
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(server, buf[:len(c2s)])
+		serverBytes <- buf[:len(c2s)]
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		_, err = server.Write([]byte(s2c))
+		serverErr <- err
+	}()
+
+	gotS2C, err := noiseExplorerClient(s1, controlKey.Public(), machineKey, []byte(c2s))
+	if err != nil {
+		t.Fatalf("failed client interop: %v", err)
+	}
+	if string(gotS2C) != s2c {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotS2C), s2c)
+	}
+
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server handshake failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server read/write failed: %v", err)
+	}
+	if got := string(<-serverBytes); got != c2s {
+		t.Fatalf("server received %q, want %q", got, c2s)
+	}
+}
+
+// Can our client talk to a reference Noise IK server?
+func TestInteropServer(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		clientErr   = make(chan error, 2)
+		clientBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		client, err := Client(context.Background(), s1, machineKey, controlKey.Public())
+		clientErr <- err
+		if err != nil {
+			return
+		}
+		_, err = client.Write([]byte(c2s))
+		if err != nil {
+			clientErr <- err
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(client, buf[:len(s2c)])
+		clientBytes <- buf[:len(s2c)]
+		clientErr <- err
+	}()
+
+	gotC2S, err := noiseExplorerServer(s2, controlKey, machineKey.Public(), []byte(s2c))
+	if err != nil {
+		t.Fatalf("failed server interop: %v", err)
+	}
+	if string(gotC2S) != c2s {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotC2S), c2s)
+	}
+
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client handshake failed: %v", err)
+	}
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client read/write failed: %v", err)
+	}
+	if got := string(<-clientBytes); got != s2c {
+		t.Fatalf("client received %q, want %q", got, s2c)
+	}
+}
+
+// noiseExplorerClient uses the Noise Explorer implementation of Noise
+// IK to handshake as a Noise client on conn, transmit payload, and
+// read+return a payload from the peer.
+func noiseExplorerClient(conn net.Conn, controlKey key.MachinePublic, machineKey key.MachinePrivate, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], machineKey.UntypedBytes())
+	copy(mk.public_key[:], machineKey.Public().UntypedBytes())
+	var peerKey [32]byte
+	copy(peerKey[:], controlKey.UntypedBytes())
+	session := InitSession(true, protocolVersionPrologue(protocolVersion), mk, peerKey)
+
+	_, msg1 := SendMessage(&session, nil)
+	var hdr [initiationHeaderLen]byte
+	binary.BigEndian.PutUint16(hdr[:2], protocolVersion)
+	hdr[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(hdr[3:5], 96)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ns); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ciphertext); err != nil {
+		return nil, err
+	}
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:51]); err != nil {
+		return nil, err
+	}
+	// ignore the header for this test, we're only checking the noise
+	// implementation.
+	msg2 := messagebuffer{
+		ciphertext: buf[35:51],
+	}
+	copy(msg2.ne[:], buf[3:35])
+	_, p, valid := RecvMessage(&session, &msg2)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg3 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg3.ciphertext)))
+	if _, err := conn.Write(hdr[:3]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg3.ciphertext); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	// Ignore all of the header except the payload length
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg4 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg4)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	return p, nil
+}
+
+func noiseExplorerServer(conn net.Conn, controlKey key.MachinePrivate, wantMachineKey key.MachinePublic, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], controlKey.UntypedBytes())
+	copy(mk.public_key[:], controlKey.Public().UntypedBytes())
+	session := InitSession(false, protocolVersionPrologue(protocolVersion), mk, [32]byte{})
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:101]); err != nil {
+		return nil, err
+	}
+	// Ignore the header, we're just checking the noise implementation.
+	msg1 := messagebuffer{
+		ns:         buf[37:85],
+		ciphertext: buf[85:101],
+	}
+	copy(msg1.ne[:], buf[5:37])
+	_, p, valid := RecvMessage(&session, &msg1)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg2 := SendMessage(&session, nil)
+	var hdr [headerLen]byte
+	hdr[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(hdr[1:3], 48)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ciphertext[:]); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg3 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg3)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	_, msg4 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg4.ciphertext)))
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg4.ciphertext); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}

control/noise/messages.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import "encoding/binary"
+
+const (
+	// msgTypeInitiation frames carry a Noise IK handshake initiation message.
+	msgTypeInitiation = 1
+	// msgTypeResponse frames carry a Noise IK handshake response message.
+	msgTypeResponse = 2
+	// msgTypeError frames carry an unauthenticated human-readable
+	// error message.
+	//
+	// Errors reported in this message type must be treated as public
+	// hints only. They are not encrypted or authenticated, and so can
+	// be seen and tampered with on the wire.
+	msgTypeError = 3
+	// msgTypeRecord frames carry session data bytes.
+	msgTypeRecord = 4
+
+	// headerLen is the size of the header on all messages except msgTypeInitiation.
+	headerLen = 3
+	// initiationHeaderLen is the size of the header on all msgTypeInitiation messages.
+	initiationHeaderLen = 5
+)
+
+// initiationMessage is the protocol message sent from a client
+// machine to a control server.
+//
+// 2b: protocol version
+// 1b: message type (0x01)
+// 2b: payload length (96)
+// 5b: header (see headerLen for fields)
+// 32b: client ephemeral public key (cleartext)
+// 48b: client machine public key (encrypted)
+// 16b: message tag (authenticates the whole message)
+type initiationMessage [101]byte
+
+func mkInitiationMessage() initiationMessage {
+	var ret initiationMessage
+	binary.BigEndian.PutUint16(ret[:2], uint16(protocolVersion))
+	ret[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(ret[3:5], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *initiationMessage) Header() []byte  { return m[:initiationHeaderLen] }
+func (m *initiationMessage) Payload() []byte { return m[initiationHeaderLen:] }
+
+func (m *initiationMessage) Version() uint16 { return binary.BigEndian.Uint16(m[:2]) }
+func (m *initiationMessage) Type() byte      { return m[2] }
+func (m *initiationMessage) Length() int     { return int(binary.BigEndian.Uint16(m[3:5])) }
+
+func (m *initiationMessage) EphemeralPub() []byte {
+	return m[initiationHeaderLen : initiationHeaderLen+32]
+}
+func (m *initiationMessage) MachinePub() []byte {
+	return m[initiationHeaderLen+32 : initiationHeaderLen+32+48]
+}
+func (m *initiationMessage) Tag() []byte { return m[initiationHeaderLen+32+48:] }
+
+// responseMessage is the protocol message sent from a control server
+// to a client machine.
+//
+// 1b: message type (0x02)
+// 2b: payload length (48)
+// 32b: control ephemeral public key (cleartext)
+// 16b: message tag (authenticates the whole message)
+type responseMessage [51]byte
+
+func mkResponseMessage() responseMessage {
+	var ret responseMessage
+	ret[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(ret[1:], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *responseMessage) Header() []byte  { return m[:headerLen] }
+func (m *responseMessage) Payload() []byte { return m[headerLen:] }
+
+func (m *responseMessage) Type() byte  { return m[0] }
+func (m *responseMessage) Length() int { return int(binary.BigEndian.Uint16(m[1:3])) }
+
+func (m *responseMessage) EphemeralPub() []byte { return m[headerLen : headerLen+32] }
+func (m *responseMessage) Tag() []byte          { return m[headerLen+32:] }

control/noise/noiseexplorer_test.go

@@ -0,0 +1,475 @@
+// This file contains the implementation of Noise IK from
+// https://noiseexplorer.com/ . Unlike the rest of this repository,
+// this file is licensed under the terms of the GNU GPL v3. See
+// https://source.symbolic.software/noiseexplorer/noiseexplorer for
+// more information.
+//
+// This file is used here to verify that Tailscale's implementation of
+// Noise IK is interoperable with another implementation.
+//lint:file-ignore SA4006 not our code.
+
+/*
+IK:
+  <- s
+  ...
+  -> e, es, s, ss
+  <- e, ee, se
+  ->
+  <-
+*/
+
+// Implementation Version: 1.0.2
+
+/* ---------------------------------------------------------------- *
+ * PARAMETERS                                                       *
+ * ---------------------------------------------------------------- */
+
+package noise
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/binary"
+	"hash"
+	"io"
+	"math"
+
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+)
+
+/* ---------------------------------------------------------------- *
+ * TYPES                                                            *
+ * ---------------------------------------------------------------- */
+
+type keypair struct {
+	public_key  [32]byte
+	private_key [32]byte
+}
+
+type messagebuffer struct {
+	ne         [32]byte
+	ns         []byte
+	ciphertext []byte
+}
+
+type cipherstate struct {
+	k [32]byte
+	n uint32
+}
+
+type symmetricstate struct {
+	cs cipherstate
+	ck [32]byte
+	h  [32]byte
+}
+
+type handshakestate struct {
+	ss  symmetricstate
+	s   keypair
+	e   keypair
+	rs  [32]byte
+	re  [32]byte
+	psk [32]byte
+}
+
+type noisesession struct {
+	hs  handshakestate
+	h   [32]byte
+	cs1 cipherstate
+	cs2 cipherstate
+	mc  uint64
+	i   bool
+}
+
+/* ---------------------------------------------------------------- *
+ * CONSTANTS                                                        *
+ * ---------------------------------------------------------------- */
+
+var emptyKey = [32]byte{
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+}
+
+var minNonce = uint32(0)
+
+/* ---------------------------------------------------------------- *
+ * UTILITY FUNCTIONS                                                *
+ * ---------------------------------------------------------------- */
+
+func getPublicKey(kp *keypair) [32]byte {
+	return kp.public_key
+}
+
+func isEmptyKey(k [32]byte) bool {
+	return subtle.ConstantTimeCompare(k[:], emptyKey[:]) == 1
+}
+
+func validatePublicKey(k []byte) bool {
+	forbiddenCurveValues := [12][]byte{
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{224, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 0},
+		{95, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 87},
+		{236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{237, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{205, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 128},
+		{76, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 215},
+		{217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{218, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{219, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25},
+	}
+
+	for _, testValue := range forbiddenCurveValues {
+		if subtle.ConstantTimeCompare(k[:], testValue[:]) == 1 {
+			panic("Invalid public key")
+		}
+	}
+	return true
+}
+
+/* ---------------------------------------------------------------- *
+ * PRIMITIVES                                                       *
+ * ---------------------------------------------------------------- */
+
+func incrementNonce(n uint32) uint32 {
+	return n + 1
+}
+
+func dh(private_key [32]byte, public_key [32]byte) [32]byte {
+	var ss [32]byte
+	curve25519.ScalarMult(&ss, &private_key, &public_key)
+	return ss
+}
+
+func generateKeypair() keypair {
+	var public_key [32]byte
+	var private_key [32]byte
+	_, _ = rand.Read(private_key[:])
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	if validatePublicKey(public_key[:]) {
+		return keypair{public_key, private_key}
+	}
+	return generateKeypair()
+}
+
+func generatePublicKey(private_key [32]byte) [32]byte {
+	var public_key [32]byte
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	return public_key
+}
+
+func encrypt(k [32]byte, n uint32, ad []byte, plaintext []byte) []byte {
+	var nonce [12]byte
+	var ciphertext []byte
+	enc, _ := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	ciphertext = enc.Seal(nil, nonce[:], plaintext, ad)
+	return ciphertext
+}
+
+func decrypt(k [32]byte, n uint32, ad []byte, ciphertext []byte) (bool, []byte, []byte) {
+	var nonce [12]byte
+	var plaintext []byte
+	enc, err := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	plaintext, err = enc.Open(nil, nonce[:], ciphertext, ad)
+	return (err == nil), ad, plaintext
+}
+
+func getHash(a []byte, b []byte) [32]byte {
+	return blake2s.Sum256(append(a, b...))
+}
+
+func hashProtocolName(protocolName []byte) [32]byte {
+	var h [32]byte
+	if len(protocolName) <= 32 {
+		copy(h[:], protocolName)
+	} else {
+		h = getHash(protocolName, []byte{})
+	}
+	return h
+}
+
+func blake2HkdfInterface() hash.Hash {
+	h, _ := blake2s.New256([]byte{})
+	return h
+}
+
+func getHkdf(ck [32]byte, ikm []byte) ([32]byte, [32]byte, [32]byte) {
+	var k1 [32]byte
+	var k2 [32]byte
+	var k3 [32]byte
+	output := hkdf.New(blake2HkdfInterface, ikm[:], ck[:], []byte{})
+	io.ReadFull(output, k1[:])
+	io.ReadFull(output, k2[:])
+	io.ReadFull(output, k3[:])
+	return k1, k2, k3
+}
+
+/* ---------------------------------------------------------------- *
+ * STATE MANAGEMENT                                                 *
+ * ---------------------------------------------------------------- */
+
+/* CipherState */
+func initializeKey(k [32]byte) cipherstate {
+	return cipherstate{k, minNonce}
+}
+
+func hasKey(cs *cipherstate) bool {
+	return !isEmptyKey(cs.k)
+}
+
+func setNonce(cs *cipherstate, newNonce uint32) *cipherstate {
+	cs.n = newNonce
+	return cs
+}
+
+func encryptWithAd(cs *cipherstate, ad []byte, plaintext []byte) (*cipherstate, []byte) {
+	e := encrypt(cs.k, cs.n, ad, plaintext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, e
+}
+
+func decryptWithAd(cs *cipherstate, ad []byte, ciphertext []byte) (*cipherstate, []byte, bool) {
+	valid, ad, plaintext := decrypt(cs.k, cs.n, ad, ciphertext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, plaintext, valid
+}
+
+func reKey(cs *cipherstate) *cipherstate {
+	e := encrypt(cs.k, math.MaxUint32, []byte{}, emptyKey[:])
+	copy(cs.k[:], e)
+	return cs
+}
+
+/* SymmetricState */
+
+func initializeSymmetric(protocolName []byte) symmetricstate {
+	h := hashProtocolName(protocolName)
+	ck := h
+	cs := initializeKey(emptyKey)
+	return symmetricstate{cs, ck, h}
+}
+
+func mixKey(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	ck, tempK, _ := getHkdf(ss.ck, ikm[:])
+	ss.cs = initializeKey(tempK)
+	ss.ck = ck
+	return ss
+}
+
+func mixHash(ss *symmetricstate, data []byte) *symmetricstate {
+	ss.h = getHash(ss.h[:], data)
+	return ss
+}
+
+func mixKeyAndHash(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	var tempH [32]byte
+	var tempK [32]byte
+	ss.ck, tempH, tempK = getHkdf(ss.ck, ikm[:])
+	ss = mixHash(ss, tempH[:])
+	ss.cs = initializeKey(tempK)
+	return ss
+}
+
+func getHandshakeHash(ss *symmetricstate) [32]byte {
+	return ss.h
+}
+
+func encryptAndHash(ss *symmetricstate, plaintext []byte) (*symmetricstate, []byte) {
+	var ciphertext []byte
+	if hasKey(&ss.cs) {
+		_, ciphertext = encryptWithAd(&ss.cs, ss.h[:], plaintext)
+	} else {
+		ciphertext = plaintext
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, ciphertext
+}
+
+func decryptAndHash(ss *symmetricstate, ciphertext []byte) (*symmetricstate, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if hasKey(&ss.cs) {
+		_, plaintext, valid = decryptWithAd(&ss.cs, ss.h[:], ciphertext)
+	} else {
+		plaintext, valid = ciphertext, true
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, plaintext, valid
+}
+
+func split(ss *symmetricstate) (cipherstate, cipherstate) {
+	tempK1, tempK2, _ := getHkdf(ss.ck, []byte{})
+	cs1 := initializeKey(tempK1)
+	cs2 := initializeKey(tempK2)
+	return cs1, cs2
+}
+
+/* HandshakeState */
+
+func initializeInitiator(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, rs[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func initializeResponder(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, s.public_key[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func writeMessageA(hs *handshakestate, payload []byte) (*handshakestate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	spk := make([]byte, len(hs.s.public_key))
+	copy(spk[:], hs.s.public_key[:])
+	_, ns = encryptAndHash(&hs.ss, spk)
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return hs, messageBuffer
+}
+
+func writeMessageB(hs *handshakestate, payload []byte) ([32]byte, messagebuffer, cipherstate, cipherstate) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, messageBuffer, cs1, cs2
+}
+
+func writeMessageRegular(cs *cipherstate, payload []byte) (*cipherstate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	cs, ciphertext = encryptWithAd(cs, []byte{}, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return cs, messageBuffer
+}
+
+func readMessageA(hs *handshakestate, message *messagebuffer) (*handshakestate, []byte, bool) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, ns, valid1 := decryptAndHash(&hs.ss, message.ns)
+	if valid1 && len(ns) == 32 && validatePublicKey(message.ns[:]) {
+		copy(hs.rs[:], ns)
+	}
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	return hs, plaintext, (valid1 && valid2)
+}
+
+func readMessageB(hs *handshakestate, message *messagebuffer) ([32]byte, []byte, bool, cipherstate, cipherstate) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, plaintext, (valid1 && valid2), cs1, cs2
+}
+
+func readMessageRegular(cs *cipherstate, message *messagebuffer) (*cipherstate, []byte, bool) {
+	/* No encrypted keys */
+	_, plaintext, valid2 := decryptWithAd(cs, []byte{}, message.ciphertext)
+	return cs, plaintext, valid2
+}
+
+/* ---------------------------------------------------------------- *
+ * PROCESSES                                                        *
+ * ---------------------------------------------------------------- */
+
+func InitSession(initiator bool, prologue []byte, s keypair, rs [32]byte) noisesession {
+	var session noisesession
+	psk := emptyKey
+	if initiator {
+		session.hs = initializeInitiator(prologue, s, rs, psk)
+	} else {
+		session.hs = initializeResponder(prologue, s, rs, psk)
+	}
+	session.i = initiator
+	session.mc = 0
+	return session
+}
+
+func SendMessage(session *noisesession, message []byte) (*noisesession, messagebuffer) {
+	var messageBuffer messagebuffer
+	if session.mc == 0 {
+		_, messageBuffer = writeMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, messageBuffer, session.cs1, session.cs2 = writeMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, messageBuffer = writeMessageRegular(&session.cs1, message)
+		} else {
+			_, messageBuffer = writeMessageRegular(&session.cs2, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, messageBuffer
+}
+
+func RecvMessage(session *noisesession, message *messagebuffer) (*noisesession, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if session.mc == 0 {
+		_, plaintext, valid = readMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, plaintext, valid, session.cs1, session.cs2 = readMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, plaintext, valid = readMessageRegular(&session.cs2, message)
+		} else {
+			_, plaintext, valid = readMessageRegular(&session.cs1, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, plaintext, valid
+}
+
+func main() {}

derp/derp_client.go

@@ -210,12 +210,12 @@ func (c *Client) send(dstKey key.NodePublic, pkt []byte) (ret error) {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
 	if c.rate != nil {
-		pktLen := frameHeaderLen + dstKey.RawLen() + len(pkt)
+		pktLen := frameHeaderLen + key.NodePublicRawLen + len(pkt)
 		if !c.rate.AllowN(time.Now(), pktLen) {
 			return nil // drop
 		}
 	}
-	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(dstKey.RawLen()+len(pkt))); err != nil {
+	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(key.NodePublicRawLen+len(pkt))); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {

derp/derp_server.go

@@ -1017,7 +1017,7 @@ func (s *Server) verifyClient(clientKey key.NodePublic, info *clientInfo) error
 }
 
 func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
-	buf := make([]byte, 0, len(magic)+s.publicKey.RawLen())
+	buf := make([]byte, 0, len(magic)+key.NodePublicRawLen)
 	buf = append(buf, magic...)
 	buf = s.publicKey.AppendTo(buf)
 	err := writeFrame(lw.bw(), frameServerKey, buf)
@@ -1469,7 +1469,7 @@ func (c *sclient) sendPacket(srcKey key.NodePublic, contents []byte) (err error)
 	withKey := !srcKey.IsZero()
 	pktLen := len(contents)
 	if withKey {
-		pktLen += srcKey.RawLen()
+		pktLen += key.NodePublicRawLen
 	}
 	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err

derp/derphttp/derphttp_client.go

@@ -429,7 +429,7 @@ func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
 	}
 	hostOrIP := host
-	dialer := netns.NewDialer()
+	dialer := netns.NewDialer(c.logf)
 
 	if c.DNSCache != nil {
 		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
@@ -519,7 +519,7 @@ func (c *Client) DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (tl
 }
 
 func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) {
-	return netns.NewDialer().DialContext(ctx, proto, addr)
+	return netns.NewDialer(c.logf).DialContext(ctx, proto, addr)
 }
 
 // shouldDialProto reports whether an explicitly provided IPv4 or IPv6

disco/disco.go

@@ -25,8 +25,9 @@ import (
 	"fmt"
 	"net"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 // Magic is the 6 byte header of all discovery messages.
@@ -115,19 +116,19 @@ type Ping struct {
 	// It shouldn't be trusted by itself, but can be combined with
 	// netmap data to reduce the discokey:nodekey relation from 1:N to
 	// 1:1.
-	NodeKey tailcfg.NodeKey
+	NodeKey key.NodePublic
 }
 
 func (m *Ping) AppendMarshal(b []byte) []byte {
 	dataLen := 12
 	hasKey := !m.NodeKey.IsZero()
 	if hasKey {
-		dataLen += len(m.NodeKey)
+		dataLen += key.NodePublicRawLen
 	}
 	ret, d := appendMsgHeader(b, TypePing, v0, dataLen)
 	n := copy(d, m.TxID[:])
 	if hasKey {
-		copy(d[n:], m.NodeKey[:])
+		m.NodeKey.AppendTo(d[:n])
 	}
 	return ret
 }
@@ -138,8 +139,10 @@ func parsePing(ver uint8, p []byte) (m *Ping, err error) {
 	}
 	m = new(Ping)
 	p = p[copy(m.TxID[:], p):]
-	if len(p) >= len(m.NodeKey) {
-		copy(m.NodeKey[:], p)
+	// Deliberately lax on longer-than-expected messages, for future
+	// compatibility.
+	if len(p) >= key.NodePublicRawLen {
+		m.NodeKey = key.NodePublicFromRaw32(mem.B(p[:key.NodePublicRawLen]))
 	}
 	return m, nil
 }

disco/disco_test.go

@@ -10,8 +10,9 @@ import (
 	"strings"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 func TestMarshalAndParse(t *testing.T) {
@@ -30,13 +31,8 @@ func TestMarshalAndParse(t *testing.T) {
 		{
 			name: "ping_with_nodekey_src",
 			m: &Ping{
-				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
-				NodeKey: tailcfg.NodeKey{
-					1:  1,
-					2:  2,
-					30: 30,
-					31: 31,
-				},
+				TxID:    [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				NodeKey: key.NodePublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})),
 			},
 			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f",
 		},

docs/k8s/README.md

@@ -32,7 +32,7 @@ There are quite a few ways of running Tailscale inside a Kubernetes Cluster, som
    ```
 
 ### Sample Sidecar
-Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
+Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivity between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
 
 1. Create and login to the sample nginx pod with a Tailscale sidecar
 
@@ -144,4 +144,4 @@ routes for the subnet-router are enabled.
    # INTERNAL_IP="$(kubectl get po <POD_NAME> -o=jsonpath='{.status.podIP}')" 
    INTERNAL_PORT=8080 
    curl http://$INTERNAL_IP:$INTERNAL_PORT
-   ```
+   ```

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	filippo.io/mkcert v1.4.3
+	github.com/akutz/memconn v0.1.0
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
@@ -13,9 +14,8 @@ require (
 	github.com/coreos/go-iptables v0.6.0
 	github.com/creack/pty v1.1.17
 	github.com/dave/jennifer v1.4.1
-	github.com/frankban/quicktest v1.13.1
+	github.com/frankban/quicktest v1.14.0
 	github.com/gliderlabs/ssh v0.3.3
-	github.com/go-multierror/multierror v1.0.2
 	github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1
 	github.com/godbus/dbus/v5 v5.0.5
 	github.com/google/go-cmp v0.5.6
@@ -39,23 +39,24 @@ require (
 	github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
 	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20211020060615-d418f374d309
+	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
+	golang.org/x/net v0.0.0-20211111083644-e5c967477495
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211020174200-9d6173849985
+	golang.org/x/sys v0.0.0-20211110154304-99a53858aa08
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.7
-	golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2
+	golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45
 	golang.zx2c4.com/wireguard/windows v0.4.10
 	honnef.co/go/tools v0.2.1
 	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
-	inet.af/netstack v0.0.0-20211027215559-ec21145de76b
+	inet.af/netstack v0.0.0-20211101182044-1c1bcf452982
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	nhooyr.io/websocket v1.8.7
@@ -192,13 +193,14 @@ require (
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.4 // indirect
 	github.com/uudashr/gocognit v1.0.1 // indirect
-	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
+	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -39,6 +39,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A=
+github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
@@ -127,14 +129,15 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8=
-github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
+github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
@@ -156,16 +159,16 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-multierror/multierror v1.0.2 h1:AwsKbEXkmf49ajdFJgcFXqSG0aLo0HEyAE9zk9JguJo=
-github.com/go-multierror/multierror v1.0.2/go.mod h1:U7SZR/D9jHgt2nkSj8XcbCWdmVM2igraCHQ3HC1HiKY=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1 h1:4dntyT+x6QTOSCIrgczbQ+ockAEha0cfxD5Wi0iCzjY=
 github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -194,8 +197,10 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.5 h1:9Eg0XUhQxtkV8ykTMKtMMYY72g4NgxtRq4jgh4Ih5YM=
@@ -484,6 +489,8 @@ github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -643,6 +650,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -666,6 +675,7 @@ github.com/u-root/uio v0.0.0-20210528114334-82958018845c h1:BFvcl34IGnw8yvJi8hlq
 github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
@@ -681,10 +691,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
 github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6 h1:167a2omrzz+nN9Of6lN/0yOB9itzw+IOioRThNZ30jA=
+github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
@@ -729,8 +739,9 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -798,8 +809,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211111083644-e5c967477495 h1:cjxxlQm6d4kYbhpZ2ghvmI8xnq0AG+jXmzrhzfkyu5A=
+golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -836,7 +848,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -847,11 +858,13 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -877,8 +890,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
-golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w=
@@ -958,9 +972,15 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 h1:Ug9qvr1myri/zFN6xL17LSCBGFDnphBBhzmILHsM5TY=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20210905140043-2ef39d47540c/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
-golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2 h1:mHJssZsxXvTmTP+sxkGZCItVGhaOWo0UnFqrM2lMqOk=
-golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2/go.mod h1:RTjaYEQboNk7+2qfPGBotaMEh/5HIvmPZ6DIe10lTqI=
+golang.zx2c4.com/wireguard v0.0.0-20211115224047-111e0566dce3 h1:7BFThRTwBwTLoMomQ/Y0GqY1VLH9D7kbbTNsfxl2fU0=
+golang.zx2c4.com/wireguard v0.0.0-20211115224047-111e0566dce3/go.mod h1:evxZIqfCetExY5piKXGAxJYwvXWkps9zTCkWpkoGFxw=
+golang.zx2c4.com/wireguard v0.0.0-20211116194326-3cae233d69f7 h1:ZeHUKruJlkbSvafSH7GrDzMDXf7+/0T5sEKE8A9rEiE=
+golang.zx2c4.com/wireguard v0.0.0-20211116194326-3cae233d69f7/go.mod h1:evxZIqfCetExY5piKXGAxJYwvXWkps9zTCkWpkoGFxw=
+golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45 h1:mEVhdMPTuebD9IUXOUB5Q2sjZpcmzkahHWd6DrGpLHA=
+golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45/go.mod h1:evxZIqfCetExY5piKXGAxJYwvXWkps9zTCkWpkoGFxw=
 golang.zx2c4.com/wireguard/windows v0.4.10 h1:HmjzJnb+G4NCdX+sfjsQlsxGPuYaThxRbZUZFLyR0/s=
 golang.zx2c4.com/wireguard/windows v0.4.10/go.mod h1:v7w/8FC48tTBm1IzScDVPEEb0/GjLta+T0ybpP9UWRg=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1036,8 +1056,8 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCU
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw=
 inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8=
-inet.af/netstack v0.0.0-20211027215559-ec21145de76b h1:5aGmvztDCPhMYl/pByQYY0eaqbDzqTtX28180pffKqw=
-inet.af/netstack v0.0.0-20211027215559-ec21145de76b/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
+inet.af/netstack v0.0.0-20211101182044-1c1bcf452982 h1:hYciifHEv98/p8ln52ybKhgQpGouZWALFxxFE65RVdU=
+inet.af/netstack v0.0.0-20211101182044-1c1bcf452982/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU=
 inet.af/wf v0.0.0-20210516214145-a5343001b756 h1:muIT3C1rH3/xpvIH8blKkMvhctV7F+OtZqs7kcwHDBQ=

health/health.go

@@ -16,8 +16,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/multierr"
 )
 
 var (
@@ -268,7 +268,7 @@ func selfCheckLocked() {
 // OverallError returns a summary of the health state.
 //
 // If there are multiple problems, the error will be of type
-// multierror.MultipleErrors.
+// multierr.Error.
 func OverallError() error {
 	mu.Lock()
 	defer mu.Unlock()
@@ -337,7 +337,7 @@ func overallErrorLocked() error {
 		// Not super efficient (stringifying these in a sort), but probably max 2 or 3 items.
 		return errs[i].Error() < errs[j].Error()
 	})
-	return multierror.New(errs)
+	return multierr.New(errs...)
 }
 
 var (

hostinfo/hostinfo.go

@@ -7,11 +7,14 @@
 package hostinfo
 
 import (
+	"bufio"
 	"io"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync/atomic"
+	"time"
 
 	"go4.org/mem"
 	"tailscale.com/tailcfg"
@@ -224,3 +227,55 @@ func inKubernetes() bool {
 	}
 	return false
 }
+
+type etcAptSrcResult struct {
+	mod      time.Time
+	disabled bool
+}
+
+var etcAptSrcCache atomic.Value // of etcAptSrcResult
+
+// DisabledEtcAptSource reports whether Ubuntu (or similar) has disabled
+// the /etc/apt/sources.list.d/tailscale.list file contents upon upgrade
+// to a new release of the distro.
+//
+// See https://github.com/tailscale/tailscale/issues/3177
+func DisabledEtcAptSource() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	const path = "/etc/apt/sources.list.d/tailscale.list"
+	fi, err := os.Stat(path)
+	if err != nil || !fi.Mode().IsRegular() {
+		return false
+	}
+	mod := fi.ModTime()
+	if c, ok := etcAptSrcCache.Load().(etcAptSrcResult); ok && c.mod == mod {
+		return c.disabled
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	v := etcAptSourceFileIsDisabled(f)
+	etcAptSrcCache.Store(etcAptSrcResult{mod: mod, disabled: v})
+	return v
+}
+
+func etcAptSourceFileIsDisabled(r io.Reader) bool {
+	bs := bufio.NewScanner(r)
+	disabled := false // did we find the "disabled on upgrade" comment?
+	for bs.Scan() {
+		line := strings.TrimSpace(bs.Text())
+		if strings.Contains(line, "# disabled on upgrade") {
+			disabled = true
+		}
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		// Well, it has some contents in it at least.
+		return false
+	}
+	return disabled
+}

hostinfo/hostinfo_test.go

@@ -6,6 +6,7 @@ package hostinfo
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 )
 
@@ -27,3 +28,25 @@ func TestOSVersion(t *testing.T) {
 	}
 	t.Logf("Got: %#q", osVersion())
 }
+
+func TestEtcAptSourceFileIsDisabled(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want bool
+	}{
+		{"empty", "", false},
+		{"normal", "deb foo\n", false},
+		{"normal-commented", "# deb foo\n", false},
+		{"normal-disabled-by-ubuntu", "# deb foo # disabled on upgrade to dingus\n", true},
+		{"normal-disabled-then-uncommented", "deb foo # disabled on upgrade to dingus\n", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := etcAptSourceFileIsDisabled(strings.NewReader(tt.in))
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

ipn/backend.go

@@ -12,6 +12,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/structs"
 )
@@ -48,7 +49,7 @@ type EngineStatus struct {
 	RBytes, WBytes int64
 	NumLive        int
 	LiveDERPs      int // number of active DERP connections
-	LivePeers      map[tailcfg.NodeKey]ipnstate.PeerStatusLite
+	LivePeers      map[key.NodePublic]ipnstate.PeerStatusLite
 }
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend

ipn/ipnlocal/local.go

@@ -25,8 +25,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
@@ -50,6 +48,7 @@ import (
 	"tailscale.com/types/preftype"
 	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
@@ -97,6 +96,7 @@ type LocalBackend struct {
 	gotPortPollRes        chan struct{}    // closed upon first readPoller result
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
+	varRoot               string // or empty if SetVarRoot never called
 
 	filterHash deephash.Sum
 
@@ -122,6 +122,7 @@ type LocalBackend struct {
 	engineStatus     ipn.EngineStatus
 	endpoints        []tailcfg.Endpoint
 	blocked          bool
+	keyExpired       bool
 	authURL          string // cleared on Notify
 	authURLSticky    string // not cleared on Notify
 	interact         bool
@@ -335,8 +336,8 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 
 		if err := health.OverallError(); err != nil {
 			switch e := err.(type) {
-			case multierror.MultipleErrors:
-				for _, err := range e {
+			case multierr.Error:
+				for _, err := range e.Errors() {
 					s.Health = append(s.Health, err.Error())
 				}
 			default:
@@ -389,7 +390,7 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 				tailscaleIPs = append(tailscaleIPs, addr.IP())
 			}
 		}
-		sb.AddPeer(key.NodePublicFromRaw32(mem.B(p.Key[:])), &ipnstate.PeerStatus{
+		sb.AddPeer(p.Key, &ipnstate.PeerStatus{
 			InNetworkMap:       true,
 			ID:                 p.StableID,
 			UserID:             p.User,
@@ -452,18 +453,35 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		// TODO(crawshaw): display in the UI.
 		if errors.Is(st.Err, io.EOF) {
 			b.logf("[v1] Received error: EOF")
-		} else {
-			b.logf("Received error: %v", st.Err)
-			e := st.Err.Error()
-			b.send(ipn.Notify{ErrMessage: &e})
+			return
+		}
+		b.logf("Received error: %v", st.Err)
+		var uerr controlclient.UserVisibleError
+		if errors.As(st.Err, &uerr) {
+			s := uerr.UserVisibleError()
+			b.send(ipn.Notify{ErrMessage: &s})
 		}
 		return
 	}
 
 	b.mu.Lock()
 	wasBlocked := b.blocked
+	keyExpiryExtended := false
+	if st.NetMap != nil {
+		wasExpired := b.keyExpired
+		isExpired := !st.NetMap.Expiry.IsZero() && st.NetMap.Expiry.Before(time.Now())
+		if wasExpired && !isExpired {
+			keyExpiryExtended = true
+		}
+		b.keyExpired = isExpired
+	}
 	b.mu.Unlock()
 
+	if keyExpiryExtended && wasBlocked {
+		// Key extended, unblock the engine
+		b.blockEngineUpdates(false)
+	}
+
 	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
@@ -847,7 +865,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		})
 	}
 
-	var discoPublic tailcfg.DiscoKey
+	var discoPublic key.DiscoPublic
 	if controlclient.Debug.Disco {
 		discoPublic = b.e.DiscoPublicKey()
 	}
@@ -1562,7 +1580,7 @@ func (b *LocalBackend) parseWgStatusLocked(s *wgengine.Status) (ret ipn.EngineSt
 	var peerStats, peerKeys strings.Builder
 
 	ret.LiveDERPs = s.DERPs
-	ret.LivePeers = map[tailcfg.NodeKey]ipnstate.PeerStatusLite{}
+	ret.LivePeers = map[key.NodePublic]ipnstate.PeerStatusLite{}
 	for _, p := range s.Peers {
 		if !p.LastHandshake.IsZero() {
 			fmt.Fprintf(&peerStats, "%d/%d ", p.RxBytes, p.TxBytes)
@@ -1771,6 +1789,12 @@ func (b *LocalBackend) NetMap() *netmap.NetworkMap {
 	return b.netMap
 }
 
+func (b *LocalBackend) isEngineBlocked() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.blocked
+}
+
 // blockEngineUpdate sets b.blocked to block, while holding b.mu. Its
 // indirect effect is to turn b.authReconfig() into a no-op if block
 // is true.
@@ -1999,34 +2023,29 @@ func normalizeResolver(cfg dnstype.Resolver) dnstype.Resolver {
 	return cfg
 }
 
+// SetVarRoot sets the root directory of Tailscale's writable
+// storage area . (e.g. "/var/lib/tailscale")
+//
+// It should only be called before the LocalBackend is used.
+func (b *LocalBackend) SetVarRoot(dir string) {
+	b.varRoot = dir
+}
+
 // TailscaleVarRoot returns the root directory of Tailscale's writable
 // storage area. (e.g. "/var/lib/tailscale")
 //
 // It returns an empty string if there's no configured or discovered
 // location.
 func (b *LocalBackend) TailscaleVarRoot() string {
+	if b.varRoot != "" {
+		return b.varRoot
+	}
 	switch runtime.GOOS {
 	case "ios", "android":
 		dir, _ := paths.AppSharedDir.Load().(string)
 		return dir
 	}
-	// Temporary (2021-09-27) transitional fix for #2927 (Synology
-	// cert dir) on the way towards a more complete fix
-	// (#2932). It fixes any case where the state file is provided
-	// to tailscaled explicitly when it's not in the default
-	// location.
-	if fs, ok := b.store.(*ipn.FileStore); ok {
-		if fp := fs.Path(); fp != "" {
-			if dir := filepath.Dir(fp); strings.EqualFold(filepath.Base(dir), "tailscale") {
-				return dir
-			}
-		}
-	}
-	stateFile := paths.DefaultTailscaledStateFile()
-	if stateFile == "" {
-		return ""
-	}
-	return filepath.Dir(stateFile)
+	return ""
 }
 
 func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
@@ -2398,6 +2417,7 @@ func (b *LocalBackend) nextState() ipn.State {
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 		st          = b.engineStatus
+		keyExpired  = b.keyExpired
 	)
 	b.mu.Unlock()
 
@@ -2430,7 +2450,9 @@ func (b *LocalBackend) nextState() ipn.State {
 		}
 	case !wantRunning:
 		return ipn.Stopped
-	case !netMap.Expiry.IsZero() && time.Until(netMap.Expiry) <= 0:
+	case keyExpired:
+		// NetMap must be non-nil for us to get here.
+		// The node key expired, need to relogin.
 		return ipn.NeedsLogin
 	case netMap.MachineStatus != tailcfg.MachineAuthorized:
 		// TODO(crawshaw): handle tailcfg.MachineInvalid
@@ -2511,6 +2533,7 @@ func (b *LocalBackend) ResetForClientDisconnect() {
 	b.userID = ""
 	b.setNetMapLocked(nil)
 	b.prefs = new(ipn.Prefs)
+	b.keyExpired = false
 	b.authURL = ""
 	b.authURLSticky = ""
 	b.activeLogin = ""
@@ -2680,7 +2703,7 @@ func (b *LocalBackend) OperatorUserID() string {
 // TestOnlyPublicKeys returns the current machine and node public
 // keys. Used in tests only to facilitate automated node authorization
 // in the test harness.
-func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey tailcfg.NodeKey) {
+func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey key.NodePublic) {
 	b.mu.Lock()
 	prefs := b.prefs
 	machinePrivKey := b.machinePrivKey
@@ -2692,7 +2715,7 @@ func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeK
 
 	mk := machinePrivKey.Public()
 	nk := prefs.Persist.PrivateNodeKey.Public()
-	return mk, tailcfg.NodeKeyFromNodePublic(nk)
+	return mk, nk
 }
 
 func (b *LocalBackend) WaitingFiles() ([]apitype.WaitingFile, error) {
@@ -2782,7 +2805,7 @@ func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
 	b.mu.Lock()
 	cc := b.cc
 	if prefs := b.prefs; prefs != nil {
-		req.NodeKey = tailcfg.NodeKeyFromNodePublic(prefs.Persist.PrivateNodeKey.Public())
+		req.NodeKey = prefs.Persist.PrivateNodeKey.Public()
 	}
 	b.mu.Unlock()
 	if cc == nil {

ipn/ipnlocal/loglines_test.go

@@ -90,7 +90,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()
@@ -105,7 +105,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       11,
 			RxBytes:       12,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()

ipn/ipnlocal/peerapi.go

@@ -6,6 +6,7 @@ package ipnlocal
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -29,11 +30,13 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine"
 )
 
@@ -500,9 +503,16 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handlePeerPut(w, r)
 		return
 	}
-	if r.URL.Path == "/v0/goroutines" {
+	switch r.URL.Path {
+	case "/v0/goroutines":
 		h.handleServeGoroutines(w, r)
 		return
+	case "/v0/env":
+		h.handleServeEnv(w, r)
+		return
+	case "/v0/metrics":
+		h.handleServeMetrics(w, r)
+		return
 	}
 	who := h.peerUser.DisplayName
 	fmt.Fprintf(w, `<html>
@@ -710,3 +720,32 @@ func (h *peerAPIHandler) handleServeGoroutines(w http.ResponseWriter, r *http.Re
 	}
 	w.Write(buf)
 }
+
+func (h *peerAPIHandler) handleServeEnv(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	var data struct {
+		Hostinfo *tailcfg.Hostinfo
+		Uid      int
+		Args     []string
+		Env      []string
+	}
+	data.Hostinfo = hostinfo.New()
+	data.Uid = os.Getuid()
+	data.Args = os.Args
+	data.Env = os.Environ()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}
+
+func (h *peerAPIHandler) handleServeMetrics(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}

ipn/ipnlocal/state_test.go

@@ -867,6 +867,45 @@ func TestStateMachine(t *testing.T) {
 		// change either.
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
+	t.Logf("\n\nExpireKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(-time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsTrue)
+	}
+
+	t.Logf("\n\nExtendKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Starting, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsFalse)
+	}
+	notifies.expect(1)
+	// Fake a DERP connection.
+	b.setWgengineStatus(&wgengine.Status{DERPs: 1}, nil)
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Running, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Running, qt.Equals, b.State())
+	}
 }
 
 type testStateStorage struct {

ipn/ipnserver/server.go

@@ -52,16 +52,13 @@ import (
 
 // Options is the configuration of the Tailscale node agent.
 type Options struct {
-	// SocketPath, on unix systems, is the unix socket path to listen
-	// on for frontend connections.
-	SocketPath string
-
-	// Port, on windows, is the localhost TCP port to listen on for
-	// frontend connections.
-	Port int
-
-	// StatePath is the path to the stored agent state.
-	StatePath string
+	// VarRoot is the the Tailscale daemon's private writable
+	// directory (usually "/var/lib/tailscale" on Linux) that
+	// contains the "tailscaled.state" file, the "certs" directory
+	// for TLS certs, and the "files" directory for incoming
+	// Taildrop files before they're moved to a user directory.
+	// If empty, Taildrop and TLS certs don't function.
+	VarRoot string
 
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
@@ -83,10 +80,6 @@ type Options struct {
 	// the actual definition of "disconnect" is when the
 	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
-
-	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
-	// to register a debug handler.
-	DebugMux *http.ServeMux
 }
 
 // Server is an IPN backend and its set of 0 or more active localhost
@@ -100,8 +93,8 @@ type Server struct {
 	// being run in "client mode" that requires an active GUI
 	// connection (such as on Windows by default).  Even if this
 	// is true, the ForceDaemon pref can override this.
-	resetOnZero bool
-	opts        Options
+	resetOnZero       bool
+	autostartStateKey ipn.StateKey
 
 	bsMu sync.Mutex // lock order: bsMu, then mu
 	bs   *ipn.BackendServer
@@ -114,6 +107,9 @@ type Server struct {
 	disconnectSub  map[chan<- struct{}]struct{} // keys are subscribers of disconnects
 }
 
+// LocalBackend returns the server's LocalBackend.
+func (s *Server) LocalBackend() *ipnlocal.LocalBackend { return s.b }
+
 // connIdentity represents the owner of a localhost TCP or unix socket connection.
 type connIdentity struct {
 	Conn       net.Conn
@@ -414,13 +410,16 @@ func (s *Server) checkConnIdentityLocked(ci connIdentity) error {
 //
 // s.mu must not be held.
 func (s *Server) localAPIPermissions(ci connIdentity) (read, write bool) {
-	if runtime.GOOS == "windows" {
+	switch runtime.GOOS {
+	case "windows":
 		s.mu.Lock()
 		defer s.mu.Unlock()
 		if s.checkConnIdentityLocked(ci) == nil {
 			return true, true
 		}
 		return false, false
+	case "js":
+		return true, true
 	}
 	if ci.IsUnixSock {
 		return true, !isReadonlyConn(ci, s.b.OperatorUserID(), logger.Discard)
@@ -606,18 +605,57 @@ func tryWindowsAppDataMigration(logf logger.Logf, path string) string {
 	return paths.TryConfigFileMigration(logf, oldFile, path)
 }
 
+// StateStore returns a StateStore from path.
+//
+// The path should be an absolute path to a file.
+//
+// Special cases:
+//
+//   * empty string means to use an in-memory store
+//   * if the string begins with "kube:", the suffix
+//     is a Kubernetes secret name
+//   * if the string begins with "arn:", the value is
+//     an AWS ARN for an SSM.
+func StateStore(path string, logf logger.Logf) (ipn.StateStore, error) {
+	if path == "" {
+		return &ipn.MemoryStore{}, nil
+	}
+	const kubePrefix = "kube:"
+	const arnPrefix = "arn:"
+	switch {
+	case strings.HasPrefix(path, kubePrefix):
+		secretName := strings.TrimPrefix(path, kubePrefix)
+		store, err := ipn.NewKubeStore(secretName)
+		if err != nil {
+			return nil, fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
+		}
+		return store, nil
+	case strings.HasPrefix(path, arnPrefix):
+		store, err := aws.NewStore(path)
+		if err != nil {
+			return nil, fmt.Errorf("aws.NewStore(%q): %v", path, err)
+		}
+		return store, nil
+	}
+	if runtime.GOOS == "windows" {
+		path = tryWindowsAppDataMigration(logf, path)
+	}
+	store, err := ipn.NewFileStore(path)
+	if err != nil {
+		return nil, fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
+	}
+	return store, nil
+}
+
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
-func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+//
+// Deprecated: use New and Server.Run instead.
+func Run(ctx context.Context, logf logger.Logf, ln net.Listener, store ipn.StateStore, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
 	getEngine = getEngineUntilItWorksWrapper(getEngine)
 	runDone := make(chan struct{})
 	defer close(runDone)
 
-	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
-	if err != nil {
-		return fmt.Errorf("safesocket.Listen: %v", err)
-	}
-
 	var serverMu sync.Mutex
 	var serverOrNil *Server
 
@@ -633,57 +671,28 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			s.stopAll()
 		}
 		serverMu.Unlock()
-		listen.Close()
+		ln.Close()
 	}()
-	logf("Listening on %v", listen.Addr())
+	logf("Listening on %v", ln.Addr())
 
 	var serverModeUser *user.User
-	var store ipn.StateStore
-	if opts.StatePath != "" {
-		const kubePrefix = "kube:"
-		const arnPrefix = "arn:"
-		path := opts.StatePath
-		switch {
-		case strings.HasPrefix(path, kubePrefix):
-			secretName := strings.TrimPrefix(path, kubePrefix)
-			store, err = ipn.NewKubeStore(secretName)
-			if err != nil {
-				return fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
-			}
-		case strings.HasPrefix(path, arnPrefix):
-			store, err = aws.NewStore(path)
-			if err != nil {
-				return fmt.Errorf("aws.NewStore(%q): %v", path, err)
-			}
-		default:
-			if runtime.GOOS == "windows" {
-				path = tryWindowsAppDataMigration(logf, path)
-			}
-			store, err = ipn.NewFileStore(path)
-			if err != nil {
-				return fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
-			}
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return fmt.Errorf("calling ReadState on state store: %w", err)
 		}
-		if opts.AutostartStateKey == "" {
-			autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
-			if err != nil && err != ipn.ErrStateNotExist {
-				return fmt.Errorf("calling ReadState on %s: %w", path, err)
-			}
-			key := string(autoStartKey)
-			if strings.HasPrefix(key, "user-") {
-				uid := strings.TrimPrefix(key, "user-")
-				u, err := lookupUserFromID(logf, uid)
-				if err != nil {
-					logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
-				} else {
-					logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
-					serverModeUser = u
-				}
-				opts.AutostartStateKey = ipn.StateKey(key)
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
 			}
+			opts.AutostartStateKey = ipn.StateKey(key)
 		}
-	} else {
-		store = &ipn.MemoryStore{}
 	}
 
 	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
@@ -693,7 +702,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	if err != nil {
 		logf("ipnserver: initial getEngine call: %v", err)
 		for i := 1; ctx.Err() == nil; i++ {
-			c, err := listen.Accept()
+			c, err := ln.Accept()
 			if err != nil {
 				logf("%d: Accept: %v", i, err)
 				bo.BackOff(ctx, err)
@@ -720,8 +729,8 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 		}
 	}
 	if unservedConn != nil {
-		listen = &listenerWithReadyConn{
-			Listener: listen,
+		ln = &listenerWithReadyConn{
+			Listener: ln,
 			c:        unservedConn,
 		}
 	}
@@ -733,49 +742,78 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	serverMu.Lock()
 	serverOrNil = server
 	serverMu.Unlock()
-	return server.Serve(ctx, listen)
+	return server.Run(ctx, ln)
 }
 
 // New returns a new Server.
 //
-// The opts.StatePath option is ignored; it's only used by Run.
+// To start it, use the Server.Run method.
 func New(logf logger.Logf, logid string, store ipn.StateStore, eng wgengine.Engine, serverModeUser *user.User, opts Options) (*Server, error) {
 	b, err := ipnlocal.NewLocalBackend(logf, logid, store, eng)
 	if err != nil {
 		return nil, fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	b.SetVarRoot(opts.VarRoot)
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)
 	})
 
-	if opts.DebugMux != nil {
-		opts.DebugMux.HandleFunc("/debug/ipn", func(w http.ResponseWriter, r *http.Request) {
-			serveHTMLStatus(w, b)
-		})
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return nil, fmt.Errorf("calling ReadState on store: %w", err)
+		}
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
+			}
+			opts.AutostartStateKey = ipn.StateKey(key)
+		}
 	}
 
 	server := &Server{
-		b:              b,
-		backendLogID:   logid,
-		logf:           logf,
-		resetOnZero:    !opts.SurviveDisconnects,
-		serverModeUser: serverModeUser,
-		opts:           opts,
+		b:                 b,
+		backendLogID:      logid,
+		logf:              logf,
+		resetOnZero:       !opts.SurviveDisconnects,
+		serverModeUser:    serverModeUser,
+		autostartStateKey: opts.AutostartStateKey,
 	}
 	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 	return server, nil
 }
 
-// Serve accepts connections from ln forever.
+// Run runs the server, accepting connections from ln forever.
 //
-// The context is only used to suppress errors
-func (s *Server) Serve(ctx context.Context, ln net.Listener) error {
+// If the context is done, the listener is closed.
+func (s *Server) Run(ctx context.Context, ln net.Listener) error {
 	defer s.b.Shutdown()
-	if s.opts.AutostartStateKey != "" {
+
+	runDone := make(chan struct{})
+	defer close(runDone)
+
+	// When the context is closed or when we return, whichever is first, close our listener
+	// and all open connections.
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-runDone:
+		}
+		s.stopAll()
+		ln.Close()
+	}()
+
+	if s.autostartStateKey != "" {
 		s.bs.GotCommand(ctx, &ipn.Command{
 			Version: version.Long,
 			Start: &ipn.StartArgs{
-				Opts: ipn.Options{StateKey: s.opts.AutostartStateKey},
+				Opts: ipn.Options{StateKey: s.autostartStateKey},
 			},
 		})
 	}
@@ -1013,13 +1051,13 @@ func (s *Server) localhostHandler(ci connIdentity) http.Handler {
 			io.WriteString(w, "<html><title>Tailscale</title><body><h1>Tailscale</h1>This is the local Tailscale daemon.")
 			return
 		}
-		serveHTMLStatus(w, s.b)
+		s.ServeHTMLStatus(w, r)
 	})
 }
 
-func serveHTMLStatus(w http.ResponseWriter, b *ipnlocal.LocalBackend) {
+func (s *Server) ServeHTMLStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	st := b.Status()
+	st := s.b.Status()
 	// TODO(bradfitz): add LogID and opts to st?
 	st.WriteHTML(w)
 }

ipn/ipnserver/server_test.go

@@ -62,10 +62,16 @@ func TestRunMultipleAccepts(t *testing.T) {
 	}
 	t.Cleanup(eng.Close)
 
-	opts := ipnserver.Options{
-		SocketPath: socketPath,
-	}
+	opts := ipnserver.Options{}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
+	store := new(ipn.MemoryStore)
+
+	ln, _, err := safesocket.Listen(socketPath, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	err = ipnserver.Run(ctx, logTriggerTestf, ln, store, "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -72,7 +72,7 @@ func (s *Status) Peers() []key.NodePublic {
 type PeerStatusLite struct {
 	TxBytes, RxBytes int64
 	LastHandshake    time.Time
-	NodeKey          tailcfg.NodeKey
+	NodeKey          key.NodePublic
 }
 
 type PeerStatus struct {

ipn/localapi/localapi.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/net/netknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/version"
 )
 
@@ -113,6 +114,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveSetDNS(w, r)
 	case "/localapi/v0/derpmap":
 		h.serveDERPMap(w, r)
+	case "/localapi/v0/metrics":
+		h.serveMetrics(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -184,6 +187,17 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+func (h *Handler) serveMetrics(w http.ResponseWriter, r *http.Request) {
+	// Require write access out of paranoia that the metrics
+	// might contain something sensitive.
+	if !h.PermitWrite {
+		http.Error(w, "metric access denied", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 // serveProfileFunc is the implementation of Handler.serveProfile, after auth,
 // for platforms where we want to link it in.
 var serveProfileFunc func(http.ResponseWriter, *http.Request)

log/filelogger/log.go

@@ -106,6 +106,7 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	if w.fday != day {
 		w.startNewFileLocked()
 	}
+	out = removeDatePrefix(out)
 	if w.f != nil {
 		// RFC3339Nano but with a fixed number (3) of nanosecond digits:
 		const formatPre = "2006-01-02T15:04:05"
@@ -118,6 +119,30 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	}
 }
 
+func isNum(b byte) bool { return '0' <= b && b <= '9' }
+
+// removeDatePrefix returns a subslice of v with the log package's
+// standard datetime prefix format removed, if present.
+func removeDatePrefix(v []byte) []byte {
+	const format = "2009/01/23 01:23:23 "
+	if len(v) < len(format) {
+		return v
+	}
+	for i, b := range v[:len(format)] {
+		fb := format[i]
+		if isNum(fb) {
+			if !isNum(b) {
+				return v
+			}
+			continue
+		}
+		if b != fb {
+			return v
+		}
+	}
+	return v[len(format):]
+}
+
 // startNewFileLocked opens a new log file for writing
 // and also cleans up any old files.
 //

log/filelogger/log_test.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package filelogger
+
+import "testing"
+
+func TestRemoveDatePrefix(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"\n", "\n"},
+		{"2009/01/23 01:23:23", "2009/01/23 01:23:23"},
+		{"2009/01/23 01:23:23 \n", "\n"},
+		{"2009/01/23 01:23:23 foo\n", "foo\n"},
+		{"9999/01/23 01:23:23 foo\n", "foo\n"},
+		{"2009_01/23 01:23:23 had an underscore\n", "2009_01/23 01:23:23 had an underscore\n"},
+	}
+	for i, tt := range tests {
+		got := removeDatePrefix([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Logf("[%d] removeDatePrefix(%q) = %q; want %q", i, tt.in, got, tt.want)
+		}
+	}
+
+}

logpolicy/logpolicy.go

@@ -31,6 +31,8 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
+	"tailscale.com/net/dnscache"
+	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/netknob"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
@@ -38,6 +40,7 @@ import (
 	"tailscale.com/paths"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/racebuild"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
@@ -500,6 +503,9 @@ func New(collection string) *Policy {
 		},
 		HTTPC: &http.Client{Transport: newLogtailTransport(logtail.DefaultHost)},
 	}
+	if collection == logtail.CollectionNode {
+		c.MetricsDelta = clientmetric.EncodeLogTailMetricsDelta
+	}
 
 	if val := getLogTarget(); val != "" {
 		log.Println("You have enabled a non-default log target. Doing without being told to by Tailscale staff or your network administrator will make getting support difficult.")
@@ -581,17 +587,29 @@ func newLogtailTransport(host string) *http.Transport {
 
 	// Log whenever we dial:
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {
-		nd := netns.FromDialer(&net.Dialer{
+		nd := netns.FromDialer(log.Printf, &net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: netknob.PlatformTCPKeepAlive(),
 		})
 		t0 := time.Now()
 		c, err := nd.DialContext(ctx, netw, addr)
 		d := time.Since(t0).Round(time.Millisecond)
-		if err != nil {
-			log.Printf("logtail: dial %q failed: %v (in %v)", addr, err, d)
-		} else {
+		if err == nil {
 			log.Printf("logtail: dialed %q in %v", addr, d)
+			return c, nil
+		}
+
+		// If we failed to dial, try again with bootstrap DNS.
+		log.Printf("logtail: dial %q failed: %v (in %v), trying bootstrap...", addr, err, d)
+		dnsCache := &dnscache.Resolver{
+			Forward:          dnscache.Get().Forward, // use default cache's forwarder
+			UseLastGood:      true,
+			LookupIPFallback: dnsfallback.Lookup,
+		}
+		dialer := dnscache.Dialer(nd.DialContext, dnsCache)
+		c, err = dialer(ctx, netw, addr)
+		if err == nil {
+			log.Printf("logtail: bootstrap dial succeeded")
 		}
 		return c, err
 	}

logtail/logtail.go

@@ -28,6 +28,12 @@ import (
 // Config.BaseURL isn't provided.
 const DefaultHost = "log.tailscale.io"
 
+const (
+	// CollectionNode is the name of a logtail Config.Collection
+	// for tailscaled (or equivalent: IPNExtension, Android app).
+	CollectionNode = "tailnode.log.tailscale.io"
+)
+
 type Encoder interface {
 	EncodeAll(src, dst []byte) []byte
 	Close() error
@@ -46,6 +52,12 @@ type Config struct {
 	Buffer         Buffer           // temp storage, if nil a MemoryBuffer
 	NewZstdEncoder func() Encoder   // if set, used to compress logs for transmission
 
+	// MetricsDelta, if non-nil, is a func that returns an encoding
+	// delta in clientmetrics to upload alongside existing logs.
+	// It can return either an empty string (for nothing) or a string
+	// that's safe to embed in a JSON string literal without further escaping.
+	MetricsDelta func() string
+
 	// DrainLogs, if non-nil, disables automatic uploading of new logs,
 	// so that logs are only uploaded when a token is sent to DrainLogs.
 	DrainLogs <-chan struct{}
@@ -84,6 +96,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 		drainLogs:      cfg.DrainLogs,
 		timeNow:        cfg.TimeNow,
 		bo:             backoff.NewBackoff("logtail", logf, 30*time.Second),
+		metricsDelta:   cfg.MetricsDelta,
 
 		shutdownStart: make(chan struct{}),
 		shutdownDone:  make(chan struct{}),
@@ -119,6 +132,7 @@ type Logger struct {
 	zstdEncoder    Encoder
 	uploadCancel   func()
 	explainedRaw   bool
+	metricsDelta   func() string // or nil
 
 	shutdownStart chan struct{} // closed when shutdown begins
 	shutdownDone  chan struct{} // closed when shutdown complete
@@ -426,6 +440,14 @@ func (l *Logger) encodeText(buf []byte, skipClientTime bool) []byte {
 		b = append(b, "\"}, "...)
 	}
 
+	if l.metricsDelta != nil {
+		if d := l.metricsDelta(); d != "" {
+			b = append(b, `"metrics": "`...)
+			b = append(b, d...)
+			b = append(b, `",`...)
+		}
+	}
+
 	b = append(b, "\"text\": \""...)
 	for i, c := range buf {
 		switch c {

net/dns/manager_linux.go

@@ -180,20 +180,55 @@ func dnsMode(logf logger.Logf, env newOSConfigEnv) (ret string, err error) {
 			return "direct", nil
 		}
 	case "NetworkManager":
-		// You'd think we would use newNMManager somewhere in
-		// here. However, as explained in
-		// https://github.com/tailscale/tailscale/issues/1699 , using
-		// NetworkManager for DNS configuration carries with it the
-		// cost of losing IPv6 configuration on the Tailscale network
-		// interface. So, when we can avoid it, we bypass
-		// NetworkManager by replacing resolv.conf directly.
-		//
-		// If you ever try to put NMManager back here, keep in mind
-		// that versions >=1.26.6 will ignore DNS configuration
-		// anyway, so you still need a fallback path that uses
-		// directManager.
 		dbg("rc", "nm")
-		return "direct", nil
+		// Sometimes, NetworkManager owns the configuration but points
+		// it at systemd-resolved.
+		if err := resolvedIsActuallyResolver(bs); err != nil {
+			dbg("resolved", "not-in-use")
+			// You'd think we would use newNMManager here. However, as
+			// explained in
+			// https://github.com/tailscale/tailscale/issues/1699 ,
+			// using NetworkManager for DNS configuration carries with
+			// it the cost of losing IPv6 configuration on the
+			// Tailscale network interface. So, when we can avoid it,
+			// we bypass NetworkManager by replacing resolv.conf
+			// directly.
+			//
+			// If you ever try to put NMManager back here, keep in mind
+			// that versions >=1.26.6 will ignore DNS configuration
+			// anyway, so you still need a fallback path that uses
+			// directManager.
+			return "direct", nil
+		}
+		dbg("nm-resolved", "yes")
+
+		if err := env.dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
+			dbg("resolved", "no")
+			return "direct", nil
+		}
+
+		// See large comment above for reasons we'd use NM rather than
+		// resolved. systemd-resolved is actually in charge of DNS
+		// configuration, but in some cases we might need to configure
+		// it via NetworkManager. All the logic below is probing for
+		// that case: is NetworkManager running? If so, is it one of
+		// the versions that requires direct interaction with it?
+		if err := env.dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return "systemd-resolved", nil
+		}
+		safe, err := env.nmVersionBetween("1.26.0", "1.26.5")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return "", fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if safe {
+			dbg("nm-safe", "yes")
+			return "network-manager", nil
+		}
+		dbg("nm-safe", "no")
+		return "systemd-resolved", nil
 	default:
 		dbg("rc", "unknown")
 		return "direct", nil
@@ -244,6 +279,13 @@ func nmIsUsingResolved() error {
 	return nil
 }
 
+// resolvedIsActuallyResolver reports whether the given resolv.conf
+// bytes describe a configuration where systemd-resolved (127.0.0.53)
+// is the only configured nameserver.
+//
+// Returns an error if the configuration is something other than
+// exclusively systemd-resolved, or nil if the config is only
+// systemd-resolved.
 func resolvedIsActuallyResolver(bs []byte) error {
 	cfg, err := readResolv(bytes.NewBuffer(bs))
 	if err != nil {

net/dns/manager_linux_test.go

@@ -34,7 +34,7 @@ func TestLinuxDNSMode(t *testing.T) {
 				resolvDotConf(
 					"# Managed by NetworkManager",
 					"nameserver 10.0.0.1")),
-			wantLog: "dns: [rc=nm ret=direct]",
+			wantLog: "dns: [rc=nm resolved=not-in-use ret=direct]",
 			want:    "direct",
 		},
 		{
@@ -172,6 +172,52 @@ func TestLinuxDNSMode(t *testing.T) {
 			wantLog: "dns: [rc=resolved resolved=no ret=direct]",
 			want:    "direct",
 		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm-safe=no ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_but_no_resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes resolved=no ret=direct]",
+			want:    "direct",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_safe_nm",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.26.3", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm-safe=yes ret=network-manager]",
+			want:    "network-manager",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_no_networkmanager",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning()),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm=no ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

net/dns/resolved.go

@@ -12,6 +12,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
@@ -171,6 +172,14 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
 		m.ifidx, linkDomains,
 	).Store()
+	if err != nil && err.Error() == "Argument list too long" { // TODO: better error match
+		// Issue 3188: older systemd-resolved had argument length limits.
+		// Trim out the *.arpa. entries and try again.
+		err = m.resolved.CallWithContext(
+			ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
+			m.ifidx, linkDomainsWithoutReverseDNS(linkDomains),
+		).Store()
+	}
 	if err != nil {
 		return fmt.Errorf("setLinkDomains: %w", err)
 	}
@@ -234,3 +243,16 @@ func (m *resolvedManager) Close() error {
 
 	return nil
 }
+
+// linkDomainsWithoutReverseDNS returns a copy of v without
+// *.arpa. entries.
+func linkDomainsWithoutReverseDNS(v []resolvedLinkDomain) (ret []resolvedLinkDomain) {
+	for _, d := range v {
+		if strings.HasSuffix(d.Domain, ".arpa.") {
+			// Oh well. At least the rest will work.
+			continue
+		}
+		ret = append(ret, d)
+	}
+	return ret
+}

net/dns/resolver/forwarder.go

@@ -342,7 +342,7 @@ func (f *forwarder) getKnownDoHClient(ip netaddr.IP) (urlBase string, c *http.Cl
 	if f.dohClient == nil {
 		f.dohClient = map[string]*http.Client{}
 	}
-	nsDialer := netns.NewDialer()
+	nsDialer := netns.NewDialer(f.logf)
 	c = &http.Client{
 		Transport: &http.Transport{
 			IdleConnTimeout: dohTransportTimeout,

net/dnsfallback/dnsfallback.go

@@ -90,7 +90,7 @@ func Lookup(ctx context.Context, host string) ([]netaddr.IP, error) {
 // serverName and serverIP of are, say, "derpN.tailscale.com".
 // queryName is the name being sought (e.g. "controlplane.tailscale.com"), passed as hint.
 func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netaddr.IP, queryName string) (dnsMap, error) {
-	dialer := netns.NewDialer()
+	dialer := netns.NewDialer(log.Printf)
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.Proxy = tshttpproxy.ProxyFromEnvironment
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {

net/interfaces/interfaces_windows.go

@@ -116,8 +116,12 @@ func notTailscaleInterface(iface *winipcfg.IPAdapterAddresses) bool {
 	// TODO(bradfitz): do this without the Description method's
 	// utf16-to-string allocation. But at least we only do it for
 	// the virtual interfaces, for which there won't be many.
-	return !(iface.IfType == winipcfg.IfTypePropVirtual &&
-		iface.Description() == tsconst.WintunInterfaceDesc)
+	if iface.IfType != winipcfg.IfTypePropVirtual {
+		return true
+	}
+	desc := iface.Description()
+	return !(strings.Contains(desc, tsconst.WintunInterfaceDesc) ||
+		strings.Contains(desc, tsconst.WintunInterfaceDesc0_14))
 }
 
 // NonTailscaleInterfaces returns a map of interface LUID to interface

net/netcheck/netcheck.go

@@ -34,6 +34,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/opt"
+	"tailscale.com/util/clientmetric"
 )
 
 // Debugging and experimentation tweakables.
@@ -232,6 +233,12 @@ func (c *Client) MakeNextReportFull() {
 func (c *Client) ReceiveSTUNPacket(pkt []byte, src netaddr.IPPort) {
 	c.vlogf("received STUN packet from %s", src)
 
+	if src.IP().Is4() {
+		metricSTUNRecv4.Add(1)
+	} else if src.IP().Is6() {
+		metricSTUNRecv6.Add(1)
+	}
+
 	c.mu.Lock()
 	if c.handleHairSTUNLocked(pkt, src) {
 		c.mu.Unlock()
@@ -737,7 +744,13 @@ func (c *Client) udpBindAddr() string {
 // GetReport gets a report.
 //
 // It may not be called concurrently with itself.
-func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, error) {
+func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (_ *Report, reterr error) {
+	defer func() {
+		if reterr != nil {
+			metricNumGetReportError.Add(1)
+		}
+	}()
+	metricNumGetReport.Add(1)
 	// Mask user context with ours that we guarantee to cancel so
 	// we can depend on it being closed in goroutines later.
 	// (User ctx might be context.Background, etc)
@@ -769,6 +782,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		last = nil // causes makeProbePlan below to do a full (initial) plan
 		c.nextFull = false
 		c.lastFull = now
+		metricNumGetReportFull.Add(1)
 	}
 	rs.incremental = last != nil
 	c.mu.Unlock()
@@ -793,7 +807,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 
 	// Create a UDP4 socket used for sending to our discovered IPv4 address.
-	rs.pc4Hair, err = netns.Listener().ListenPacket(ctx, "udp4", ":0")
+	rs.pc4Hair, err = netns.Listener(c.logf).ListenPacket(ctx, "udp4", ":0")
 	if err != nil {
 		c.logf("udp4: %v", err)
 		return nil, err
@@ -821,7 +835,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	if f := c.GetSTUNConn4; f != nil {
 		rs.pc4 = f()
 	} else {
-		u4, err := netns.Listener().ListenPacket(ctx, "udp4", c.udpBindAddr())
+		u4, err := netns.Listener(c.logf).ListenPacket(ctx, "udp4", c.udpBindAddr())
 		if err != nil {
 			c.logf("udp4: %v", err)
 			return nil, err
@@ -834,7 +848,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		if f := c.GetSTUNConn6; f != nil {
 			rs.pc6 = f()
 		} else {
-			u6, err := netns.Listener().ListenPacket(ctx, "udp6", c.udpBindAddr())
+			u6, err := netns.Listener(c.logf).ListenPacket(ctx, "udp6", c.udpBindAddr())
 			if err != nil {
 				c.logf("udp6: %v", err)
 			} else {
@@ -983,6 +997,7 @@ func (c *Client) runHTTPOnlyChecks(ctx context.Context, last *Report, rs *report
 }
 
 func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netaddr.IP, error) {
+	metricHTTPSend.Add(1)
 	var result httpstat.Result
 	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), overallProbeTimeout)
 	defer cancel()
@@ -1217,6 +1232,7 @@ func (rs *reportState) runProbe(ctx context.Context, dm *tailcfg.DERPMap, probe
 
 	switch probe.proto {
 	case probeIPv4:
+		metricSTUNSend4.Add(1)
 		n, err := rs.pc4.WriteTo(req, addr)
 		if n == len(req) && err == nil {
 			rs.mu.Lock()
@@ -1224,6 +1240,7 @@ func (rs *reportState) runProbe(ctx context.Context, dm *tailcfg.DERPMap, probe
 			rs.mu.Unlock()
 		}
 	case probeIPv6:
+		metricSTUNSend6.Add(1)
 		n, err := rs.pc6.WriteTo(req, addr)
 		if n == len(req) && err == nil {
 			rs.mu.Lock()
@@ -1322,3 +1339,15 @@ func conciseOptBool(b opt.Bool, trueVal string) string {
 	}
 	return ""
 }
+
+var (
+	metricNumGetReport      = clientmetric.NewCounter("netcheck_report")
+	metricNumGetReportFull  = clientmetric.NewCounter("netcheck_report_full")
+	metricNumGetReportError = clientmetric.NewCounter("netcheck_report_error")
+
+	metricSTUNSend4 = clientmetric.NewCounter("netcheck_stun_send_ipv4")
+	metricSTUNSend6 = clientmetric.NewCounter("netcheck_stun_send_ipv6")
+	metricSTUNRecv4 = clientmetric.NewCounter("netcheck_stun_recv_ipv4")
+	metricSTUNRecv6 = clientmetric.NewCounter("netcheck_stun_recv_ipv6")
+	metricHTTPSend  = clientmetric.NewCounter("netcheck_https_measure")
+)

net/netns/netns.go

@@ -21,6 +21,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/net/netknob"
 	"tailscale.com/syncs"
+	"tailscale.com/types/logger"
 )
 
 var disabled syncs.AtomicBool
@@ -34,19 +35,19 @@ func SetEnabled(on bool) {
 // Listener returns a new net.Listener with its Control hook func
 // initialized as necessary to run in logical network namespace that
 // doesn't route back into Tailscale.
-func Listener() *net.ListenConfig {
+func Listener(logf logger.Logf) *net.ListenConfig {
 	if disabled.Get() {
 		return new(net.ListenConfig)
 	}
-	return &net.ListenConfig{Control: control}
+	return &net.ListenConfig{Control: control(logf)}
 }
 
 // NewDialer returns a new Dialer using a net.Dialer with its Control
 // hook func initialized as necessary to run in a logical network
 // namespace that doesn't route back into Tailscale. It also handles
 // using a SOCKS if configured in the environment with ALL_PROXY.
-func NewDialer() Dialer {
-	return FromDialer(&net.Dialer{
+func NewDialer(logf logger.Logf) Dialer {
+	return FromDialer(logf, &net.Dialer{
 		KeepAlive: netknob.PlatformTCPKeepAlive(),
 	})
 }
@@ -55,11 +56,11 @@ func NewDialer() Dialer {
 // network namespace that doesn't route back into Tailscale. It also
 // handles using a SOCKS if configured in the environment with
 // ALL_PROXY.
-func FromDialer(d *net.Dialer) Dialer {
+func FromDialer(logf logger.Logf, d *net.Dialer) Dialer {
 	if disabled.Get() {
 		return d
 	}
-	d.Control = control
+	d.Control = control(logf)
 	if wrapDialer != nil {
 		return wrapDialer(d)
 	}

net/netns/netns_android.go

@@ -11,6 +11,8 @@ import (
 	"fmt"
 	"sync"
 	"syscall"
+
+	"tailscale.com/types/logger"
 )
 
 var (
@@ -44,11 +46,15 @@ func SetAndroidProtectFunc(f func(fd int) error) {
 	androidProtectFunc = f
 }
 
-// control marks c as necessary to dial in a separate network namespace.
+func control(logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return controlC
+}
+
+// controlC marks c as necessary to dial in a separate network namespace.
 //
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
-func control(network, address string, c syscall.RawConn) error {
+func controlC(network, address string, c syscall.RawConn) error {
 	var sockErr error
 	err := c.Control(func(fd uintptr) {
 		androidProtectFuncMu.Lock()

net/netns/netns_darwin_tailscaled.go

@@ -9,26 +9,32 @@ package netns
 
 import (
 	"fmt"
-	"log"
 	"strings"
 	"syscall"
 
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/types/logger"
 )
 
-// control marks c as necessary to dial in a separate network namespace.
+func control(logf logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return func(network, address string, c syscall.RawConn) error {
+		return controlLogf(logf, network, address, c)
+	}
+}
+
+// controlLogf marks c as necessary to dial in a separate network namespace.
 //
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
-func control(network, address string, c syscall.RawConn) error {
+func controlLogf(logf logger.Logf, network, address string, c syscall.RawConn) error {
 	if strings.HasPrefix(address, "127.") || address == "::1" {
 		// Don't bind to an interface for localhost connections.
 		return nil
 	}
 	idx, err := interfaces.DefaultRouteInterfaceIndex()
 	if err != nil {
-		log.Printf("netns: DefaultRouteInterfaceIndex: %v", err)
+		logf("[unexpected] netns: DefaultRouteInterfaceIndex: %v", err)
 		return nil
 	}
 	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
@@ -47,7 +53,7 @@ func control(network, address string, c syscall.RawConn) error {
 		return fmt.Errorf("RawConn.Control on %T: %w", c, err)
 	}
 	if sockErr != nil {
-		log.Printf("netns: control(%q, %q), v6=%v, index=%v: %v", network, address, v6, idx, sockErr)
+		logf("[unexpected] netns: control(%q, %q), v6=%v, index=%v: %v", network, address, v6, idx, sockErr)
 	}
 	return sockErr
 }

net/netns/netns_default.go

@@ -7,9 +7,17 @@
 
 package netns
 
-import "syscall"
+import (
+	"syscall"
 
-// control does nothing to c.
-func control(network, address string, c syscall.RawConn) error {
+	"tailscale.com/types/logger"
+)
+
+func control(logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return controlC
+}
+
+// controlC does nothing to c.
+func controlC(network, address string, c syscall.RawConn) error {
 	return nil
 }

net/netns/netns_linux.go

@@ -17,6 +17,7 @@ import (
 
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/types/logger"
 )
 
 // tailscaleBypassMark is the mark indicating that packets originating
@@ -82,11 +83,15 @@ func ignoreErrors() bool {
 	return false
 }
 
-// control marks c as necessary to dial in a separate network namespace.
+func control(logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return controlC
+}
+
+// controlC marks c as necessary to dial in a separate network namespace.
 //
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
-func control(network, address string, c syscall.RawConn) error {
+func controlC(network, address string, c syscall.RawConn) error {
 	if isLocalhost(address) {
 		// Don't bind to an interface for localhost connections.
 		return nil

net/netns/netns_test.go

@@ -25,7 +25,7 @@ func TestDial(t *testing.T) {
 	if !*extNetwork {
 		t.Skip("skipping test without --use-external-network")
 	}
-	d := NewDialer()
+	d := NewDialer(t.Logf)
 	c, err := d.Dial("tcp", "google.com:80")
 	if err != nil {
 		t.Fatal(err)

net/netns/netns_windows.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/types/logger"
 	"tailscale.com/util/endian"
 )
 
@@ -26,9 +27,13 @@ func interfaceIndex(iface *winipcfg.IPAdapterAddresses) uint32 {
 	return iface.IfIndex
 }
 
-// control binds c to the Windows interface that holds a default
+func control(logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return controlC
+}
+
+// controlC binds c to the Windows interface that holds a default
 // route, and is not the Tailscale WinTun interface.
-func control(network, address string, c syscall.RawConn) error {
+func controlC(network, address string, c syscall.RawConn) error {
 	if strings.HasPrefix(address, "127.") {
 		// Don't bind to an interface for localhost connections,
 		// otherwise we get:

net/portmapper/portmapper.go

@@ -251,7 +251,7 @@ func (c *Client) listenPacket(ctx context.Context, network, addr string) (net.Pa
 		var lc net.ListenConfig
 		return lc.ListenPacket(ctx, network, addr)
 	}
-	return netns.Listener().ListenPacket(ctx, network, addr)
+	return netns.Listener(c.logf).ListenPacket(ctx, network, addr)
 }
 
 func (c *Client) invalidateMappingsLocked(releaseOld bool) {

net/portmapper/upnp.go

@@ -219,7 +219,7 @@ func (c *Client) upnpHTTPClientLocked() *http.Client {
 	if c.uPnPHTTPClient == nil {
 		c.uPnPHTTPClient = &http.Client{
 			Transport: &http.Transport{
-				DialContext:     netns.NewDialer().DialContext,
+				DialContext:     netns.NewDialer(c.logf).DialContext,
 				IdleConnTimeout: 2 * time.Second, // LAN is cheap
 			},
 		}

net/tstun/wrap.go

@@ -7,7 +7,6 @@
 package tstun
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -17,16 +16,18 @@ import (
 	"sync/atomic"
 	"time"
 
+	"go4.org/mem"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/disco"
 	"tailscale.com/net/packet"
-	"tailscale.com/tailcfg"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/pad32"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -79,7 +80,7 @@ type Wrapper struct {
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 	destMACAtomic  atomic.Value // of [6]byte
-	discoKey       atomic.Value // of tailcfg.DiscoKey
+	discoKey       atomic.Value // of key.DiscoPublic
 
 	// buffer stores the oldest unconsumed packet from tdev.
 	// It is made a static buffer in order to avoid allocations.
@@ -204,7 +205,7 @@ func (t *Wrapper) SetDestIPActivityFuncs(m map[netaddr.IP]func()) {
 //
 // It is only used for filtering out bogus traffic when network
 // stack(s) get confused; see Issue 1526.
-func (t *Wrapper) SetDiscoKey(k tailcfg.DiscoKey) {
+func (t *Wrapper) SetDiscoKey(k key.DiscoPublic) {
 	t.discoKey.Store(k)
 }
 
@@ -216,12 +217,13 @@ func (t *Wrapper) isSelfDisco(p *packet.Parsed) bool {
 		return false
 	}
 	pkt := p.Payload()
-	discoSrc, ok := disco.Source(pkt)
+	discobs, ok := disco.Source(pkt)
 	if !ok {
 		return false
 	}
-	selfDiscoPub, ok := t.discoKey.Load().(tailcfg.DiscoKey)
-	return ok && bytes.Equal(selfDiscoPub[:], discoSrc)
+	discoSrc := key.DiscoPublicFromRaw32(mem.B(discobs))
+	selfDiscoPub, ok := t.discoKey.Load().(key.DiscoPublic)
+	return ok && selfDiscoPub == discoSrc
 }
 
 func (t *Wrapper) Close() error {
@@ -420,11 +422,13 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 	if p.IPProto == ipproto.UDP && // disco is over UDP; avoid isSelfDisco call for TCP/etc
 		t.isSelfDisco(p) {
 		t.logf("[unexpected] received self disco out packet over tstun; dropping")
+		metricPacketOutDropSelfDisco.Add(1)
 		return filter.DropSilently
 	}
 
 	if t.PreFilterOut != nil {
 		if res := t.PreFilterOut(p, t); res.IsDrop() {
+			// Handled by userspaceEngine.handleLocalPackets (quad-100 DNS primarily).
 			return res
 		}
 	}
@@ -436,6 +440,7 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 	}
 
 	if filt.RunOut(p, t.filterFlags) != filter.Accept {
+		metricPacketOutDropFilter.Add(1)
 		return filter.Drop
 	}
 
@@ -470,6 +475,8 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	if res.err != nil {
 		return 0, res.err
 	}
+
+	metricPacketOut.Add(1)
 	pkt := res.data
 	n := copy(buf[offset:], pkt)
 	// t.buffer has a fixed location in memory.
@@ -495,6 +502,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	if !isInjectedPacket && !t.disableFilter {
 		response := t.filterOut(p)
 		if response != filter.Accept {
+			metricPacketOutDrop.Add(1)
 			// Wireguard considers read errors fatal; pretend nothing was read
 			return 0, nil
 		}
@@ -528,6 +536,7 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 	if p.IPProto == ipproto.UDP && // disco is over UDP; avoid isSelfDisco call for TCP/etc
 		t.isSelfDisco(p) {
 		t.logf("[unexpected] received self disco in packet over tstun; dropping")
+		metricPacketInDropSelfDisco.Add(1)
 		return filter.DropSilently
 	}
 
@@ -557,6 +566,7 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 	}
 
 	if outcome != filter.Accept {
+		metricPacketInDropFilter.Add(1)
 
 		// Tell them, via TSMP, we're dropping them due to the ACL.
 		// Their host networking stack can translate this into ICMP
@@ -595,8 +605,10 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 // Write accepts an incoming packet. The packet begins at buf[offset:],
 // like wireguard-go/tun.Device.Write.
 func (t *Wrapper) Write(buf []byte, offset int) (int, error) {
+	metricPacketIn.Add(1)
 	if !t.disableFilter {
 		if t.filterIn(buf[offset:]) != filter.Accept {
+			metricPacketInDrop.Add(1)
 			// If we're not accepting the packet, lie to wireguard-go and pretend
 			// that everything is okay with a nil error, so wireguard-go
 			// doesn't log about this Write "failure".
@@ -720,3 +732,15 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 func (t *Wrapper) Unwrap() tun.Device {
 	return t.tdev
 }
+
+var (
+	metricPacketIn              = clientmetric.NewGauge("tstun_in_from_wg")
+	metricPacketInDrop          = clientmetric.NewGauge("tstun_in_from_wg_drop")
+	metricPacketInDropFilter    = clientmetric.NewGauge("tstun_in_from_wg_drop_filter")
+	metricPacketInDropSelfDisco = clientmetric.NewGauge("tstun_in_from_wg_drop_self_disco")
+
+	metricPacketOut              = clientmetric.NewGauge("tstun_out_to_wg")
+	metricPacketOutDrop          = clientmetric.NewGauge("tstun_out_to_wg_drop")
+	metricPacketOutDropFilter    = clientmetric.NewGauge("tstun_out_to_wg_drop_filter")
+	metricPacketOutDropSelfDisco = clientmetric.NewGauge("tstun_out_to_wg_drop_self_disco")
+)

net/tstun/wrap_test.go

@@ -13,14 +13,15 @@ import (
 	"testing"
 	"unsafe"
 
+	"go4.org/mem"
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/disco"
 	"tailscale.com/net/packet"
-	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
 )
@@ -493,7 +494,7 @@ func TestPeerAPIBypass(t *testing.T) {
 // Issue 1526: drop disco frames from ourselves.
 func TestFilterDiscoLoop(t *testing.T) {
 	var memLog tstest.MemLogger
-	discoPub := tailcfg.DiscoKey{1: 1, 2: 2}
+	discoPub := key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 31: 0}))
 	tw := &Wrapper{logf: memLog.Logf}
 	tw.SetDiscoKey(discoPub)
 	uh := packet.UDP4Header{
@@ -505,7 +506,8 @@ func TestFilterDiscoLoop(t *testing.T) {
 		SrcPort: 9,
 		DstPort: 10,
 	}
-	discoPayload := fmt.Sprintf("%s%s%s", disco.Magic, discoPub[:], [disco.NonceLen]byte{})
+	discobs := discoPub.Raw32()
+	discoPayload := fmt.Sprintf("%s%s%s", disco.Magic, discobs[:], [disco.NonceLen]byte{})
 	pkt := make([]byte, uh.Len()+len(discoPayload))
 	uh.Marshal(pkt)
 	copy(pkt[uh.Len():], discoPayload)

paths/paths_js.go

@@ -7,3 +7,5 @@ package paths
 func ensureStateDirPerms(dirPath string) error {
 	return nil
 }
+
+func LegacyStateFilePath() string { return "" }

safesocket/safesocket.go

@@ -13,6 +13,10 @@ import (
 	"time"
 )
 
+// WindowsLocalPort is the default localhost TCP port
+// used by safesocket on Windows.
+const WindowsLocalPort = 41112
+
 type closeable interface {
 	CloseRead() error
 	CloseWrite() error

safesocket/safesocket_js.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package safesocket
+
+import (
+	"net"
+
+	"github.com/akutz/memconn"
+)
+
+const memName = "Tailscale-IPN"
+
+func listen(path string, port uint16) (_ net.Listener, gotPort uint16, _ error) {
+	ln, err := memconn.Listen("memu", memName)
+	return ln, 1, err
+}
+
+func connect(path string, port uint16) (net.Conn, error) {
+	return memconn.Dial("memu", memName)
+}

safesocket/unixsocket.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !windows
-// +build !windows
+//go:build !windows && !js
+// +build !windows,!js
 
 package safesocket
 

scripts/check_license_headers.sh

@@ -38,6 +38,12 @@ for file in $(find $1 -name '*.go' -not -path '*/.git/*'); do
         $1/wgengine/router/ifconfig_windows.go)
             # WireGuard copyright.
         ;;
+		*_string.go)
+			# Generated file from go:generate stringer
+		;;
+		$1/control/noise/noiseexplorer_test.go)
+			# Noiseexplorer.com copyright.
+		;;
         *)
             header="$(head -3 $file)"
             if ! check_file "$header"; then

scripts/installer.sh

@@ -250,6 +250,9 @@ main() {
 				OS_UNSUPPORTED=1
 			fi
 			;;
+		fedora)
+			# All versions supported, no version checking required.
+			;;
 		arch)
 			# Rolling release, no version checking needed.
 			;;

tailcfg/tailcfg.go

@@ -14,7 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/key"
@@ -48,7 +47,8 @@ import (
 //    22: 2021-06-16: added MapResponse.DNSConfig.ExtraRecords
 //    23: 2021-08-25: DNSConfig.Routes values may be empty (for ExtraRecords support in 1.14.1+)
 //    24: 2021-09-18: MapResponse.Health from control to node; node shows in "tailscale status"
-const CurrentMapRequestVersion = 24
+//    25: 2021-11-01: MapResponse.Debug.Exit
+const CurrentMapRequestVersion = 25
 
 type StableID string
 
@@ -78,34 +78,6 @@ func (u StableNodeID) IsZero() bool {
 	return u == ""
 }
 
-// NodeKey is the WireGuard public key for a node.
-//
-// Deprecated: prefer to use key.NodePublic instead. If you must have
-// a NodeKey, use NodePublic.AsNodeKey.
-type NodeKey = key.NodeKey
-
-// NodeKeyFromNodePublic returns k converted to a NodeKey.
-//
-// Deprecated: exists only as a compatibility bridge while NodeKey
-// gets removed from the codebase. Do not introduce new uses that
-// aren't related to #3206.
-func NodeKeyFromNodePublic(k key.NodePublic) NodeKey {
-	return k.AsNodeKey()
-}
-
-// DiscoKey is the curve25519 public key for path discovery key.
-// It's never written to disk or reused between network start-ups.
-type DiscoKey [32]byte
-
-// DiscoKeyFromNodePublic returns k converted to a DiscoKey.
-//
-// Deprecated: exists only as a compatibility bridge while DiscoKey
-// gets removed from the codebase. Do not introduce new uses that
-// aren't related to #3206.
-func DiscoKeyFromDiscoPublic(k key.DiscoPublic) DiscoKey {
-	return k.Raw32()
-}
-
 // User is an IPN user.
 //
 // A user can have multiple logins associated with it (e.g. gmail and github oauth).
@@ -174,10 +146,10 @@ type Node struct {
 	// Sharer, if non-zero, is the user who shared this node, if different than User.
 	Sharer UserID `json:",omitempty"`
 
-	Key        NodeKey
+	Key        key.NodePublic
 	KeyExpiry  time.Time
 	Machine    key.MachinePublic
-	DiscoKey   DiscoKey
+	DiscoKey   key.DiscoPublic
 	Addresses  []netaddr.IPPrefix // IP addresses of this Node directly
 	AllowedIPs []netaddr.IPPrefix // range of IP addresses to route to this node
 	Endpoints  []string           `json:",omitempty"` // IP+port (public via STUN, and local LANs)
@@ -646,8 +618,8 @@ func (st SignatureType) String() string {
 type RegisterRequest struct {
 	_          structs.Incomparable
 	Version    int // currently 1
-	NodeKey    NodeKey
-	OldNodeKey NodeKey
+	NodeKey    key.NodePublic
+	OldNodeKey key.NodePublic
 	Auth       struct {
 		_ structs.Incomparable
 		// One of Provider/LoginName, Oauth2Token, or AuthKey is set.
@@ -764,8 +736,8 @@ type MapRequest struct {
 
 	Compress    string // "zstd" or "" (no compression)
 	KeepAlive   bool   // whether server should send keep-alives back to us
-	NodeKey     NodeKey
-	DiscoKey    DiscoKey
+	NodeKey     key.NodePublic
+	DiscoKey    key.DiscoPublic
 	IncludeIPv6 bool `json:",omitempty"` // include IPv6 endpoints in returned Node Endpoints (for Version 4 clients)
 	Stream      bool // if true, multiple MapResponse objects are returned
 	Hostinfo    *Hostinfo
@@ -1128,12 +1100,16 @@ type Debug struct {
 	// fixed port.
 	RandomizeClientPort bool `json:",omitempty"`
 
-	/// DisableUPnP is whether the client will attempt to perform a UPnP portmapping.
+	// DisableUPnP is whether the client will attempt to perform a UPnP portmapping.
 	// By default, we want to enable it to see if it works on more clients.
 	//
 	// If UPnP catastrophically fails for people, this should be set to True to kill
 	// new attempts at UPnP connections.
 	DisableUPnP opt.Bool `json:",omitempty"`
+
+	// Exit optionally specifies that the client should os.Exit
+	// with this code.
+	Exit *int `json:",omitempty"`
 }
 
 func appendKey(base []byte, prefix string, k [32]byte) []byte {
@@ -1148,25 +1124,6 @@ func keyMarshalText(prefix string, k [32]byte) []byte {
 	return appendKey(nil, prefix, k)
 }
 
-func (k DiscoKey) String() string { return fmt.Sprintf("discokey:%x", k[:]) }
-func (k DiscoKey) MarshalText() ([]byte, error) {
-	dk := key.DiscoPublicFromRaw32(mem.B(k[:]))
-	return dk.MarshalText()
-}
-func (k *DiscoKey) UnmarshalText(text []byte) error {
-	var dk key.DiscoPublic
-	if err := dk.UnmarshalText(text); err != nil {
-		return err
-	}
-	dk.AppendTo(k[:0])
-	return nil
-}
-func (k DiscoKey) ShortString() string      { return fmt.Sprintf("d:%x", k[:8]) }
-func (k DiscoKey) AppendTo(b []byte) []byte { return appendKey(b, "discokey:", k) }
-
-// IsZero reports whether k is the zero value.
-func (k DiscoKey) IsZero() bool { return k == DiscoKey{} }
-
 func (id ID) String() string      { return fmt.Sprintf("id:%x", int64(id)) }
 func (id UserID) String() string  { return fmt.Sprintf("userid:%x", int64(id)) }
 func (id LoginID) String() string { return fmt.Sprintf("loginid:%x", int64(id)) }
@@ -1288,7 +1245,7 @@ type SetDNSRequest struct {
 	Version int
 
 	// NodeKey is the client's current node key.
-	NodeKey NodeKey
+	NodeKey key.NodePublic
 
 	// Name is the domain name for which to create a record.
 	// For ACME DNS-01 challenges, it should be one of the domains

tailcfg/tailcfg_clone.go

@@ -72,10 +72,10 @@ var _NodeCloneNeedsRegeneration = Node(struct {
 	Name                    string
 	User                    UserID
 	Sharer                  UserID
-	Key                     NodeKey
+	Key                     key.NodePublic
 	KeyExpiry               time.Time
 	Machine                 key.MachinePublic
-	DiscoKey                DiscoKey
+	DiscoKey                key.DiscoPublic
 	Addresses               []netaddr.IPPrefix
 	AllowedIPs              []netaddr.IPPrefix
 	Endpoints               []string

tailcfg/tailcfg_test.go

@@ -264,13 +264,13 @@ func TestNodeEqual(t *testing.T) {
 			true,
 		},
 		{
-			&Node{Key: n1.AsNodeKey()},
-			&Node{Key: key.NewNode().Public().AsNodeKey()},
+			&Node{Key: n1},
+			&Node{Key: key.NewNode().Public()},
 			false,
 		},
 		{
-			&Node{Key: n1.AsNodeKey()},
-			&Node{Key: n1.AsNodeKey()},
+			&Node{Key: n1},
+			&Node{Key: n1},
 			true,
 		},
 		{
@@ -407,14 +407,6 @@ func TestNetInfoFields(t *testing.T) {
 	}
 }
 
-func TestDiscoKeyMarshal(t *testing.T) {
-	var k1, k2 DiscoKey
-	for i := range k1 {
-		k1[i] = byte(i)
-	}
-	testKey(t, "discokey:", k1, &k2)
-}
-
 type keyIn interface {
 	String() string
 	MarshalText() ([]byte, error)
@@ -542,15 +534,6 @@ func TestAppendKeyAllocs(t *testing.T) {
 	}
 }
 
-func TestDiscoKeyAppend(t *testing.T) {
-	d := DiscoKey{1: 1, 2: 2}
-	got := string(d.AppendTo([]byte("foo")))
-	want := "foodiscokey:0001020000000000000000000000000000000000000000000000000000000000"
-	if got != want {
-		t.Errorf("got %q; want %q", got, want)
-	}
-}
-
 func TestRegisterRequestNilClone(t *testing.T) {
 	var nilReq *RegisterRequest
 	got := nilReq.Clone()

tsconst/interface.go

@@ -7,5 +7,6 @@
 package tsconst
 
 // WintunInterfaceDesc is the description attached to Tailscale
-// interfaces on Windows. This is set by our modified WinTun driver.
+// interfaces on Windows. This is set by the WinTun driver.
 const WintunInterfaceDesc = "Tailscale Tunnel"
+const WintunInterfaceDesc0_14 = "Wintun Userspace Tunnel"

tsnet/tsnet.go

@@ -127,10 +127,11 @@ func (s *Server) start() error {
 		return fmt.Errorf("%T is not a wgengine.InternalsGetter", eng)
 	}
 
-	ns, err := netstack.Create(logf, tunDev, eng, magicConn, false)
+	ns, err := netstack.Create(logf, tunDev, eng, magicConn)
 	if err != nil {
 		return fmt.Errorf("netstack.Create: %w", err)
 	}
+	ns.ProcessLocalIPs = true
 	ns.ForwardTCPIn = s.forwardTCP
 	if err := ns.Start(); err != nil {
 		return fmt.Errorf("failed to start netstack: %w", err)
@@ -147,6 +148,7 @@ func (s *Server) start() error {
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	lb.SetVarRoot(s.dir)
 	s.lb = lb
 	lb.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)

tstest/archtest/archtest_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package archtest
+
+import (
+	"runtime"
+	"testing"
+
+	"inet.af/netstack/atomicbitops"
+)
+
+// tests netstack's AlignedAtomicInt64.
+func TestAlignedAtomicInt64(t *testing.T) {
+	type T struct {
+		A atomicbitops.AlignedAtomicInt64
+		x int32
+		B atomicbitops.AlignedAtomicInt64
+	}
+
+	t.Logf("I am %v/%v\n", runtime.GOOS, runtime.GOARCH)
+	var x T
+	x.A.Store(1)
+	x.B.Store(2)
+	if got, want := x.A.Load(), int64(1); got != want {
+		t.Errorf("A = %v; want %v", got, want)
+	}
+	if got, want := x.B.Load(), int64(2); got != want {
+		t.Errorf("A = %v; want %v", got, want)
+	}
+}

tstest/archtest/qemu_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && amd64 && !race
+// +build linux,amd64,!race
+
+package archtest
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestInQemu(t *testing.T) {
+	t.Parallel()
+	type Arch struct {
+		Goarch string // GOARCH value
+		Qarch  string // qemu name
+	}
+	arches := []Arch{
+		{"arm", "arm"},
+		{"arm64", "aarch64"},
+		{"mips", "mips"},
+		{"mipsle", "mipsel"},
+		{"mips64", "mips64"},
+		{"mips64le", "mips64el"},
+		{"386", "386"},
+	}
+	inCI := os.Getenv("CI") == "true"
+	for _, arch := range arches {
+		arch := arch
+		t.Run(arch.Goarch, func(t *testing.T) {
+			t.Parallel()
+			qemuUser := "qemu-" + arch.Qarch
+			execVia := qemuUser
+			if arch.Goarch == "386" {
+				execVia = "" // amd64 can run it fine
+			} else {
+				look, err := exec.LookPath(qemuUser)
+				if err != nil {
+					if inCI {
+						t.Fatalf("in CI and qemu not available: %v", err)
+					}
+					t.Skipf("%s not found; skipping test. error was: %v", qemuUser, err)
+				}
+				t.Logf("using %v", look)
+			}
+			cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"),
+				"test",
+				"--exec="+execVia,
+				"-v",
+				"tailscale.com/tstest/archtest",
+			)
+			cmd.Env = append(os.Environ(), "GOARCH="+arch.Goarch)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				if strings.Contains(string(out), "fatal error: sigaction failed") && !inCI {
+					t.Skip("skipping; qemu too old. use 5.x.")
+				}
+				t.Errorf("failed: %s", out)
+			}
+			sub := fmt.Sprintf("I am linux/%s", arch.Goarch)
+			if !bytes.Contains(out, []byte(sub)) {
+				t.Errorf("output didn't contain %q: %s", sub, out)
+			}
+		})
+	}
+}

tstest/integration/gen_deps.go

@@ -15,6 +15,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"strings"
 )
 
 func main() {
@@ -52,6 +53,10 @@ import (
 	// process and can cache a prior success when a dependency changes.
 `)
 	for _, dep := range x.Imports {
+		if !strings.Contains(dep, ".") {
+			// Omit stanard library deps.
+			continue
+		}
 		fmt.Fprintf(&out, "\t_ %q\n", dep)
 	}
 	fmt.Fprintf(&out, ")\n")

tstest/integration/integration_test.go

@@ -84,6 +84,47 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+func TestOneNodeExpiredKey(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp()
+	n1.AwaitRunning(t)
+
+	nodes := env.Control.AllNodes()
+	if len(nodes) != 1 {
+		t.Fatalf("expected 1 node, got %d nodes", len(nodes))
+	}
+
+	nodeKey := nodes[0].Key
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	env.Control.SetExpireAllNodes(true)
+	n1.AwaitNeedsLogin(t)
+	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
+	if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	env.Control.SetExpireAllNodes(false)
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+}
+
 func TestCollectPanic(t *testing.T) {
 	t.Parallel()
 	bins := BuildTestBinaries(t)
@@ -713,7 +754,7 @@ func (n *testNode) MustDown() {
 // over its localhost IPC mechanism. (Unix socket, etc)
 func (n *testNode) AwaitListening(t testing.TB) {
 	if err := tstest.WaitFor(20*time.Second, func() (err error) {
-		c, err := safesocket.Connect(n.sockFile, 41112)
+		c, err := safesocket.Connect(n.sockFile, safesocket.WindowsLocalPort)
 		if err != nil {
 			return err
 		}
@@ -780,6 +821,23 @@ func (n *testNode) AwaitRunning(t testing.TB) {
 	}
 }
 
+// AwaitNeedsLogin waits for n to reach the IPN state "NeedsLogin".
+func (n *testNode) AwaitNeedsLogin(t testing.TB) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		st, err := n.Status()
+		if err != nil {
+			return err
+		}
+		if st.BackendState != "NeedsLogin" {
+			return fmt.Errorf("in state %q", st.BackendState)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failure/timeout waiting for transition to NeedsLogin status: %v", err)
+	}
+}
+
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {

tstest/integration/tailscaled_deps_test_darwin.go

@@ -11,37 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/exec"
-	_ "os/signal"
-	_ "path/filepath"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -50,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -61,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -11,35 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -48,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -59,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_linux.go

@@ -11,35 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -48,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -59,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -11,35 +11,13 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/chirp"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
 	_ "tailscale.com/net/netns"
@@ -48,10 +26,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/version"
 	_ "tailscale.com/version/distro"
@@ -59,5 +40,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/tailscaled_deps_test_windows.go

@@ -11,38 +11,16 @@ import (
 	// Otherwise cmd/go never sees that we depend on these packages'
 	// transitive deps when we run "go install tailscaled" in a child
 	// process and can cache a prior success when a dependency changes.
-	_ "context"
-	_ "crypto/tls"
-	_ "encoding/json"
-	_ "errors"
-	_ "flag"
-	_ "fmt"
-	_ "github.com/go-multierror/multierror"
 	_ "golang.org/x/sys/windows"
 	_ "golang.org/x/sys/windows/svc"
 	_ "golang.org/x/sys/windows/svc/mgr"
 	_ "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	_ "inet.af/netaddr"
-	_ "io"
-	_ "io/ioutil"
-	_ "log"
-	_ "net"
-	_ "net/http"
-	_ "net/http/httptrace"
-	_ "net/http/httputil"
-	_ "net/http/pprof"
-	_ "net/url"
-	_ "os"
-	_ "os/signal"
-	_ "runtime"
-	_ "runtime/debug"
-	_ "strconv"
-	_ "strings"
-	_ "syscall"
 	_ "tailscale.com/derp/derphttp"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
 	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail"
 	_ "tailscale.com/logtail/backoff"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
@@ -52,10 +30,13 @@ import (
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"
 	_ "tailscale.com/paths"
+	_ "tailscale.com/safesocket"
 	_ "tailscale.com/tailcfg"
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/clientmetric"
+	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"
 	_ "tailscale.com/util/winutil"
 	_ "tailscale.com/version"
@@ -65,5 +46,4 @@ import (
 	_ "tailscale.com/wgengine/monitor"
 	_ "tailscale.com/wgengine/netstack"
 	_ "tailscale.com/wgengine/router"
-	_ "time"
 )

tstest/integration/testcontrol/testcontrol.go

@@ -58,13 +58,14 @@ type Server struct {
 	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        key.MachinePublic
 	privKey       key.ControlPrivate // not strictly needed vs. MachinePrivate, but handy to test type interactions.
-	nodes         map[tailcfg.NodeKey]*tailcfg.Node
-	users         map[tailcfg.NodeKey]*tailcfg.User
-	logins        map[tailcfg.NodeKey]*tailcfg.Login
+	nodes         map[key.NodePublic]*tailcfg.Node
+	users         map[key.NodePublic]*tailcfg.User
+	logins        map[key.NodePublic]*tailcfg.Login
 	updates       map[tailcfg.NodeID]chan updateType
 	authPath      map[string]*AuthPath
-	nodeKeyAuthed map[tailcfg.NodeKey]bool // key => true once authenticated
-	pingReqsToAdd map[tailcfg.NodeKey]*tailcfg.PingRequest
+	nodeKeyAuthed map[key.NodePublic]bool // key => true once authenticated
+	pingReqsToAdd map[key.NodePublic]*tailcfg.PingRequest
+	allExpired    bool // All nodes will be told their node key is expired.
 }
 
 // BaseURL returns the server's base URL, without trailing slash.
@@ -103,7 +104,7 @@ func (s *Server) condLocked() *sync.Cond {
 
 // AwaitNodeInMapRequest waits for node k to be stuck in a map poll.
 // It returns an error if and only if the context is done first.
-func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k tailcfg.NodeKey) error {
+func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k key.NodePublic) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	cond := s.condLocked()
@@ -135,11 +136,11 @@ func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k tailcfg.NodeKey) e
 
 // AddPingRequest sends the ping pr to nodeKeyDst. It reports whether it did so. That is,
 // it reports whether nodeKeyDst was connected.
-func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequest) bool {
+func (s *Server) AddPingRequest(nodeKeyDst key.NodePublic, pr *tailcfg.PingRequest) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.pingReqsToAdd == nil {
-		s.pingReqsToAdd = map[tailcfg.NodeKey]*tailcfg.PingRequest{}
+		s.pingReqsToAdd = map[key.NodePublic]*tailcfg.PingRequest{}
 	}
 	// Now send the update to the channel
 	node := s.nodeLocked(nodeKeyDst)
@@ -153,8 +154,20 @@ func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequ
 	return sendUpdate(oldUpdatesCh, updateDebugInjection)
 }
 
+// Mark the Node key of every node as expired
+func (s *Server) SetExpireAllNodes(expired bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.allExpired = expired
+
+	for _, node := range s.nodes {
+		sendUpdate(s.updates[node.ID], updateSelfChanged)
+	}
+}
+
 type AuthPath struct {
-	nodeKey tailcfg.NodeKey
+	nodeKey key.NodePublic
 
 	closeOnce sync.Once
 	ch        chan struct{}
@@ -254,7 +267,7 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
 }
 
 // Node returns the node for nodeKey. It's always nil or cloned memory.
-func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
+func (s *Server) Node(nodeKey key.NodePublic) *tailcfg.Node {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	return s.nodeLocked(nodeKey)
@@ -263,7 +276,7 @@ func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 // nodeLocked returns the node for nodeKey. It's always nil or cloned memory.
 //
 // s.mu must be held.
-func (s *Server) nodeLocked(nodeKey tailcfg.NodeKey) *tailcfg.Node {
+func (s *Server) nodeLocked(nodeKey key.NodePublic) *tailcfg.Node {
 	return s.nodes[nodeKey].Clone()
 }
 
@@ -272,13 +285,14 @@ func (s *Server) AddFakeNode() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.nodes == nil {
-		s.nodes = make(map[tailcfg.NodeKey]*tailcfg.Node)
+		s.nodes = make(map[key.NodePublic]*tailcfg.Node)
 	}
-	nk := tailcfg.NodeKeyFromNodePublic(key.NewNode().Public())
+	nk := key.NewNode().Public()
 	mk := key.NewMachine().Public()
-	dk := tailcfg.DiscoKeyFromDiscoPublic(key.NewDisco().Public())
-	id := int64(binary.LittleEndian.Uint64(nk[:]))
-	ip := netaddr.IPv4(nk[0], nk[1], nk[2], nk[3])
+	dk := key.NewDisco().Public()
+	r := nk.Raw32()
+	id := int64(binary.LittleEndian.Uint64(r[:]))
+	ip := netaddr.IPv4(r[0], r[1], r[2], r[3])
 	addr := netaddr.IPPrefixFrom(ip, 32)
 	s.nodes[nk] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(id),
@@ -306,14 +320,14 @@ func (s *Server) AllNodes() (nodes []*tailcfg.Node) {
 	return nodes
 }
 
-func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login) {
+func (s *Server) getUser(nodeKey key.NodePublic) (*tailcfg.User, *tailcfg.Login) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.users == nil {
-		s.users = map[tailcfg.NodeKey]*tailcfg.User{}
+		s.users = map[key.NodePublic]*tailcfg.User{}
 	}
 	if s.logins == nil {
-		s.logins = map[tailcfg.NodeKey]*tailcfg.Login{}
+		s.logins = map[key.NodePublic]*tailcfg.Login{}
 	}
 	if u, ok := s.users[nodeKey]; ok {
 		return u, s.logins[nodeKey]
@@ -353,7 +367,7 @@ func (s *Server) authPathDone(authPath string) <-chan struct{} {
 	return nil
 }
 
-func (s *Server) addAuthPath(authPath string, nodeKey tailcfg.NodeKey) {
+func (s *Server) addAuthPath(authPath string, nodeKey key.NodePublic) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.authPath == nil {
@@ -385,7 +399,7 @@ func (s *Server) CompleteAuth(authPathOrURL string) bool {
 		panic("zero AuthPath.NodeKey")
 	}
 	if s.nodeKeyAuthed == nil {
-		s.nodeKeyAuthed = map[tailcfg.NodeKey]bool{}
+		s.nodeKeyAuthed = map[key.NodePublic]bool{}
 	}
 	s.nodeKeyAuthed[ap.nodeKey] = true
 	ap.CompleteSuccessfully()
@@ -433,10 +447,12 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		// some follow-ups? For now all are successes.
 	}
 
-	user, login := s.getUser(req.NodeKey)
+	nk := req.NodeKey
+
+	user, login := s.getUser(nk)
 	s.mu.Lock()
 	if s.nodes == nil {
-		s.nodes = map[tailcfg.NodeKey]*tailcfg.Node{}
+		s.nodes = map[key.NodePublic]*tailcfg.Node{}
 	}
 
 	machineAuthorized := true // TODO: add Server.RequireMachineAuth
@@ -449,7 +465,7 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		v6Prefix,
 	}
 
-	s.nodes[req.NodeKey] = &tailcfg.Node{
+	s.nodes[nk] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(user.ID),
 		StableID:          tailcfg.StableNodeID(fmt.Sprintf("TESTCTRL%08x", int(user.ID))),
 		User:              user.ID,
@@ -461,9 +477,10 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		Hostinfo:          *req.Hostinfo,
 	}
 	requireAuth := s.RequireAuth
-	if requireAuth && s.nodeKeyAuthed[req.NodeKey] {
+	if requireAuth && s.nodeKeyAuthed[nk] {
 		requireAuth = false
 	}
+	allExpired := s.allExpired
 	s.mu.Unlock()
 
 	authURL := ""
@@ -471,14 +488,14 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		randHex := make([]byte, 10)
 		crand.Read(randHex)
 		authPath := fmt.Sprintf("/auth/%x", randHex)
-		s.addAuthPath(authPath, req.NodeKey)
+		s.addAuthPath(authPath, nk)
 		authURL = s.BaseURL() + authPath
 	}
 
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
 		User:              *user,
 		Login:             *login,
-		NodeKeyExpired:    false,
+		NodeKeyExpired:    allExpired,
 		MachineAuthorized: machineAuthorized,
 		AuthURL:           authURL,
 	})
@@ -639,6 +656,13 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey key.Machi
 		if res == nil {
 			return // done
 		}
+
+		s.mu.Lock()
+		allExpired := s.allExpired
+		s.mu.Unlock()
+		if allExpired {
+			res.Node.KeyExpiry = time.Now().Add(-1 * time.Minute)
+		}
 		// TODO: add minner if/when needed
 		resBytes, err := json.Marshal(res)
 		if err != nil {
@@ -690,12 +714,13 @@ var keepAliveMsg = &struct {
 //
 // No updates to s are done here.
 func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, err error) {
-	node := s.Node(req.NodeKey)
+	nk := req.NodeKey
+	node := s.Node(nk)
 	if node == nil {
 		// node key rotated away (once test server supports that)
 		return nil, nil
 	}
-	user, _ := s.getUser(req.NodeKey)
+	user, _ := s.getUser(nk)
 	res = &tailcfg.MapResponse{
 		Node:            node,
 		DERPMap:         s.DERPMap,
@@ -727,9 +752,9 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 
 	// Consume the PingRequest while protected by mutex if it exists
 	s.mu.Lock()
-	if pr, ok := s.pingReqsToAdd[node.Key]; ok {
+	if pr, ok := s.pingReqsToAdd[nk]; ok {
 		res.PingRequest = pr
-		delete(s.pingReqsToAdd, node.Key)
+		delete(s.pingReqsToAdd, nk)
 	}
 	s.mu.Unlock()
 	return res, nil

types/key/disco.go

@@ -21,6 +21,10 @@ const (
 	// This prefix is used in the control protocol, so cannot be
 	// changed.
 	discoPublicHexPrefix = "discokey:"
+
+	// DiscoPublicRawLen is the length in bytes of a DiscoPublic, when
+	// serialized with AppendTo, Raw32 or WriteRawWithoutAllocating.
+	DiscoPublicRawLen = 32
 )
 
 // DiscoPrivate is a disco key, used for peer-to-peer path discovery.
@@ -115,12 +119,6 @@ func (k DiscoPublic) AppendTo(buf []byte) []byte {
 	return append(buf, k.k[:]...)
 }
 
-// RawLen returns the length of k when to the format handled by
-// ReadRawWithoutAllocating and WriteRawWithoutAllocating.
-func (k DiscoPublic) RawLen() int {
-	return 32
-}
-
 // String returns the output of MarshalText as a string.
 func (k DiscoPublic) String() string {
 	bs, err := k.MarshalText()

types/key/machine.go

@@ -77,6 +77,19 @@ func (k *MachinePrivate) UnmarshalText(b []byte) error {
 	return parseHex(k.k[:], mem.B(b), mem.S(machinePrivateHexPrefix))
 }
 
+// UntypedBytes returns k, encoded as an untyped 64-character hex
+// string.
+//
+// Deprecated: this function is risky to use, because it produces
+// serialized values that do not identify themselves as a
+// MachinePrivate, allowing other code to potentially parse it back in
+// as the wrong key type. For new uses that don't require this
+// specific raw byte serialization, please use
+// MarshalText/UnmarshalText.
+func (k MachinePrivate) UntypedBytes() []byte {
+	return append([]byte(nil), k.k[:]...)
+}
+
 // SealTo wraps cleartext into a NaCl box (see
 // golang.org/x/crypto/nacl) to p, authenticated from k, using a
 // random nonce.
@@ -112,6 +125,19 @@ type MachinePublic struct {
 	k [32]byte
 }
 
+// MachinePublicFromRaw32 parses a 32-byte raw value as a MachinePublic.
+//
+// This should be used only when deserializing a MachinePublic from a
+// binary protocol.
+func MachinePublicFromRaw32(raw mem.RO) MachinePublic {
+	if raw.Len() != 32 {
+		panic("input has wrong size")
+	}
+	var ret MachinePublic
+	raw.Copy(ret.k[:])
+	return ret
+}
+
 // ParseMachinePublicUntyped parses an untyped 64-character hex value
 // as a MachinePublic.
 //
@@ -153,6 +179,19 @@ func (k MachinePublic) UntypedHexString() string {
 	return hex.EncodeToString(k.k[:])
 }
 
+// UntypedBytes returns k, encoded as an untyped 64-character hex
+// string.
+//
+// Deprecated: this function is risky to use, because it produces
+// serialized values that do not identify themselves as a
+// MachinePublic, allowing other code to potentially parse it back in
+// as the wrong key type. For new uses that don't require this
+// specific raw byte serialization, please use
+// MarshalText/UnmarshalText.
+func (k MachinePublic) UntypedBytes() []byte {
+	return append([]byte(nil), k.k[:]...)
+}
+
 // String returns the output of MarshalText as a string.
 func (k MachinePublic) String() string {
 	bs, err := k.MarshalText()

types/key/node.go

@@ -33,6 +33,10 @@ const (
 	// This prefix is used in the control protocol, so cannot be
 	// changed.
 	nodePublicHexPrefix = "nodekey:"
+
+	// NodePublicRawLen is the length in bytes of a NodePublic, when
+	// serialized with AppendTo, Raw32 or WriteRawWithoutAllocating.
+	NodePublicRawLen = 32
 )
 
 // NodePrivate is a node key, used for WireGuard tunnels and
@@ -190,12 +194,6 @@ func (k NodePublic) AppendTo(buf []byte) []byte {
 	return append(buf, k.k[:]...)
 }
 
-// RawLen returns the length of k when to the format handled by
-// ReadRawWithoutAllocating and WriteRawWithoutAllocating.
-func (k NodePublic) RawLen() int {
-	return 32
-}
-
 // ReadRawWithoutAllocating initializes k with bytes read from br.
 // The reading is done ~4x slower than io.ReadFull, but in exchange is
 // allocation-free.
@@ -307,10 +305,3 @@ func (k NodePublic) WireGuardGoString() string {
 	b[second+3] = b64((k.k[31] << 2) & 63)
 	return string(b)
 }
-
-// AsNodeKey returns k converted to a NodeKey.
-//
-// Cross-compatibility shim as part of #3206.
-func (k NodePublic) AsNodeKey() NodeKey {
-	return k.Raw32()
-}

types/key/node_legacy.go

@@ -1,28 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package key
-
-import (
-	"go4.org/mem"
-)
-
-// NodeKey is the legacy form of NodePublic.
-// See #3206 for removal effort.
-type NodeKey [32]byte
-
-func (k NodeKey) ShortString() string          { return k.AsNodePublic().ShortString() }
-func (k NodeKey) String() string               { return k.AsNodePublic().String() }
-func (k NodeKey) MarshalText() ([]byte, error) { return k.AsNodePublic().MarshalText() }
-func (k NodeKey) AsNodePublic() NodePublic     { return NodePublicFromRaw32(mem.B(k[:])) }
-func (k NodeKey) IsZero() bool                 { return k == NodeKey{} }
-
-func (k *NodeKey) UnmarshalText(text []byte) error {
-	var nk NodePublic
-	if err := nk.UnmarshalText(text); err != nil {
-		return err
-	}
-	*k = nk.AsNodeKey()
-	return nil
-}

types/key/node_legacy_test.go

@@ -1,75 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package key
-
-import (
-	"bytes"
-	"encoding/json"
-	"strings"
-	"testing"
-)
-
-func TestNodeKeyMarshal(t *testing.T) {
-	var k1, k2 NodeKey
-	for i := range k1 {
-		k1[i] = byte(i)
-	}
-
-	const prefix = "nodekey:"
-	got, err := k1.MarshalText()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := k2.UnmarshalText(got); err != nil {
-		t.Fatal(err)
-	}
-	if s := k1.String(); string(got) != s {
-		t.Errorf("MarshalText = %q != String %q", got, s)
-	}
-	if !strings.HasPrefix(string(got), prefix) {
-		t.Errorf("%q didn't start with prefix %q", got, prefix)
-	}
-	if k2 != k1 {
-		t.Errorf("mismatch after unmarshal")
-	}
-}
-
-func TestNodeKeyRoundTrip(t *testing.T) {
-	serialized := `{
-      "Pub":"nodekey:50d20b455ecf12bc453f83c2cfdb2a24925d06cf2598dcaa54e91af82ce9f765"
-    }`
-
-	// Carefully check that the expected serialized data decodes and
-	// re-encodes to the expected keys. These types are serialized to
-	// disk all over the place and need to be stable.
-	pub := NodeKey{
-		0x50, 0xd2, 0xb, 0x45, 0x5e, 0xcf, 0x12, 0xbc, 0x45, 0x3f, 0x83,
-		0xc2, 0xcf, 0xdb, 0x2a, 0x24, 0x92, 0x5d, 0x6, 0xcf, 0x25, 0x98,
-		0xdc, 0xaa, 0x54, 0xe9, 0x1a, 0xf8, 0x2c, 0xe9, 0xf7, 0x65,
-	}
-
-	type key struct {
-		Pub NodeKey
-	}
-
-	var a key
-	if err := json.Unmarshal([]byte(serialized), &a); err != nil {
-		t.Fatal(err)
-	}
-	if a.Pub != pub {
-		t.Errorf("wrong deserialization of public key, got %#v want %#v", a.Pub, pub)
-	}
-
-	bs, err := json.MarshalIndent(a, "", "  ")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var b bytes.Buffer
-	json.Indent(&b, []byte(serialized), "", "  ")
-	if got, want := string(bs), b.String(); got != want {
-		t.Error("json serialization doesn't roundtrip")
-	}
-}

types/netmap/netmap.go

@@ -26,7 +26,7 @@ type NetworkMap struct {
 	// Core networking
 
 	SelfNode   *tailcfg.Node
-	NodeKey    tailcfg.NodeKey
+	NodeKey    key.NodePublic
 	PrivateKey key.NodePrivate
 	Expiry     time.Time
 	// Name is the DNS name assigned to this node.

types/netmap/netmap_test.go

@@ -8,24 +8,30 @@ import (
 	"encoding/hex"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
-func testNodeKey(b byte) (ret tailcfg.NodeKey) {
-	for i := range ret {
-		ret[i] = b
+func testNodeKey(b byte) (ret key.NodePublic) {
+	var bs [key.NodePublicRawLen]byte
+	for i := range bs {
+		bs[i] = b
 	}
-	return
+	return key.NodePublicFromRaw32(mem.B(bs[:]))
 }
 
-func testDiscoKey(hexPrefix string) (ret tailcfg.DiscoKey) {
+func testDiscoKey(hexPrefix string) (ret key.DiscoPublic) {
 	b, err := hex.DecodeString(hexPrefix)
 	if err != nil {
 		panic(err)
 	}
-	copy(ret[:], b)
-	return
+	// this function is used with short hexes, so zero-extend the raw
+	// value.
+	var bs [32]byte
+	copy(bs[:], b)
+	return key.DiscoPublicFromRaw32(mem.B(bs[:]))
 }
 
 func TestNetworkMapConcise(t *testing.T) {

util/clientmetric/clientmetric.go

@@ -0,0 +1,306 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package clientmetric provides client-side metrics whose values
+// get occasionally logged.
+package clientmetric
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	mu          sync.Mutex // guards vars in this block
+	metrics     = map[string]*Metric{}
+	numWireID   int         // how many wireIDs have been allocated
+	lastDelta   time.Time   // time of last call to EncodeLogTailMetricsDelta
+	sortedDirty bool        // whether sorted needs to be rebuilt
+	sorted      []*Metric   // by name
+	lastLogVal  []scanEntry // by Metric.regIdx
+	unsorted    []*Metric   // by Metric.regIdx
+
+	// valFreeList is a set of free contiguous int64s whose
+	// element addresses get assigned to Metric.v.
+	// Any memory address in len(valFreeList) is free for use.
+	// They're contiguous to reduce cache churn during diff scans.
+	// When out of length, a new backing array is made.
+	valFreeList []int64
+)
+
+// scanEntry contains the minimal data needed for quickly scanning
+// memory for changed values. It's small to reduce memory pressure.
+type scanEntry struct {
+	v          *int64 // Metric.v
+	lastLogged int64  // last logged value
+}
+
+// Type is a metric type: counter or gauge.
+type Type uint8
+
+const (
+	TypeGauge Type = iota
+	TypeCounter
+)
+
+// Metric is an integer metric value that's tracked over time.
+//
+// It's safe for concurrent use.
+type Metric struct {
+	v      *int64 // atomic; the metric value
+	regIdx int    // index into lastLogVal and unsorted
+	name   string
+	typ    Type
+
+	// The following fields are owned by the package-level 'mu':
+
+	// wireID is the lazily-allocated "wire ID". Until a metric is encoded
+	// in the logs (by EncodeLogTailMetricsDelta), it has no wireID. This
+	// ensures that unused metrics don't waste valuable low numbers, which
+	// encode with varints with fewer bytes.
+	wireID int
+
+	// lastNamed is the last time the name of this metric was
+	// written on the wire.
+	lastNamed time.Time
+}
+
+func (m *Metric) Name() string { return m.name }
+func (m *Metric) Value() int64 { return atomic.LoadInt64(m.v) }
+func (m *Metric) Type() Type   { return m.typ }
+
+// Add increments m's value by n.
+//
+// If m is of type counter, n should not be negative.
+func (m *Metric) Add(n int64) {
+	atomic.AddInt64(m.v, n)
+}
+
+// Set sets m's value to v.
+//
+// If m is of type counter, Set should not be used.
+func (m *Metric) Set(v int64) {
+	atomic.StoreInt64(m.v, v)
+}
+
+// Publish registers a metric in the global map.
+// It panics if the name is a duplicate anywhere in the process.
+func (m *Metric) Publish() {
+	mu.Lock()
+	defer mu.Unlock()
+	if m.name == "" {
+		panic("unnamed Metric")
+	}
+	if _, dup := metrics[m.name]; dup {
+		panic("duplicate metric " + m.name)
+	}
+	metrics[m.name] = m
+	sortedDirty = true
+
+	if len(valFreeList) == 0 {
+		valFreeList = make([]int64, 256)
+	}
+	m.v = &valFreeList[0]
+	valFreeList = valFreeList[1:]
+
+	m.regIdx = len(unsorted)
+	unsorted = append(unsorted, m)
+	lastLogVal = append(lastLogVal, scanEntry{v: m.v})
+}
+
+// Metrics returns the sorted list of metrics.
+//
+// The returned slice should not be mutated.
+func Metrics() []*Metric {
+	mu.Lock()
+	defer mu.Unlock()
+	if sortedDirty {
+		sortedDirty = false
+		sorted = make([]*Metric, 0, len(metrics))
+		for _, m := range metrics {
+			sorted = append(sorted, m)
+		}
+		sort.Slice(sorted, func(i, j int) bool {
+			return sorted[i].name < sorted[j].name
+		})
+	}
+	return sorted
+}
+
+// NewUnpublished initializes a new Metric without calling Publish on
+// it.
+func NewUnpublished(name string, typ Type) *Metric {
+	if i := strings.IndexFunc(name, isIllegalMetricRune); name == "" || i != -1 {
+		panic(fmt.Sprintf("illegal metric name %q (index %v)", name, i))
+	}
+	return &Metric{
+		name: name,
+		typ:  typ,
+	}
+}
+
+func isIllegalMetricRune(r rune) bool {
+	return !(r >= 'a' && r <= 'z' ||
+		r >= 'A' && r <= 'Z' ||
+		r >= '0' && r <= '9' ||
+		r == '_')
+}
+
+// NewCounter returns a new metric that can only increment.
+func NewCounter(name string) *Metric {
+	m := NewUnpublished(name, TypeCounter)
+	m.Publish()
+	return m
+}
+
+// NewGauge returns a new metric that can both increment and decrement.
+func NewGauge(name string) *Metric {
+	m := NewUnpublished(name, TypeGauge)
+	m.Publish()
+	return m
+}
+
+// WritePrometheusExpositionFormat writes all client metrics to w in
+// the Prometheus text-based exposition format.
+//
+// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md
+func WritePrometheusExpositionFormat(w io.Writer) {
+	for _, m := range Metrics() {
+		switch m.Type() {
+		case TypeGauge:
+			fmt.Fprintf(w, "# TYPE %s gauge\n", m.Name())
+		case TypeCounter:
+			fmt.Fprintf(w, "# TYPE %s counter\n", m.Name())
+		}
+		fmt.Fprintf(w, "%s %v\n", m.Name(), m.Value())
+	}
+}
+
+const (
+	// metricLogNameFrequency is how often a metric's name=>id
+	// mapping is redundantly put in the logs. In other words,
+	// this is how how far in the logs you need to fetch from a
+	// given point in time to recompute the metrics at that point
+	// in time.
+	metricLogNameFrequency = 4 * time.Hour
+
+	// minMetricEncodeInterval is the minimum interval that the
+	// metrics will be scanned for changes before being encoded
+	// for logtail.
+	minMetricEncodeInterval = 15 * time.Second
+)
+
+// EncodeLogTailMetricsDelta return an encoded string representing the metrics
+// differences since the previous call.
+//
+// It implements the requirements of a logtail.Config.MetricsDelta
+// func. Notably, its output is safe to embed in a JSON string literal
+// without further escaping.
+//
+// The current encoding is:
+//   * name immediately following metric:
+//     'N' + hex(varint(len(name))) + name
+//   * set value of a metric:
+//     'S' + hex(varint(wireid)) + hex(varint(value))
+//   * increment a metric: (decrements if negative)
+//     'I' + hex(varint(wireid)) + hex(varint(value))
+func EncodeLogTailMetricsDelta() string {
+	mu.Lock()
+	defer mu.Unlock()
+
+	now := time.Now()
+	if !lastDelta.IsZero() && now.Sub(lastDelta) < minMetricEncodeInterval {
+		return ""
+	}
+	lastDelta = now
+
+	var enc *deltaEncBuf // lazy
+	for i, ent := range lastLogVal {
+		val := atomic.LoadInt64(ent.v)
+		delta := val - ent.lastLogged
+		if delta == 0 {
+			continue
+		}
+		lastLogVal[i].lastLogged = val
+		m := unsorted[i]
+		if enc == nil {
+			enc = deltaPool.Get().(*deltaEncBuf)
+			enc.buf.Reset()
+		}
+		if m.wireID == 0 {
+			numWireID++
+			m.wireID = numWireID
+		}
+		if m.lastNamed.IsZero() || now.Sub(m.lastNamed) > metricLogNameFrequency {
+			enc.writeName(m.Name())
+			m.lastNamed = now
+			enc.writeValue(m.wireID, val)
+		} else {
+			enc.writeDelta(m.wireID, delta)
+		}
+	}
+	if enc == nil {
+		return ""
+	}
+	defer deltaPool.Put(enc)
+	return enc.buf.String()
+}
+
+var deltaPool = &sync.Pool{
+	New: func() interface{} {
+		return new(deltaEncBuf)
+	},
+}
+
+// deltaEncBuf encodes metrics per the format described
+// on EncodeLogTailMetricsDelta above.
+type deltaEncBuf struct {
+	buf     bytes.Buffer
+	scratch [binary.MaxVarintLen64]byte
+}
+
+// writeName writes a "name" (N) record to the buffer, which notes
+// that the immediately following record's wireID has the provided
+// name.
+func (b *deltaEncBuf) writeName(name string) {
+	b.buf.WriteByte('N')
+	b.writeHexVarint(int64(len(name)))
+	b.buf.WriteString(name)
+}
+
+// writeDelta writes a "set" (S) record to the buffer, noting that the
+// metric with the given wireID now has value v.
+func (b *deltaEncBuf) writeValue(wireID int, v int64) {
+	b.buf.WriteByte('S')
+	b.writeHexVarint(int64(wireID))
+	b.writeHexVarint(v)
+}
+
+// writeDelta writes an "increment" (I) delta value record to the
+// buffer, noting that the metric with the given wireID now has a
+// value that's v larger (or smaller if v is negative).
+func (b *deltaEncBuf) writeDelta(wireID int, v int64) {
+	b.buf.WriteByte('I')
+	b.writeHexVarint(int64(wireID))
+	b.writeHexVarint(v)
+}
+
+// writeHexVarint writes v to the buffer as a hex-encoded varint.
+func (b *deltaEncBuf) writeHexVarint(v int64) {
+	n := binary.PutVarint(b.scratch[:], v)
+	hexLen := n * 2
+	oldLen := b.buf.Len()
+	b.buf.Grow(hexLen)
+	hexBuf := b.buf.Bytes()[oldLen : oldLen+hexLen]
+	hex.Encode(hexBuf, b.scratch[:n])
+	b.buf.Write(hexBuf)
+}

util/deephash/deephash_test.go

@@ -15,6 +15,7 @@ import (
 	"runtime"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/dnstype"
@@ -163,11 +164,11 @@ func getVal() []interface{} {
 			dnsname.FQDN("d."): {netaddr.MustParseIPPort("8.8.8.8:13"), netaddr.MustParseIPPort("9.9.9.9:24")},
 			dnsname.FQDN("e."): {netaddr.MustParseIPPort("8.8.8.8:14"), netaddr.MustParseIPPort("9.9.9.9:25")},
 		},
-		map[tailcfg.DiscoKey]bool{
-			{1: 1}: true,
-			{1: 2}: false,
-			{2: 3}: true,
-			{3: 4}: false,
+		map[key.DiscoPublic]bool{
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 31: 0})): true,
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 2, 31: 0})): false,
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 3, 31: 0})): true,
+			key.DiscoPublicFromRaw32(mem.B([]byte{1: 4, 31: 0})): false,
 		},
 		&tailcfg.MapResponse{
 			DERPMap: &tailcfg.DERPMap{

util/multierr/multierr.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package multierr provides a simple multiple-error type.
+// It was inspired by github.com/go-multierror/multierror.
+package multierr
+
+import (
+	"errors"
+	"strings"
+)
+
+// An Error represents multiple errors.
+type Error struct {
+	errs []error
+}
+
+// Error implements the error interface.
+func (e Error) Error() string {
+	s := new(strings.Builder)
+	s.WriteString("multiple errors:")
+	for _, err := range e.errs {
+		s.WriteString("\n\t")
+		s.WriteString(err.Error())
+	}
+	return s.String()
+}
+
+// Errors returns a slice containing all errors in e.
+func (e Error) Errors() []error {
+	return append(e.errs[:0:0], e.errs...)
+}
+
+// New returns an error composed from errs.
+// Some errors in errs get special treatment:
+//   * nil errors are discarded
+//   * errors of type Error are expanded into the top level
+// If the resulting slice has length 0, New returns nil.
+// If the resulting slice has length 1, New returns that error.
+// If the resulting slice has length > 1, New returns that slice as an Error.
+func New(errs ...error) error {
+	dst := make([]error, 0, len(errs))
+	for _, e := range errs {
+		switch e := e.(type) {
+		case nil:
+			continue
+		case Error:
+			dst = append(dst, e.errs...)
+		default:
+			dst = append(dst, e)
+		}
+	}
+	// dst has been filtered and splatted.
+	switch len(dst) {
+	case 0:
+		return nil
+	case 1:
+		return dst[0]
+	}
+	// Zero any remaining elements of dst left over from errs, for GC.
+	tail := dst[len(dst):cap(dst)]
+	for i := range tail {
+		tail[i] = nil
+	}
+	return Error{errs: dst}
+}
+
+// Is reports whether any error in e matches target.
+func (e Error) Is(target error) bool {
+	for _, err := range e.errs {
+		if errors.Is(err, target) {
+			return true
+		}
+	}
+	return false
+}
+
+// As finds the first error in e that matches target, and if any is found,
+// sets target to that error value and returns true. Otherwise, it returns false.
+func (e Error) As(target interface{}) bool {
+	for _, err := range e.errs {
+		if ok := errors.As(err, target); ok {
+			return true
+		}
+	}
+	return false
+}

util/multierr/multierr_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package multierr_test
+
+import (
+	"errors"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"tailscale.com/util/multierr"
+)
+
+func TestAll(t *testing.T) {
+	C := qt.New(t)
+	eqErr := qt.CmpEquals(cmpopts.EquateErrors())
+
+	type E = []error
+	N := multierr.New
+
+	a := errors.New("a")
+	b := errors.New("b")
+	c := errors.New("c")
+	d := errors.New("d")
+	x := errors.New("x")
+	abcd := E{a, b, c, d}
+
+	tests := []struct {
+		In         E       // input to New
+		WantNil    bool    // want nil returned?
+		WantSingle error   // if non-nil, want this single error returned
+		WantErrors []error // if non-nil, want an Error composed of these errors returned
+	}{
+		{In: nil, WantNil: true},
+
+		{In: E{nil}, WantNil: true},
+		{In: E{nil, nil}, WantNil: true},
+
+		{In: E{a}, WantSingle: a},
+		{In: E{a, nil}, WantSingle: a},
+		{In: E{nil, a}, WantSingle: a},
+		{In: E{nil, a, nil}, WantSingle: a},
+
+		{In: E{a, b}, WantErrors: E{a, b}},
+		{In: E{nil, a, nil, b, nil}, WantErrors: E{a, b}},
+
+		{In: E{a, b, N(c, d)}, WantErrors: E{a, b, c, d}},
+		{In: E{a, N(b, c), d}, WantErrors: E{a, b, c, d}},
+		{In: E{N(a, b), c, d}, WantErrors: E{a, b, c, d}},
+		{In: E{N(a, b), N(c, d)}, WantErrors: E{a, b, c, d}},
+		{In: E{nil, N(a, nil, b), nil, N(c, d)}, WantErrors: E{a, b, c, d}},
+
+		{In: E{N(a, N(b, N(c, N(d))))}, WantErrors: E{a, b, c, d}},
+		{In: E{N(N(N(N(a), b), c), d)}, WantErrors: E{a, b, c, d}},
+
+		{In: E{N(abcd...)}, WantErrors: E{a, b, c, d}},
+		{In: E{N(abcd...), N(abcd...)}, WantErrors: E{a, b, c, d, a, b, c, d}},
+	}
+
+	for _, test := range tests {
+		got := multierr.New(test.In...)
+		if test.WantNil {
+			C.Assert(got, qt.IsNil)
+			continue
+		}
+		if test.WantSingle != nil {
+			C.Assert(got, eqErr, test.WantSingle)
+			continue
+		}
+		ee, _ := got.(multierr.Error)
+		C.Assert(ee.Errors(), eqErr, test.WantErrors)
+
+		for _, e := range test.WantErrors {
+			C.Assert(ee.Is(e), qt.IsTrue)
+		}
+		C.Assert(ee.Is(x), qt.IsFalse)
+	}
+}

version/version.go

@@ -14,7 +14,7 @@ import (
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-var Long = "date.20211022"
+var Long = "date.20211101"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.