ipn/ipnserver, log/filelogger, logpolicy: when tailscaled is running as a windows service, ensure the service's log messages are written to file

This patch moves the Windows-only initialization of the filelogger into logpolicy.
Previously we only did it when babysitting the tailscaled subprocess, but this meant
that log messages from the service itself never made it to disk. This meant that
if logtail could not dial out, its log messages would be lost.

I modified filelogger.New to work a bit differently and added a `maybeWrapForPlatform`
to logpolicy to ensure that the filelogger is plugged in front of logtail ASAP.

Fixes https://github.com/tailscale/tailscale/issues/3570

Signed-off-by: Aaron Klotz <aaron@tailscale.com>

.gitattributes

@@ -1 +1,2 @@
 go.mod filter=go-mod
+*.go  diff=golang

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -41,6 +41,7 @@ body:
         - Windows
         - iOS
         - Android
+        - Synology
         - Other
     validations:
       required: false
@@ -49,7 +50,7 @@ body:
     attributes:
       label: OS version
       description: What OS version are you using?
-      placeholder: e.g., Debian 11.0, macOS Big Sur 11.6
+      placeholder: e.g., Debian 11.0, macOS Big Sur 11.6, Synology DSM 7
     validations:
       required: false
   - type: input

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,8 @@
 blank_issues_enabled: true
 contact_links:
-  - name: Support requests and Troubleshooting
+  - name: Support
+    url: https://tailscale.com/contact/support/
+    about: Contact us for support
+  - name: Troubleshooting
     url: https://tailscale.com/kb/1023/troubleshooting
-    about: Troubleshoot common issues. Contact us by email at support@tailscale.com.
+    about: Troubleshoot common issues

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,14 +7,7 @@ body:
     attributes:
       value: |
         Please check if your feature request is [already filed](https://github.com/tailscale/tailscale/issues).
-  - type: input
-    id: request
-    attributes:
-      label: Tell us about your idea!
-      description: What is your feature request?
-      placeholder: e.g., A pet pangolin
-    validations:
-      required: true
+        Tell us about your idea!
   - type: textarea
     id: problem
     attributes:

.github/dependabot.yml

@@ -2,15 +2,20 @@
 # https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
 version: 2
 updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "go.mod:"
+  ## Disabled between releases. We reenable it briefly after every
+  ## stable release, pull in all changes, and close it again so that
+  ## the tree remains more stable during development and the upstream
+  ## changes have time to soak before the next release.
+  # - package-ecosystem: "gomod"
+  #   directory: "/"
+  #   schedule:
+  #     interval: "daily"
+  #   commit-message:
+  #     prefix: "go.mod:"
+  #   open-pull-requests-limit: 100
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     commit-message:
       prefix: ".github:"

.github/workflows/cifuzz.yml

@@ -0,0 +1,26 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tailscale'
+        dry-run: false
+        language: go
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tailscale'
+        fuzz-seconds: 300
+        dry-run: false
+        language: go
+    - name: Upload Crash
+      uses: actions/upload-artifact@v2.2.4
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/go_generate.yml

@@ -30,7 +30,7 @@ jobs:
           then
             pkgs=$(go list ./... | grep -v dnsfallback)
           else
-            pkgs=$(go list ./...)
+            pkgs=$(go list ./... | grep -v dnsfallback)
           fi
           go generate $pkgs
           echo

.github/workflows/linux-race.yml

@@ -31,6 +31,21 @@ jobs:
     - name: Run tests and benchmarks with -race flag on linux
       run: go test -race -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/linux.yml

@@ -28,9 +28,33 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
+    - name: Get QEMU
+      run: |
+        # The qemu in Ubuntu 20.04 (Focal) is too old; we need 5.x something
+        # to run Go binaries. 5.2.0 (Debian bullseye) empirically works, and
+        # use this PPA which brings in a modern qemu.
+        sudo add-apt-repository -y ppa:jacob/virtualisation
+        sudo apt-get -y update
+        sudo apt-get -y install qemu-user
+
     - name: Run tests on linux
       run: go test -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/linux32.yml

@@ -31,6 +31,21 @@ jobs:
     - name: Run tests on linux
       run: GOARCH=386 go test -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

Dockerfile

@@ -17,11 +17,11 @@
 #
 # To build the Dockerfile:
 #
-#     $ docker build -t tailscale:tailscale .
+#     $ docker build -t tailscale/tailscale .
 #
 # To run the tailscaled agent:
 #
-#     $ docker run -d --name=tailscaled -v /var/lib:/var/lib -v /dev/net/tun:/dev/net/tun --network=host --privileged tailscale:tailscale tailscaled
+#     $ docker run -d --name=tailscaled -v /var/lib:/var/lib -v /dev/net/tun:/dev/net/tun --network=host --privileged tailscale/tailscale tailscaled
 #
 # To then log in:
 #
@@ -48,13 +48,13 @@ ARG VERSION_SHORT=""
 ENV VERSION_SHORT=$VERSION_SHORT
 ARG VERSION_GIT_HASH=""
 ENV VERSION_GIT_HASH=$VERSION_GIT_HASH
+ARG TARGETARCH
 
-RUN go install -tags=xversion -ldflags="\
+RUN GOARCH=$TARGETARCH go install -tags=xversion -ldflags="\
       -X tailscale.com/version.Long=$VERSION_LONG \
       -X tailscale.com/version.Short=$VERSION_SHORT \
       -X tailscale.com/version.GitCommit=$VERSION_GIT_HASH" \
       -v ./cmd/tailscale ./cmd/tailscaled
 
-FROM alpine:3.14
-RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables
+FROM ghcr.io/tailscale/alpine-base:3.14
 COPY --from=build-env /go/bin/* /usr/local/bin/

Dockerfile.base

@@ -0,0 +1,6 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM alpine:3.14
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables

Makefile

@@ -1,3 +1,5 @@
+IMAGE_REPO ?= tailscale/tailscale
+
 usage:
 	echo "See Makefile"
 
@@ -21,7 +23,18 @@ build386:
 buildlinuxarm:
 	GOOS=linux GOARCH=arm go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled
 
+buildmultiarchimage:
+	./build_docker.sh
+
 check: staticcheck vet depaware buildwindows build386 buildlinuxarm
 
 staticcheck:
 	go run honnef.co/go/tools/cmd/staticcheck -- $$(go list ./... | grep -v tempfork)
+
+spk:
+	go run github.com/tailscale/tailscale-synology@main --version=build -o tailscale.spk --source=.
+
+pushspk: spk
+	echo "Pushing SPKG to root@${SYNOHOST} (env var SYNOHOST) ..."
+	scp tailscale.spk root@${SYNOHOST}:
+	ssh root@${SYNOHOST} /usr/syno/bin/synopkg install tailscale.spk

README.md

@@ -8,11 +8,12 @@ Private WireGuard® networks made easy
 
 This repository contains all the open source Tailscale client code and
 the `tailscaled` daemon and `tailscale` CLI tool. The `tailscaled`
-daemon runs primarily on Linux; it also works to varying degrees on
-FreeBSD, OpenBSD, Darwin, and Windows.
+daemon runs on Linux, Windows and [macOS](https://tailscale.com/kb/1065/macos-variants/), and to varying degrees on FreeBSD, OpenBSD, and Darwin. (The Tailscale iOS and Android apps use this repo's code, but this repo doesn't contain the mobile GUI code.)
 
 The Android app is at https://github.com/tailscale/tailscale-android
 
+The Synology package is at https://github.com/tailscale/tailscale-synology
+
 ## Using
 
 We serve packages for a variety of distros at

VERSION.txt

@@ -1 +1 @@
-1.17.0
+1.19.0

build_dist.sh

@@ -30,12 +30,14 @@ else
 fi
 
 long_suffix="$change_suffix-t$short_hash"
-SHORT="$major.$minor.$patch"
+MINOR="$major.$minor"
+SHORT="$MINOR.$patch"
 LONG="${SHORT}$long_suffix"
 GIT_HASH="$git_hash"
 
 if [ "$1" = "shellvars" ]; then
 	cat <<EOF
+VERSION_MINOR="$MINOR"
 VERSION_SHORT="$SHORT"
 VERSION_LONG="$LONG"
 VERSION_GIT_HASH="$GIT_HASH"

build_docker.sh

@@ -21,8 +21,15 @@ set -eu
 
 eval $(./build_dist.sh shellvars)
 
-docker build \
-  --build-arg VERSION_LONG=$VERSION_LONG \
-  --build-arg VERSION_SHORT=$VERSION_SHORT \
-  --build-arg VERSION_GIT_HASH=$VERSION_GIT_HASH \
-  -t tailscale:$VERSION_SHORT -t tailscale:latest .
+go run github.com/tailscale/mkctr@latest \
+  --base="ghcr.io/tailscale/alpine-base:3.14" \
+  --gopaths="\
+    tailscale.com/cmd/tailscale:/usr/local/bin/tailscale, \
+    tailscale.com/cmd/tailscaled:/usr/local/bin/tailscaled" \
+  --ldflags="\
+    -X tailscale.com/version.Long=${VERSION_LONG} \
+    -X tailscale.com/version.Short=${VERSION_SHORT} \
+    -X tailscale.com/version.GitCommit=${VERSION_GIT_HASH}" \
+  --tags="v${VERSION_SHORT},v${VERSION_MINOR}" \
+  --repos="tailscale/tailscale,ghcr.io/tailscale/tailscale" \
+  --push

client/tailscale/tailscale.go

@@ -38,6 +38,9 @@ var (
 	// TailscaledSocket is the tailscaled Unix socket. It's used by the TailscaledDialer.
 	TailscaledSocket = paths.DefaultTailscaledSocket()
 
+	// TailscaledSocketSetExplicitly reports whether the user explicitly set TailscaledSocket.
+	TailscaledSocketSetExplicitly bool
+
 	// TailscaledDialer is the DialContext func that connects to the local machine's
 	// tailscaled or equivalent.
 	TailscaledDialer = defaultDialer
@@ -47,7 +50,8 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 	if addr != "local-tailscaled.sock:80" {
 		return nil, fmt.Errorf("unexpected URL address %q", addr)
 	}
-	if TailscaledSocket == paths.DefaultTailscaledSocket() {
+	// TODO: make this part of a safesocket.ConnectionStrategy
+	if !TailscaledSocketSetExplicitly {
 		// On macOS, when dialing from non-sandboxed program to sandboxed GUI running
 		// a TCP server on a random port, find the random port. For HTTP connections,
 		// we don't send the token. It gets added in an HTTP Basic-Auth header.
@@ -56,7 +60,11 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 			return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(TailscaledSocket, 41112)
+	s := safesocket.DefaultConnectionStrategy(TailscaledSocket)
+	// The user provided a non-default tailscaled socket address.
+	// Connect only to exactly what they provided.
+	s.UseFallback(false)
+	return safesocket.Connect(s)
 }
 
 var (
@@ -90,6 +98,27 @@ func DoLocalRequest(req *http.Request) (*http.Response, error) {
 	return tsClient.Do(req)
 }
 
+func doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
+	res, err := DoLocalRequest(req)
+	if err == nil {
+		if server := res.Header.Get("Tailscale-Version"); server != "" && server != version.Long && onVersionMismatch != nil {
+			onVersionMismatch(version.Long, server)
+		}
+		return res, nil
+	}
+	if ue, ok := err.(*url.Error); ok {
+		if oe, ok := ue.Err.(*net.OpError); ok && oe.Op == "dial" {
+			path := req.URL.Path
+			pathPrefix := path
+			if i := strings.Index(path, "?"); i != -1 {
+				pathPrefix = path[:i]
+			}
+			return nil, fmt.Errorf("Failed to connect to local Tailscale daemon for %s; %s Error: %w", pathPrefix, tailscaledConnectHint(), oe)
+		}
+	}
+	return nil, err
+}
+
 type errorJSON struct {
 	Error string
 }
@@ -140,23 +169,11 @@ func send(ctx context.Context, method, path string, wantStatus int, body io.Read
 	if err != nil {
 		return nil, err
 	}
-	res, err := DoLocalRequest(req)
+	res, err := doLocalRequestNiceError(req)
 	if err != nil {
-		if ue, ok := err.(*url.Error); ok {
-			if oe, ok := ue.Err.(*net.OpError); ok && oe.Op == "dial" {
-				pathPrefix := path
-				if i := strings.Index(path, "?"); i != -1 {
-					pathPrefix = path[:i]
-				}
-				return nil, fmt.Errorf("Failed to connect to local Tailscale daemon for %s; %s Error: %w", pathPrefix, tailscaledConnectHint(), oe)
-			}
-		}
 		return nil, err
 	}
 	defer res.Body.Close()
-	if server := res.Header.Get("Tailscale-Version"); server != "" && server != version.Long && onVersionMismatch != nil {
-		onVersionMismatch(version.Long, server)
-	}
 	slurp, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
@@ -196,6 +213,12 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return get200(ctx, "/localapi/v0/goroutines")
 }
 
+// DaemonMetrics returns the Tailscale daemon's metrics in
+// the Prometheus text exposition format.
+func DaemonMetrics(ctx context.Context) ([]byte, error) {
+	return get200(ctx, "/localapi/v0/metrics")
+}
+
 // Profile returns a pprof profile of the Tailscale daemon.
 func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string
@@ -222,7 +245,7 @@ func Status(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "")
 }
 
-// StatusWithPeers returns the Tailscale daemon's status, without the peer info.
+// StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
 func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "?peers=false")
 }
@@ -289,6 +312,30 @@ func FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
 	return fts, nil
 }
 
+// PushFile sends Taildrop file r to target.
+//
+// A size of -1 means unknown.
+// The name parameter is the original filename, not escaped.
+func PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
+	req, err := http.NewRequestWithContext(ctx, "PUT", "http://local-tailscaled.sock/localapi/v0/file-put/"+string(target)+"/"+url.PathEscape(name), r)
+	if err != nil {
+		return err
+	}
+	if size != -1 {
+		req.ContentLength = size
+	}
+	res, err := doLocalRequestNiceError(req)
+	if err != nil {
+		return err
+	}
+	if res.StatusCode == 200 {
+		io.Copy(io.Discard, res.Body)
+		return nil
+	}
+	all, _ := io.ReadAll(res.Body)
+	return fmt.Errorf("%s: %s", res.Status, all)
+}
+
 func CheckIPForwarding(ctx context.Context) error {
 	body, err := get200(ctx, "/localapi/v0/check-ip-forwarding")
 	if err != nil {

cmd/cloner/cloner.go

@@ -95,7 +95,20 @@ func main() {
 	}
 
 	contents := new(bytes.Buffer)
-	fmt.Fprintf(contents, header, *flagTypes, pkg.Name)
+	var flagArgs []string
+	if *flagTypes != "" {
+		flagArgs = append(flagArgs, "-type="+*flagTypes)
+	}
+	if *flagOutput != "" {
+		flagArgs = append(flagArgs, "-output="+*flagOutput)
+	}
+	if *flagBuildTags != "" {
+		flagArgs = append(flagArgs, "-tags="+*flagBuildTags)
+	}
+	if *flagCloneFunc {
+		flagArgs = append(flagArgs, "-clonefunc")
+	}
+	fmt.Fprintf(contents, header, strings.Join(flagArgs, " "), pkg.Name)
 	fmt.Fprintf(contents, "import (\n")
 	for s := range imports {
 		fmt.Fprintf(contents, "\t%q\n", s)
@@ -117,8 +130,8 @@ const header = `// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserve
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by the following command; DO NOT EDIT.
-//   tailscale.com/cmd/cloner -type %s
+// Code generated by tailscale.com/cmd/cloner; DO NOT EDIT.
+//` + `go:generate` + ` go run tailscale.com/cmd/cloner %s
 
 package %s
 

cmd/derper/derper.go

@@ -12,6 +12,7 @@ import (
 	"errors"
 	"expvar"
 	"flag"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -31,12 +32,12 @@ import (
 	"tailscale.com/net/stun"
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
-	"tailscale.com/types/wgkey"
 )
 
 var (
 	dev           = flag.Bool("dev", false, "run in localhost development mode")
 	addr          = flag.String("a", ":443", "server address")
+	httpPort      = flag.Int("http-port", 80, "The port on which to serve HTTP. Set to -1 to disable")
 	configPath    = flag.String("c", "", "config file path")
 	certMode      = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
 	certDir       = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
@@ -70,12 +71,12 @@ func init() {
 }
 
 type config struct {
-	PrivateKey wgkey.Private
+	PrivateKey key.NodePrivate
 }
 
 func loadConfig() config {
 	if *dev {
-		return config{PrivateKey: mustNewKey()}
+		return config{PrivateKey: key.NewNode()}
 	}
 	if *configPath == "" {
 		if os.Getuid() == 0 {
@@ -101,21 +102,13 @@ func loadConfig() config {
 	}
 }
 
-func mustNewKey() wgkey.Private {
-	key, err := wgkey.NewPrivate()
-	if err != nil {
-		log.Fatal(err)
-	}
-	return key
-}
-
 func writeNewConfig() config {
-	key := mustNewKey()
+	k := key.NewNode()
 	if err := os.MkdirAll(filepath.Dir(*configPath), 0777); err != nil {
 		log.Fatal(err)
 	}
 	cfg := config{
-		PrivateKey: key,
+		PrivateKey: k,
 	}
 	b, err := json.MarshalIndent(cfg, "", "\t")
 	if err != nil {
@@ -150,9 +143,9 @@ func main() {
 
 	cfg := loadConfig()
 
-	serveTLS := tsweb.IsProd443(*addr)
+	serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual"
 
-	s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf)
+	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)
 
 	if *meshPSKFile != "" {
@@ -173,7 +166,10 @@ func main() {
 	expvar.Publish("derp", s.ExpVar())
 
 	mux := http.NewServeMux()
-	mux.Handle("/derp", derphttp.Handler(s))
+	derpHandler := derphttp.Handler(s)
+	derpHandler = addWebSocketSupport(s, derpHandler)
+	mux.Handle("/derp", derpHandler)
+	mux.HandleFunc("/derp/probe", probeHandler)
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", handleBootstrapDNS)
 	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -241,24 +237,41 @@ func main() {
 			cert.Certificate = append(cert.Certificate, s.MetaCert())
 			return cert, nil
 		}
-		go func() {
-			port80srv := &http.Server{
-				Addr:        net.JoinHostPort(listenHost, "80"),
-				Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
-				ReadTimeout: 30 * time.Second,
-				// Crank up WriteTimeout a bit more than usually
-				// necessary just so we can do long CPU profiles
-				// and not hit net/http/pprof's "profile
-				// duration exceeds server's WriteTimeout".
-				WriteTimeout: 5 * time.Minute,
-			}
-			err := port80srv.ListenAndServe()
-			if err != nil {
-				if err != http.ErrServerClosed {
-					log.Fatal(err)
+		httpsrv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Set HTTP headers to appease automated security scanners.
+			//
+			// Security automation gets cranky when HTTPS sites don't
+			// set HSTS, and when they don't specify a content
+			// security policy for XSS mitigation.
+			//
+			// DERP's HTTP interface is only ever used for debug
+			// access (for which trivial safe policies work just
+			// fine), and by DERP clients which don't obey any of
+			// these browser-centric headers anyway.
+			w.Header().Set("Strict-Transport-Security", "max-age=63072000; includeSubDomains")
+			w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'; form-action 'none'; base-uri 'self'; block-all-mixed-content; plugin-types 'none'")
+			mux.ServeHTTP(w, r)
+		})
+		if *httpPort > -1 {
+			go func() {
+				port80srv := &http.Server{
+					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
+					Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
+					ReadTimeout: 30 * time.Second,
+					// Crank up WriteTimeout a bit more than usually
+					// necessary just so we can do long CPU profiles
+					// and not hit net/http/pprof's "profile
+					// duration exceeds server's WriteTimeout".
+					WriteTimeout: 5 * time.Minute,
 				}
-			}
-		}()
+				err := port80srv.ListenAndServe()
+				if err != nil {
+					if err != http.ErrServerClosed {
+						log.Fatal(err)
+					}
+				}
+			}()
+		}
 		err = httpsrv.ListenAndServeTLS("", "")
 	} else {
 		log.Printf("derper: serving on %s", *addr)
@@ -269,8 +282,18 @@ func main() {
 	}
 }
 
-func serveSTUN(host string) {
+// probeHandler is the endpoint that js/wasm clients hit to measure
+// DERP latency, since they can't do UDP STUN queries.
+func probeHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case "HEAD", "GET":
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+	default:
+		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
+	}
+}
 
+func serveSTUN(host string) {
 	pc, err := net.ListenPacket("udp", net.JoinHostPort(host, "3478"))
 	if err != nil {
 		log.Fatalf("failed to open STUN listener: %v", err)

cmd/derper/mesh.go

@@ -69,8 +69,8 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return d.DialContext(ctx, network, addr)
 	})
 
-	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
-	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
+	add := func(k key.NodePublic) { s.AddPacketForwarder(k, c) }
+	remove := func(k key.NodePublic) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)
 	return nil
 }

cmd/derper/websocket.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bufio"
+	"expvar"
+	"log"
+	"net/http"
+	"strings"
+
+	"nhooyr.io/websocket"
+	"tailscale.com/derp"
+	"tailscale.com/derp/wsconn"
+)
+
+var counterWebSocketAccepts = expvar.NewInt("derp_websocket_accepts")
+
+// addWebSocketSupport returns a Handle wrapping base that adds WebSocket server support.
+func addWebSocketSupport(s *derp.Server, base http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		up := strings.ToLower(r.Header.Get("Upgrade"))
+
+		// Very early versions of Tailscale set "Upgrade: WebSocket" but didn't actually
+		// speak WebSockets (they still assumed DERP's binary framining). So to distinguish
+		// clients that actually want WebSockets, look for an explicit "derp" subprotocol.
+		if up != "websocket" || !strings.Contains(r.Header.Get("Sec-Websocket-Protocol"), "derp") {
+			base.ServeHTTP(w, r)
+			return
+		}
+
+		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
+			Subprotocols:   []string{"derp"},
+			OriginPatterns: []string{"*"},
+		})
+		if err != nil {
+			log.Printf("websocket.Accept: %v", err)
+			return
+		}
+		defer c.Close(websocket.StatusInternalError, "closing")
+		if c.Subprotocol() != "derp" {
+			c.Close(websocket.StatusPolicyViolation, "client must speak the derp subprotocol")
+			return
+		}
+		counterWebSocketAccepts.Add(1)
+		wc := wsconn.New(c)
+		brw := bufio.NewReadWriter(bufio.NewReader(wc), bufio.NewWriter(wc))
+		s.Accept(wc, brw, r.RemoteAddr)
+	})
+}

cmd/derpprobe/derpprobe.go

@@ -344,7 +344,7 @@ func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.D
 }
 
 func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode) (*derphttp.Client, error) {
-	priv := key.NewPrivate()
+	priv := key.NewNode()
 	dc := derphttp.NewRegionClient(priv, log.Printf, func() *tailcfg.DERPRegion {
 		rid := n.RegionID
 		return &tailcfg.DERPRegion{

cmd/tailscale/cli/bugreport.go

@@ -7,7 +7,6 @@ package cli
 import (
 	"context"
 	"errors"
-	"fmt"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -33,6 +32,6 @@ func runBugReport(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
-	fmt.Println(logMarker)
+	outln(logMarker)
 	return nil
 }

cmd/tailscale/cli/cert.go

@@ -29,7 +29,7 @@ var certCmd = &ffcli.Command{
 	ShortHelp:  "get TLS certs",
 	ShortUsage: "cert [flags] <domain>",
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("cert", flag.ExitOnError)
+		fs := newFlagSet("cert")
 		fs.StringVar(&certArgs.certFile, "cert-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.crt if --cert-file and --key-file are both unset")
 		fs.StringVar(&certArgs.keyFile, "key-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.key if --cert-file and --key-file are both unset")
 		fs.BoolVar(&certArgs.serve, "serve-demo", false, "if true, serve on port :443 using the cert as a demo, instead of writing out the files to disk")
@@ -81,7 +81,7 @@ func runCert(ctx context.Context, args []string) error {
 	domain := args[0]
 
 	printf := func(format string, a ...interface{}) {
-		fmt.Printf(format, a...)
+		printf(format, a...)
 	}
 	if certArgs.certFile == "-" || certArgs.keyFile == "-" {
 		printf = log.Printf
@@ -143,7 +143,7 @@ func runCert(ctx context.Context, args []string) error {
 
 func writeIfChanged(filename string, contents []byte, mode os.FileMode) (changed bool, err error) {
 	if filename == "-" {
-		os.Stdout.Write(contents)
+		Stdout.Write(contents)
 		return false, nil
 	}
 	if old, err := os.ReadFile(filename); err == nil && bytes.Equal(contents, old) {

cmd/tailscale/cli/cli.go

@@ -31,6 +31,22 @@ import (
 	"tailscale.com/syncs"
 )
 
+var Stderr io.Writer = os.Stderr
+var Stdout io.Writer = os.Stdout
+
+func printf(format string, a ...interface{}) {
+	fmt.Fprintf(Stdout, format, a...)
+}
+
+// outln is like fmt.Println in the common case, except when Stdout is
+// changed (as in js/wasm).
+//
+// It's not named println because that looks like the Go built-in
+// which goes to stderr and formats slightly differently.
+func outln(a ...interface{}) {
+	fmt.Fprintln(Stdout, a...)
+}
+
 // ActLikeCLI reports whether a GUI application should act like the
 // CLI based on os.Args, GOOS, the context the process is running in
 // (pty, parent PID), etc.
@@ -77,6 +93,16 @@ func ActLikeCLI() bool {
 	return false
 }
 
+func newFlagSet(name string) *flag.FlagSet {
+	onError := flag.ExitOnError
+	if runtime.GOOS == "js" {
+		onError = flag.ContinueOnError
+	}
+	fs := flag.NewFlagSet(name, onError)
+	fs.SetOutput(Stderr)
+	return fs
+}
+
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) error {
 	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
@@ -86,11 +112,11 @@ func Run(args []string) error {
 	var warnOnce sync.Once
 	tailscale.SetVersionMismatchHandler(func(clientVer, serverVer string) {
 		warnOnce.Do(func() {
-			fmt.Fprintf(os.Stderr, "Warning: client version %q != tailscaled server version %q\n", clientVer, serverVer)
+			fmt.Fprintf(Stderr, "Warning: client version %q != tailscaled server version %q\n", clientVer, serverVer)
 		})
 	})
 
-	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
+	rootfs := newFlagSet("tailscale")
 	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
 
 	rootCmd := &ffcli.Command{
@@ -131,23 +157,38 @@ change in the future.
 	}
 
 	if err := rootCmd.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
 		return err
 	}
 
 	tailscale.TailscaledSocket = rootArgs.socket
+	rootfs.Visit(func(f *flag.Flag) {
+		if f.Name == "socket" {
+			tailscale.TailscaledSocketSetExplicitly = true
+		}
+	})
 
 	err := rootCmd.Run(context.Background())
-	if err == flag.ErrHelp {
+	if errors.Is(err, flag.ErrHelp) {
 		return nil
 	}
 	return err
 }
 
 func fatalf(format string, a ...interface{}) {
+	if Fatalf != nil {
+		Fatalf(format, a...)
+		return
+	}
 	log.SetFlags(0)
 	log.Fatalf(format, a...)
 }
 
+// Fatalf, if non-nil, is used instead of log.Fatalf.
+var Fatalf func(format string, a ...interface{})
+
 var rootArgs struct {
 	socket string
 }
@@ -155,7 +196,8 @@ var rootArgs struct {
 var gotSignal syncs.AtomicBool
 
 func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	s := safesocket.DefaultConnectionStrategy(rootArgs.socket)
+	c, err := safesocket.Connect(s)
 	if err != nil {
 		if runtime.GOOS != "windows" && rootArgs.socket == "" {
 			fatalf("--socket cannot be empty")

cmd/tailscale/cli/cli_test.go

@@ -18,8 +18,10 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tstest"
+	"tailscale.com/types/key"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
+	"tailscale.com/version/distro"
 )
 
 // geese is a collection of gooses. It need not be complete.
@@ -57,6 +59,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 		curExitNodeIP netaddr.IP
 		curUser       string // os.Getenv("USER") on the client side
 		goos          string // empty means "linux"
+		distro        distro.Distro
 
 		want string
 	}{
@@ -313,6 +316,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				ControlURL:       ipn.DefaultControlURL,
 				AllowSingleHosts: true,
 				CorpDNS:          true,
+				RouteAll:         true,
 
 				// And assume this no-op accidental pre-1.8 value:
 				NoSNAT: true,
@@ -329,7 +333,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 
 				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
 			},
-			goos: "windows",
+			goos: "openbsd",
 			want: "", // not an error
 		},
 		{
@@ -405,6 +409,21 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
+		{
+			name:          "error_exit_node_and_allow_lan_omit_with_id_pref", // Isue 3480
+			flags:         []string{"--hostname=foo"},
+			curExitNodeIP: netaddr.MustParseIP("100.2.3.4"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeAllowLANAccess: true,
+				ExitNodeID:             "some_stable_id",
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node-allow-lan-access --exit-node=100.2.3.4",
+		},
 		{
 			name:  "ignore_login_server_synonym",
 			flags: []string{"--login-server=https://controlplane.tailscale.com"},
@@ -427,6 +446,38 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
 		},
+		{
+			// Issue 3176: on Synology, don't require --accept-routes=false because user
+			// migth've had old an install, and we don't support --accept-routes anyway.
+			name:  "synology_permit_omit_accept_routes",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				RouteAll:         true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			goos:   "linux",
+			distro: distro.Synology,
+			want:   "",
+		},
+		{
+			// Same test case as "synology_permit_omit_accept_routes" above, but
+			// on non-Synology distro.
+			name:  "not_synology_dont_permit_omit_accept_routes",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				RouteAll:         true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			goos:   "linux",
+			distro: "", // not Synology
+			want:   accidentalUpPrefix + " --hostname=foo --accept-routes",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -447,6 +498,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				goos:          goos,
 				flagSet:       flagSet,
 				curExitNodeIP: tt.curExitNodeIP,
+				distro:        tt.distro,
 			}); err != nil {
 				got = err.Error()
 			}
@@ -495,6 +547,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				WantRunning:      true,
 				CorpDNS:          true,
 				AllowSingleHosts: true,
+				RouteAll:         true,
 				NetfilterMode:    preftype.NetfilterOn,
 			},
 		},
@@ -532,7 +585,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			args: upArgsT{
 				exitNodeIP: "foo",
 			},
-			wantErr: `invalid IP address "foo" for --exit-node: ParseIP("foo"): unable to parse IP`,
+			wantErr: `invalid value "foo" for --exit-node; must be IP or unique node name`,
 		},
 		{
 			name: "error_exit_node_allow_lan_without_exit_node",
@@ -759,6 +812,18 @@ func TestUpdatePrefs(t *testing.T) {
 			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
 			wantErrSubtr:   "can't change --login-server without --force-reauth",
 		},
+		{
+			name:  "change_tags",
+			flags: []string{"--advertise-tags=tag:foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				Persist:          &persist.Persist{LoginName: "crawshaw.github"},
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			env: upCheckEnv{backendState: "Running"},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -794,3 +859,133 @@ func TestUpdatePrefs(t *testing.T) {
 		})
 	}
 }
+
+func TestExitNodeIPOfArg(t *testing.T) {
+	mustIP := netaddr.MustParseIP
+	tests := []struct {
+		name    string
+		arg     string
+		st      *ipnstate.Status
+		want    netaddr.IP
+		wantErr string
+	}{
+		{
+			name: "ip_while_stopped_okay",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Stopped",
+			},
+			want: mustIP("1.2.3.4"),
+		},
+		{
+			name: "ip_not_found",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+			},
+			wantErr: `no node found in netmap with IP 1.2.3.4`,
+		},
+		{
+			name: "ip_not_exit",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						TailscaleIPs: []netaddr.IP{mustIP("1.2.3.4")},
+					},
+				},
+			},
+			wantErr: `node 1.2.3.4 is not advertising an exit node`,
+		},
+		{
+			name: "ip",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						TailscaleIPs:   []netaddr.IP{mustIP("1.2.3.4")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			want: mustIP("1.2.3.4"),
+		},
+		{
+			name:    "no_match",
+			arg:     "unknown",
+			st:      &ipnstate.Status{MagicDNSSuffix: ".foo"},
+			wantErr: `invalid value "unknown" for --exit-node; must be IP or unique node name`,
+		},
+		{
+			name: "name",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:        "skippy.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			want: mustIP("1.0.0.2"),
+		},
+		{
+			name: "name_not_exit",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:      "skippy.foo.",
+						TailscaleIPs: []netaddr.IP{mustIP("1.0.0.2")},
+					},
+				},
+			},
+			wantErr: `node "skippy" is not advertising an exit node`,
+		},
+		{
+			name: "ambiguous",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:        "skippy.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+					key.NewNode().Public(): {
+						DNSName:        "SKIPPY.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			wantErr: `ambiguous exit node name "skippy"`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := exitNodeIPOfArg(tt.arg, tt.st)
+			if err != nil {
+				if err.Error() == tt.wantErr {
+					return
+				}
+				if tt.wantErr == "" {
+					t.Fatal(err)
+				}
+				t.Fatalf("error = %#q; want %#q", err, tt.wantErr)
+			}
+			if tt.wantErr != "" {
+				t.Fatalf("got %v; want error %#q", got, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Fatalf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/tailscale/cli/debug.go

@@ -5,6 +5,8 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -14,7 +16,9 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -24,44 +28,81 @@ import (
 )
 
 var debugCmd = &ffcli.Command{
-	Name: "debug",
-	Exec: runDebug,
+	Name:     "debug",
+	Exec:     runDebug,
+	LongHelp: `"tailscale debug" contains misc debug facilities; it is not a stable interface.`,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("debug", flag.ExitOnError)
-		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
-		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
-		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
-		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
-		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
-		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
-		fs.BoolVar(&debugArgs.env, "env", false, "dump environment")
-		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
+		fs := newFlagSet("debug")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file; - for stdout")
 		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout")
 		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs
 	})(),
+	Subcommands: []*ffcli.Command{
+		{
+			Name:      "derp-map",
+			Exec:      runDERPMap,
+			ShortHelp: "print DERP map",
+		},
+		{
+			Name:      "daemon-goroutines",
+			Exec:      runDaemonGoroutines,
+			ShortHelp: "print tailscaled's goroutines",
+		},
+		{
+			Name:      "metrics",
+			Exec:      runDaemonMetrics,
+			ShortHelp: "print tailscaled's metrics",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("metrics")
+				fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values")
+				return fs
+			})(),
+		},
+		{
+			Name:      "env",
+			Exec:      runEnv,
+			ShortHelp: "print cmd/tailscale environment",
+		},
+		{
+			Name:      "local-creds",
+			Exec:      runLocalCreds,
+			ShortHelp: "print how to access Tailscale local API",
+		},
+		{
+			Name:      "prefs",
+			Exec:      runPrefs,
+			ShortHelp: "print prefs",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("prefs")
+				fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output")
+				return fs
+			})(),
+		},
+		{
+			Name:      "watch-ipn",
+			Exec:      runWatchIPN,
+			ShortHelp: "subscribe to IPN message bus",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("watch-ipn")
+				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
+				return fs
+			})(),
+		},
+	},
 }
 
 var debugArgs struct {
-	env        bool
-	localCreds bool
-	goroutines bool
-	ipn        bool
-	netMap     bool
-	derpMap    bool
-	file       string
-	prefs      bool
-	pretty     bool
-	cpuSec     int
-	cpuFile    string
-	memFile    string
+	file    string
+	cpuSec  int
+	cpuFile string
+	memFile string
 }
 
 func writeProfile(dst string, v []byte) error {
 	if dst == "-" {
-		_, err := os.Stdout.Write(v)
+		_, err := Stdout.Write(v)
 		return err
 	}
 	return os.WriteFile(dst, v, 0600)
@@ -81,26 +122,9 @@ func runDebug(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("unknown arguments")
 	}
-	if debugArgs.env {
-		for _, e := range os.Environ() {
-			fmt.Println(e)
-		}
-		return nil
-	}
-	if debugArgs.localCreds {
-		port, token, err := safesocket.LocalTCPPortAndToken()
-		if err == nil {
-			fmt.Printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
-			return nil
-		}
-		if runtime.GOOS == "windows" {
-			fmt.Printf("curl http://localhost:41112/localapi/v0/status\n")
-			return nil
-		}
-		fmt.Printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
-		return nil
-	}
+	var usedFlag bool
 	if out := debugArgs.cpuFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec)
 		if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil {
 			return err
@@ -112,6 +136,7 @@ func runDebug(ctx context.Context, args []string) error {
 		}
 	}
 	if out := debugArgs.memFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing memory profile ...")
 		if v, err := tailscale.Profile(ctx, "heap", 0); err != nil {
 			return err
@@ -122,61 +147,14 @@ func runDebug(ctx context.Context, args []string) error {
 			log.Printf("Memory profile written to %s", outName(out))
 		}
 	}
-	if debugArgs.prefs {
-		prefs, err := tailscale.GetPrefs(ctx)
-		if err != nil {
-			return err
-		}
-		if debugArgs.pretty {
-			fmt.Println(prefs.Pretty())
-		} else {
-			j, _ := json.MarshalIndent(prefs, "", "\t")
-			fmt.Println(string(j))
-		}
-		return nil
-	}
-	if debugArgs.goroutines {
-		goroutines, err := tailscale.Goroutines(ctx)
-		if err != nil {
-			return err
-		}
-		os.Stdout.Write(goroutines)
-		return nil
-	}
-	if debugArgs.derpMap {
-		dm, err := tailscale.CurrentDERPMap(ctx)
-		if err != nil {
-			return fmt.Errorf(
-				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
-			)
-		}
-		enc := json.NewEncoder(os.Stdout)
-		enc.SetIndent("", "\t")
-		enc.Encode(dm)
-		return nil
-	}
-	if debugArgs.ipn {
-		c, bc, ctx, cancel := connect(ctx)
-		defer cancel()
-
-		bc.SetNotifyCallback(func(n ipn.Notify) {
-			if !debugArgs.netMap {
-				n.NetMap = nil
-			}
-			j, _ := json.MarshalIndent(n, "", "\t")
-			fmt.Printf("%s\n", j)
-		})
-		bc.RequestEngineStatus()
-		pump(ctx, bc, c)
-		return errors.New("exit")
-	}
 	if debugArgs.file != "" {
+		usedFlag = true // TODO(bradfitz): add "file" subcommand
 		if debugArgs.file == "get" {
 			wfs, err := tailscale.WaitingFiles(ctx)
 			if err != nil {
-				log.Fatal(err)
+				fatalf("%v\n", err)
 			}
-			e := json.NewEncoder(os.Stdout)
+			e := json.NewEncoder(Stdout)
 			e.SetIndent("", "\t")
 			e.Encode(wfs)
 			return nil
@@ -190,8 +168,151 @@ func runDebug(ctx context.Context, args []string) error {
 			return err
 		}
 		log.Printf("Size: %v\n", size)
-		io.Copy(os.Stdout, rc)
+		io.Copy(Stdout, rc)
 		return nil
 	}
+	if usedFlag {
+		// TODO(bradfitz): delete this path when all debug flags are migrated
+		// to subcommands.
+		return nil
+	}
+	return errors.New("see 'tailscale debug --help")
+}
+
+func runLocalCreds(ctx context.Context, args []string) error {
+	port, token, err := safesocket.LocalTCPPortAndToken()
+	if err == nil {
+		printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
+		return nil
+	}
+	if runtime.GOOS == "windows" {
+		printf("curl http://localhost:%v/localapi/v0/status\n", safesocket.WindowsLocalPort)
+		return nil
+	}
+	printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
+	return nil
+}
+
+var prefsArgs struct {
+	pretty bool
+}
+
+func runPrefs(ctx context.Context, args []string) error {
+	prefs, err := tailscale.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+	if prefsArgs.pretty {
+		outln(prefs.Pretty())
+	} else {
+		j, _ := json.MarshalIndent(prefs, "", "\t")
+		outln(string(j))
+	}
 	return nil
 }
+
+var watchIPNArgs struct {
+	netmap bool
+}
+
+func runWatchIPN(ctx context.Context, args []string) error {
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if !watchIPNArgs.netmap {
+			n.NetMap = nil
+		}
+		j, _ := json.MarshalIndent(n, "", "\t")
+		printf("%s\n", j)
+	})
+	bc.RequestEngineStatus()
+	pump(ctx, bc, c)
+	return errors.New("exit")
+}
+
+func runDERPMap(ctx context.Context, args []string) error {
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+		)
+	}
+	enc := json.NewEncoder(Stdout)
+	enc.SetIndent("", "\t")
+	enc.Encode(dm)
+	return nil
+}
+
+func runEnv(ctx context.Context, args []string) error {
+	for _, e := range os.Environ() {
+		outln(e)
+	}
+	return nil
+}
+
+func runDaemonGoroutines(ctx context.Context, args []string) error {
+	goroutines, err := tailscale.Goroutines(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(goroutines)
+	return nil
+}
+
+var metricsArgs struct {
+	watch bool
+}
+
+func runDaemonMetrics(ctx context.Context, args []string) error {
+	last := map[string]int64{}
+	for {
+		out, err := tailscale.DaemonMetrics(ctx)
+		if err != nil {
+			return err
+		}
+		if !metricsArgs.watch {
+			Stdout.Write(out)
+			return nil
+		}
+		bs := bufio.NewScanner(bytes.NewReader(out))
+		type change struct {
+			name     string
+			from, to int64
+		}
+		var changes []change
+		var maxNameLen int
+		for bs.Scan() {
+			line := bytes.TrimSpace(bs.Bytes())
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			f := strings.Fields(string(line))
+			if len(f) != 2 {
+				continue
+			}
+			name := f[0]
+			n, _ := strconv.ParseInt(f[1], 10, 64)
+			prev, ok := last[name]
+			if ok && prev == n {
+				continue
+			}
+			last[name] = n
+			if !ok {
+				continue
+			}
+			changes = append(changes, change{name, prev, n})
+			if len(name) > maxNameLen {
+				maxNameLen = len(name)
+			}
+		}
+		if len(changes) > 0 {
+			format := fmt.Sprintf("%%-%ds %%+5d => %%v\n", maxNameLen)
+			for _, c := range changes {
+				fmt.Fprintf(Stdout, format, c.name, c.to-c.from, c.to)
+			}
+			io.WriteString(Stdout, "\n")
+		}
+		time.Sleep(time.Second)
+	}
+}

cmd/tailscale/cli/diag.go

@@ -25,23 +25,23 @@ func fixTailscaledConnectError(origErr error) error {
 	if err != nil {
 		return fmt.Errorf("failed to connect to local Tailscaled process and failed to enumerate processes while looking for it")
 	}
-	found := false
+	var foundProc ps.Process
 	for _, proc := range procs {
 		base := filepath.Base(proc.Executable())
 		if base == "tailscaled" {
-			found = true
+			foundProc = proc
 			break
 		}
 		if runtime.GOOS == "darwin" && base == "IPNExtension" {
-			found = true
+			foundProc = proc
 			break
 		}
 		if runtime.GOOS == "windows" && strings.EqualFold(base, "tailscaled.exe") {
-			found = true
+			foundProc = proc
 			break
 		}
 	}
-	if !found {
+	if foundProc == nil {
 		switch runtime.GOOS {
 		case "windows":
 			return fmt.Errorf("failed to connect to local tailscaled process; is the Tailscale service running?")
@@ -52,5 +52,5 @@ func fixTailscaledConnectError(origErr error) error {
 		}
 		return fmt.Errorf("failed to connect to local tailscaled process; it doesn't appear to be running")
 	}
-	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running). Got error: %w", origErr)
+	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running as %v, pid %v). Got error: %w", foundProc.Executable(), foundProc.Pid(), origErr)
 }

cmd/tailscale/cli/down.go

@@ -7,8 +7,6 @@ package cli
 import (
 	"context"
 	"fmt"
-	"log"
-	"os"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -25,7 +23,7 @@ var downCmd = &ffcli.Command{
 
 func runDown(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 
 	st, err := tailscale.Status(ctx)
@@ -33,7 +31,7 @@ func runDown(ctx context.Context, args []string) error {
 		return fmt.Errorf("error fetching current status: %w", err)
 	}
 	if st.BackendState == "Stopped" {
-		fmt.Fprintf(os.Stderr, "Tailscale was already stopped.\n")
+		fmt.Fprintf(Stderr, "Tailscale was already stopped.\n")
 		return nil
 	}
 	_, err = tailscale.EditPrefs(ctx, &ipn.MaskedPrefs{

cmd/tailscale/cli/file.go

@@ -11,11 +11,9 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"mime"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -30,6 +28,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/tailcfg"
 	"tailscale.com/version"
 )
 
@@ -55,7 +54,7 @@ var fileCpCmd = &ffcli.Command{
 	ShortHelp:  "Copy file(s) to a host",
 	Exec:       runCp,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("cp", flag.ExitOnError)
+		fs := newFlagSet("cp")
 		fs.StringVar(&cpArgs.name, "name", "", "alternate filename to use, especially useful when <file> is \"-\" (stdin)")
 		fs.BoolVar(&cpArgs.verbose, "verbose", false, "verbose output")
 		fs.BoolVar(&cpArgs.targets, "targets", false, "list possible file cp targets")
@@ -96,12 +95,12 @@ func runCp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	peerAPIBase, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	stableID, isOffline, err := getTargetStableID(ctx, ip)
 	if err != nil {
 		return fmt.Errorf("can't send to %s: %v", target, err)
 	}
 	if isOffline {
-		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
+		fmt.Fprintf(Stderr, "# warning: %s is offline\n", target)
 	}
 
 	if len(files) > 1 {
@@ -154,32 +153,21 @@ func runCp(ctx context.Context, args []string) error {
 			}
 		}
 
-		dstURL := peerAPIBase + "/v0/put/" + url.PathEscape(name)
-		req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
-		if err != nil {
-			return err
-		}
-		req.ContentLength = contentLength
 		if cpArgs.verbose {
-			log.Printf("sending to %v ...", dstURL)
+			log.Printf("sending %q to %v/%v/%v ...", name, target, ip, stableID)
 		}
-		res, err := http.DefaultClient.Do(req)
+		err := tailscale.PushFile(ctx, stableID, contentLength, name, fileContents)
 		if err != nil {
 			return err
 		}
-		if res.StatusCode == 200 {
-			io.Copy(ioutil.Discard, res.Body)
-			res.Body.Close()
-			continue
+		if cpArgs.verbose {
+			log.Printf("sent %q", name)
 		}
-		io.Copy(os.Stdout, res.Body)
-		res.Body.Close()
-		return errors.New(res.Status)
 	}
 	return nil
 }
 
-func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffline bool, err error) {
+func getTargetStableID(ctx context.Context, ipStr string) (id tailcfg.StableNodeID, isOffline bool, err error) {
 	ip, err := netaddr.ParseIP(ipStr)
 	if err != nil {
 		return "", false, err
@@ -195,7 +183,7 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffl
 				continue
 			}
 			isOffline = n.Online != nil && !*n.Online
-			return ft.PeerAPIURL, isOffline, nil
+			return n.StableID, isOffline, nil
 		}
 	}
 	return "", false, fileTargetErrorDetail(ctx, ip)
@@ -293,7 +281,7 @@ func runCpTargets(ctx context.Context, args []string) error {
 		if detail != "" {
 			detail = "\t" + detail
 		}
-		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
+		printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
 	}
 	return nil
 }
@@ -304,7 +292,7 @@ var fileGetCmd = &ffcli.Command{
 	ShortHelp:  "Move files out of the Tailscale file inbox",
 	Exec:       runFileGet,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("get", flag.ExitOnError)
+		fs := newFlagSet("get")
 		fs.BoolVar(&getArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty")
 		fs.BoolVar(&getArgs.verbose, "verbose", false, "verbose output")
 		return fs
@@ -415,7 +403,7 @@ func waitForFile(ctx context.Context) error {
 	fileWaiting := make(chan bool, 1)
 	bc.SetNotifyCallback(func(n ipn.Notify) {
 		if n.ErrMessage != nil {
-			log.Fatal(*n.ErrMessage)
+			fatalf("Notify.ErrMessage: %v\n", *n.ErrMessage)
 		}
 		if n.FilesWaiting != nil {
 			select {

cmd/tailscale/cli/ip.go

@@ -18,12 +18,13 @@ import (
 
 var ipCmd = &ffcli.Command{
 	Name:       "ip",
-	ShortUsage: "ip [-4] [-6] [peername]",
-	ShortHelp:  "Show current Tailscale IP address(es)",
-	LongHelp:   "Shows the Tailscale IP address of the current machine without an argument. With an argument, it shows the IP of a named peer.",
+	ShortUsage: "ip [-1] [-4] [-6] [peer hostname or ip address]",
+	ShortHelp:  "Show Tailscale IP addresses",
+	LongHelp:   "Show Tailscale IP addresses for peer. Peer defaults to the current machine.",
 	Exec:       runIP,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("ip", flag.ExitOnError)
+		fs := newFlagSet("ip")
+		fs.BoolVar(&ipArgs.want1, "1", false, "only print one IP address")
 		fs.BoolVar(&ipArgs.want4, "4", false, "only print IPv4 address")
 		fs.BoolVar(&ipArgs.want6, "6", false, "only print IPv6 address")
 		return fs
@@ -31,13 +32,14 @@ var ipCmd = &ffcli.Command{
 }
 
 var ipArgs struct {
+	want1 bool
 	want4 bool
 	want6 bool
 }
 
 func runIP(ctx context.Context, args []string) error {
 	if len(args) > 1 {
-		return errors.New("unknown arguments")
+		return errors.New("too many arguments, expected at most one peer")
 	}
 	var of string
 	if len(args) == 1 {
@@ -45,8 +47,14 @@ func runIP(ctx context.Context, args []string) error {
 	}
 
 	v4, v6 := ipArgs.want4, ipArgs.want6
-	if v4 && v6 {
-		return errors.New("tailscale ip -4 and -6 are mutually exclusive")
+	nflags := 0
+	for _, b := range []bool{ipArgs.want1, v4, v6} {
+		if b {
+			nflags++
+		}
+	}
+	if nflags > 1 {
+		return errors.New("tailscale ip -1, -4, and -6 are mutually exclusive")
 	}
 	if !v4 && !v6 {
 		v4, v6 = true, true
@@ -71,11 +79,14 @@ func runIP(ctx context.Context, args []string) error {
 		return fmt.Errorf("no current Tailscale IPs; state: %v", st.BackendState)
 	}
 
+	if ipArgs.want1 {
+		ips = ips[:1]
+	}
 	match := false
 	for _, ip := range ips {
 		if ip.Is4() && v4 || ip.Is6() && v6 {
 			match = true
-			fmt.Println(ip)
+			outln(ip)
 		}
 	}
 	if !match {

cmd/tailscale/cli/logout.go

@@ -6,7 +6,7 @@ package cli
 
 import (
 	"context"
-	"log"
+	"fmt"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
@@ -28,7 +28,7 @@ a reauthentication.
 
 func runLogout(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 	return tailscale.Logout(ctx)
 }

cmd/tailscale/cli/netcheck.go

@@ -33,7 +33,7 @@ var netcheckCmd = &ffcli.Command{
 	ShortHelp:  "Print an analysis of local network conditions",
 	Exec:       runNetcheck,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("netcheck", flag.ExitOnError)
+		fs := newFlagSet("netcheck")
 		fs.StringVar(&netcheckArgs.format, "format", "", `output format; empty (for human-readable), "json" or "json-line"`)
 		fs.DurationVar(&netcheckArgs.every, "every", 0, "if non-zero, do an incremental report with the given frequency")
 		fs.BoolVar(&netcheckArgs.verbose, "verbose", false, "verbose logs")
@@ -60,7 +60,7 @@ func runNetcheck(ctx context.Context, args []string) error {
 	}
 
 	if strings.HasPrefix(netcheckArgs.format, "json") {
-		fmt.Fprintln(os.Stderr, "# Warning: this JSON format is not yet considered a stable interface")
+		fmt.Fprintln(Stderr, "# Warning: this JSON format is not yet considered a stable interface")
 	}
 
 	dm, err := tailscale.CurrentDERPMap(ctx)
@@ -82,7 +82,7 @@ func runNetcheck(ctx context.Context, args []string) error {
 			c.Logf("GetReport took %v; err=%v", d.Round(time.Millisecond), err)
 		}
 		if err != nil {
-			log.Fatalf("netcheck: %v", err)
+			return fmt.Errorf("netcheck: %w", err)
 		}
 		if err := printReport(dm, report); err != nil {
 			return err
@@ -112,36 +112,36 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	if j != nil {
 		j = append(j, '\n')
-		os.Stdout.Write(j)
+		Stdout.Write(j)
 		return nil
 	}
 
-	fmt.Printf("\nReport:\n")
-	fmt.Printf("\t* UDP: %v\n", report.UDP)
+	printf("\nReport:\n")
+	printf("\t* UDP: %v\n", report.UDP)
 	if report.GlobalV4 != "" {
-		fmt.Printf("\t* IPv4: yes, %v\n", report.GlobalV4)
+		printf("\t* IPv4: yes, %v\n", report.GlobalV4)
 	} else {
-		fmt.Printf("\t* IPv4: (no addr found)\n")
+		printf("\t* IPv4: (no addr found)\n")
 	}
 	if report.GlobalV6 != "" {
-		fmt.Printf("\t* IPv6: yes, %v\n", report.GlobalV6)
+		printf("\t* IPv6: yes, %v\n", report.GlobalV6)
 	} else if report.IPv6 {
-		fmt.Printf("\t* IPv6: (no addr found)\n")
+		printf("\t* IPv6: (no addr found)\n")
 	} else {
-		fmt.Printf("\t* IPv6: no\n")
+		printf("\t* IPv6: no\n")
 	}
-	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
-	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
-	fmt.Printf("\t* PortMapping: %v\n", portMapping(report))
+	printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
+	printf("\t* HairPinning: %v\n", report.HairPinning)
+	printf("\t* PortMapping: %v\n", portMapping(report))
 
 	// When DERP latency checking failed,
 	// magicsock will try to pick the DERP server that
 	// most of your other nodes are also using
 	if len(report.RegionLatency) == 0 {
-		fmt.Printf("\t* Nearest DERP: unknown (no response to latency probes)\n")
+		printf("\t* Nearest DERP: unknown (no response to latency probes)\n")
 	} else {
-		fmt.Printf("\t* Nearest DERP: %v\n", dm.Regions[report.PreferredDERP].RegionName)
-		fmt.Printf("\t* DERP latency:\n")
+		printf("\t* Nearest DERP: %v\n", dm.Regions[report.PreferredDERP].RegionName)
+		printf("\t* DERP latency:\n")
 		var rids []int
 		for rid := range dm.Regions {
 			rids = append(rids, rid)
@@ -168,7 +168,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 			if netcheckArgs.verbose {
 				derpNum = fmt.Sprintf("derp%d, ", rid)
 			}
-			fmt.Printf("\t\t- %3s: %-7s (%s%s)\n", r.RegionCode, latency, derpNum, r.RegionName)
+			printf("\t\t- %3s: %-7s (%s%s)\n", r.RegionCode, latency, derpNum, r.RegionName)
 		}
 	}
 	return nil

cmd/tailscale/cli/ping.go

@@ -45,7 +45,7 @@ relay node.
 `),
 	Exec: runPing,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("ping", flag.ExitOnError)
+		fs := newFlagSet("ping")
 		fs.BoolVar(&pingArgs.verbose, "verbose", false, "verbose output")
 		fs.BoolVar(&pingArgs.untilDirect, "until-direct", true, "stop once a direct path is established")
 		fs.BoolVar(&pingArgs.tsmp, "tsmp", false, "do a TSMP-level ping (through IP + wireguard, but not involving host OS stack)")
@@ -74,7 +74,7 @@ func runPing(ctx context.Context, args []string) error {
 	prc := make(chan *ipnstate.PingResult, 1)
 	bc.SetNotifyCallback(func(n ipn.Notify) {
 		if n.ErrMessage != nil {
-			log.Fatal(*n.ErrMessage)
+			fatalf("Notify.ErrMessage: %v", *n.ErrMessage)
 		}
 		if pr := n.PingResult; pr != nil && pr.IP == ip {
 			prc <- pr
@@ -89,7 +89,7 @@ func runPing(ctx context.Context, args []string) error {
 		return err
 	}
 	if self {
-		fmt.Printf("%v is local Tailscale IP\n", ip)
+		printf("%v is local Tailscale IP\n", ip)
 		return nil
 	}
 
@@ -105,14 +105,14 @@ func runPing(ctx context.Context, args []string) error {
 		timer := time.NewTimer(pingArgs.timeout)
 		select {
 		case <-timer.C:
-			fmt.Printf("timeout waiting for ping reply\n")
+			printf("timeout waiting for ping reply\n")
 		case err := <-pumpErr:
 			return err
 		case pr := <-prc:
 			timer.Stop()
 			if pr.Err != "" {
 				if pr.IsLocalIP {
-					fmt.Println(pr.Err)
+					outln(pr.Err)
 					return nil
 				}
 				return errors.New(pr.Err)
@@ -132,7 +132,7 @@ func runPing(ctx context.Context, args []string) error {
 			if pr.PeerAPIPort != 0 {
 				extra = fmt.Sprintf(", %d", pr.PeerAPIPort)
 			}
-			fmt.Printf("pong from %s (%s%s) via %v in %v\n", pr.NodeName, pr.NodeIP, extra, via, latency)
+			printf("pong from %s (%s%s) via %v in %v\n", pr.NodeName, pr.NodeIP, extra, via, latency)
 			if pingArgs.tsmp {
 				return nil
 			}

cmd/tailscale/cli/status.go

@@ -29,9 +29,24 @@ var statusCmd = &ffcli.Command{
 	Name:       "status",
 	ShortUsage: "status [--active] [--web] [--json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
-	Exec:       runStatus,
+	LongHelp: strings.TrimSpace(`
+
+JSON FORMAT
+
+Warning: this format has changed between releases and might change more
+in the future.
+
+For a description of the fields, see the "type Status" declaration at:
+
+https://github.com/tailscale/tailscale/blob/main/ipn/ipnstate/ipnstate.go
+
+(and be sure to select branch/tag that corresponds to the version
+ of Tailscale you're running)
+
+`),
+	Exec: runStatus,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("status", flag.ExitOnError)
+		fs := newFlagSet("status")
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 		fs.BoolVar(&statusArgs.web, "web", false, "run webserver with HTML showing status")
 		fs.BoolVar(&statusArgs.active, "active", false, "filter output to only peers with active sessions (not applicable to web mode)")
@@ -70,7 +85,7 @@ func runStatus(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s", j)
+		printf("%s", j)
 		return nil
 	}
 	if statusArgs.web {
@@ -79,7 +94,7 @@ func runStatus(ctx context.Context, args []string) error {
 			return err
 		}
 		statusURL := interfaces.HTTPOfListener(ln)
-		fmt.Printf("Serving Tailscale status at %v ...\n", statusURL)
+		printf("Serving Tailscale status at %v ...\n", statusURL)
 		go func() {
 			<-ctx.Done()
 			ln.Close()
@@ -108,30 +123,30 @@ func runStatus(ctx context.Context, args []string) error {
 
 	switch st.BackendState {
 	default:
-		fmt.Fprintf(os.Stderr, "unexpected state: %s\n", st.BackendState)
+		fmt.Fprintf(Stderr, "unexpected state: %s\n", st.BackendState)
 		os.Exit(1)
 	case ipn.Stopped.String():
-		fmt.Println("Tailscale is stopped.")
+		outln("Tailscale is stopped.")
 		os.Exit(1)
 	case ipn.NeedsLogin.String():
-		fmt.Println("Logged out.")
+		outln("Logged out.")
 		if st.AuthURL != "" {
-			fmt.Printf("\nLog in at: %s\n", st.AuthURL)
+			printf("\nLog in at: %s\n", st.AuthURL)
 		}
 		os.Exit(1)
 	case ipn.NeedsMachineAuth.String():
-		fmt.Println("Machine is not yet authorized by tailnet admin.")
+		outln("Machine is not yet authorized by tailnet admin.")
 		os.Exit(1)
 	case ipn.Running.String(), ipn.Starting.String():
 		// Run below.
 	}
 
 	if len(st.Health) > 0 {
-		fmt.Printf("# Health check:\n")
+		printf("# Health check:\n")
 		for _, m := range st.Health {
-			fmt.Printf("#     - %s\n", m)
+			printf("#     - %s\n", m)
 		}
-		fmt.Println()
+		outln()
 	}
 
 	var buf bytes.Buffer
@@ -145,11 +160,19 @@ func runStatus(ctx context.Context, args []string) error {
 		)
 		relay := ps.Relay
 		anyTraffic := ps.TxBytes != 0 || ps.RxBytes != 0
+		var offline string
+		if !ps.Online {
+			offline = "; offline"
+		}
 		if !ps.Active {
 			if ps.ExitNode {
-				f("idle; exit node")
+				f("idle; exit node" + offline)
+			} else if ps.ExitNodeOption {
+				f("idle; offers exit node" + offline)
 			} else if anyTraffic {
-				f("idle")
+				f("idle" + offline)
+			} else if !ps.Online {
+				f("offline")
 			} else {
 				f("-")
 			}
@@ -157,12 +180,17 @@ func runStatus(ctx context.Context, args []string) error {
 			f("active; ")
 			if ps.ExitNode {
 				f("exit node; ")
+			} else if ps.ExitNodeOption {
+				f("offers exit node; ")
 			}
 			if relay != "" && ps.CurAddr == "" {
 				f("relay %q", relay)
 			} else if ps.CurAddr != "" {
 				f("direct %s", ps.CurAddr)
 			}
+			if !ps.Online {
+				f("; offline")
+			}
 		}
 		if anyTraffic {
 			f(", tx %d rx %d", ps.TxBytes, ps.RxBytes)
@@ -190,7 +218,7 @@ func runStatus(ctx context.Context, args []string) error {
 			printPS(ps)
 		}
 	}
-	os.Stdout.Write(buf.Bytes())
+	Stdout.Write(buf.Bytes())
 	return nil
 }
 

cmd/tailscale/cli/up.go

@@ -6,6 +6,8 @@ package cli
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -28,6 +30,8 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
 
@@ -46,8 +50,10 @@ down").
 
 If flags are specified, the flags must be the complete set of desired
 settings. An error is returned if any setting would be changed as a
-result of an unspecified flag's default value, unless the --reset
-flag is also used.
+result of an unspecified flag's default value, unless the --reset flag
+is also used. (The flags --authkey, --force-reauth, and --qr are not
+considered settings that need to be re-specified when modifying
+settings.)
 `),
 	FlagSet: upFlagSet,
 	Exec:    runUp,
@@ -60,20 +66,34 @@ func effectiveGOOS() string {
 	return runtime.GOOS
 }
 
+// acceptRouteDefault returns the CLI's default value of --accept-routes as
+// a function of the platform it's running on.
+func acceptRouteDefault(goos string) bool {
+	switch goos {
+	case "windows":
+		return true
+	case "darwin":
+		return version.IsSandboxedMacOS()
+	default:
+		return false
+	}
+}
+
 var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
-	upf := flag.NewFlagSet("up", flag.ExitOnError)
+	upf := newFlagSet("up")
 
 	upf.BoolVar(&upArgs.qr, "qr", false, "show QR code for login URLs")
+	upf.BoolVar(&upArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 	upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
 	upf.BoolVar(&upArgs.reset, "reset", false, "reset unspecified settings to their default values")
 
 	upf.StringVar(&upArgs.server, "login-server", ipn.DefaultControlURL, "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
@@ -121,6 +141,7 @@ type upArgsT struct {
 	authKeyOrFile          string // "secret" or "file:/path/to/secret"
 	hostname               string
 	opUser                 string
+	json                   bool
 }
 
 func (a upArgsT) getAuthKey() (string, error) {
@@ -138,8 +159,35 @@ func (a upArgsT) getAuthKey() (string, error) {
 
 var upArgs upArgsT
 
+// Fields output when `tailscale up --json` is used. Two JSON blocks will be output.
+//
+// When "tailscale up" is run it first outputs a block with AuthURL and QR populated,
+// providing the link for where to authenticate this client. BackendState would be
+// valid but boring, as it will almost certainly be "NeedsLogin". Error would be
+// populated if something goes badly wrong.
+//
+// When the client is authenticated by having someone visit the AuthURL, a second
+// JSON block will be output. The AuthURL and QR fields will not be present, the
+// BackendState and Error fields will give the result of the authentication.
+// Ex:
+// {
+//    "AuthURL": "https://login.tailscale.com/a/0123456789abcdef",
+//    "QR": "data:image/png;base64,0123...cdef"
+//    "BackendState": "NeedsLogin"
+// }
+// {
+//    "BackendState": "Running"
+// }
+//
+type upOutputJSON struct {
+	AuthURL      string `json:",omitempty"` // Authentication URL of the form https://login.tailscale.com/a/0123456789
+	QR           string `json:",omitempty"` // a DataURL (base64) PNG of a QR code AuthURL
+	BackendState string `json:",omitempty"` // name of state like Running or NeedsMachineAuth
+	Error        string `json:",omitempty"` // description of an error
+}
+
 func warnf(format string, args ...interface{}) {
-	fmt.Printf("Warning: "+format+"\n", args...)
+	printf("Warning: "+format+"\n", args...)
 }
 
 var (
@@ -190,6 +238,65 @@ func calcAdvertiseRoutes(advertiseRoutes string, advertiseDefaultRoute bool) ([]
 	return routes, nil
 }
 
+// peerWithTailscaleIP returns the peer in st with the provided
+// Tailscale IP.
+func peerWithTailscaleIP(st *ipnstate.Status, ip netaddr.IP) (ps *ipnstate.PeerStatus, ok bool) {
+	for _, ps := range st.Peer {
+		for _, ip2 := range ps.TailscaleIPs {
+			if ip == ip2 {
+				return ps, true
+			}
+		}
+	}
+	return nil, false
+}
+
+// exitNodeIPOfArg maps from a user-provided CLI flag value to an IP
+// address they want to use as an exit node.
+func exitNodeIPOfArg(arg string, st *ipnstate.Status) (ip netaddr.IP, err error) {
+	if arg == "" {
+		return ip, errors.New("invalid use of exitNodeIPOfArg with empty string")
+	}
+	ip, err = netaddr.ParseIP(arg)
+	if err == nil {
+		// If we're online already and have a netmap, double check that the IP
+		// address specified is valid.
+		if st.BackendState == "Running" {
+			ps, ok := peerWithTailscaleIP(st, ip)
+			if !ok {
+				return ip, fmt.Errorf("no node found in netmap with IP %v", ip)
+			}
+			if !ps.ExitNodeOption {
+				return ip, fmt.Errorf("node %v is not advertising an exit node", ip)
+			}
+		}
+		return ip, err
+	}
+	match := 0
+	for _, ps := range st.Peer {
+		baseName := dnsname.TrimSuffix(ps.DNSName, st.MagicDNSSuffix)
+		if !strings.EqualFold(arg, baseName) {
+			continue
+		}
+		match++
+		if len(ps.TailscaleIPs) == 0 {
+			return ip, fmt.Errorf("node %q has no Tailscale IP?", arg)
+		}
+		if !ps.ExitNodeOption {
+			return ip, fmt.Errorf("node %q is not advertising an exit node", arg)
+		}
+		ip = ps.TailscaleIPs[0]
+	}
+	switch match {
+	case 0:
+		return ip, fmt.Errorf("invalid value %q for --exit-node; must be IP or unique node name", arg)
+	case 1:
+		return ip, nil
+	default:
+		return ip, fmt.Errorf("ambiguous exit node name %q", arg)
+	}
+}
+
 // prefsFromUpArgs returns the ipn.Prefs for the provided args.
 //
 // Note that the parameters upArgs and warnf are named intentionally
@@ -205,9 +312,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	var exitNodeIP netaddr.IP
 	if upArgs.exitNodeIP != "" {
 		var err error
-		exitNodeIP, err = netaddr.ParseIP(upArgs.exitNodeIP)
+		exitNodeIP, err = exitNodeIPOfArg(upArgs.exitNodeIP, st)
 		if err != nil {
-			return nil, fmt.Errorf("invalid IP address %q for --exit-node: %v", upArgs.exitNodeIP, err)
+			return nil, err
 		}
 	} else if upArgs.exitNodeAllowLANAccess {
 		return nil, fmt.Errorf("--exit-node-allow-lan-access can only be used with --exit-node")
@@ -277,7 +384,7 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 // It returns a non-nil justEditMP if we're already running and none of
 // the flags require a restart, so we can just do an EditPrefs call and
 // change the prefs at runtime (e.g. changing hostname, changing
-// advertised tags, routes, etc).
+// advertised routes, etc).
 //
 // It returns simpleUp if we're running a simple "tailscale up" to
 // transition to running from a previously-logged-in but down state,
@@ -297,6 +404,8 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, fmt.Errorf("can't change --login-server without --force-reauth")
 	}
 
+	tagsChanged := !reflect.DeepEqual(curPrefs.AdvertiseTags, prefs.AdvertiseTags)
+
 	simpleUp = env.flagSet.NFlag() == 0 &&
 		curPrefs.Persist != nil &&
 		curPrefs.Persist.LoginName != "" &&
@@ -306,7 +415,8 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		!env.upArgs.forceReauth &&
 		!env.upArgs.reset &&
 		env.upArgs.authKeyOrFile == "" &&
-		!controlURLChanged
+		!controlURLChanged &&
+		!tagsChanged
 	if justEdit {
 		justEditMP = new(ipn.MaskedPrefs)
 		justEditMP.WantRunningSet = true
@@ -377,11 +487,12 @@ func runUp(ctx context.Context, args []string) error {
 
 	env := upCheckEnv{
 		goos:          effectiveGOOS(),
+		distro:        distro.Get(),
 		user:          os.Getenv("USER"),
 		flagSet:       upFlagSet,
 		upArgs:        upArgs,
 		backendState:  st.BackendState,
-		curExitNodeIP: exitNodeIP(prefs, st),
+		curExitNodeIP: exitNodeIP(curPrefs, st),
 	}
 	simpleUp, justEditMP, err := updatePrefs(prefs, curPrefs, env)
 	if err != nil {
@@ -432,12 +543,18 @@ func runUp(ctx context.Context, args []string) error {
 				startLoginInteractive()
 			case ipn.NeedsMachineAuth:
 				printed = true
-				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
+				if env.upArgs.json {
+					printUpDoneJSON(ipn.NeedsMachineAuth, "")
+				} else {
+					fmt.Fprintf(Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
+				}
 			case ipn.Running:
 				// Done full authentication process
-				if printed {
+				if env.upArgs.json {
+					printUpDoneJSON(ipn.Running, "")
+				} else if printed {
 					// Only need to print an update if we printed the "please click" message earlier.
-					fmt.Fprintf(os.Stderr, "Success.\n")
+					fmt.Fprintf(Stderr, "Success.\n")
 				}
 				select {
 				case running <- true:
@@ -448,15 +565,33 @@ func runUp(ctx context.Context, args []string) error {
 		}
 		if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
 			printed = true
-			fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			if upArgs.qr {
+			if upArgs.json {
+				js := &upOutputJSON{AuthURL: *url, BackendState: st.BackendState}
+
 				q, err := qrcode.New(*url, qrcode.Medium)
-				if err != nil {
-					log.Printf("QR code error: %v", err)
-				} else {
-					fmt.Fprintf(os.Stderr, "%s\n", q.ToString(false))
+				if err == nil {
+					png, err := q.PNG(128)
+					if err == nil {
+						js.QR = "data:image/png;base64," + base64.StdEncoding.EncodeToString(png)
+					}
 				}
 
+				data, err := json.MarshalIndent(js, "", "\t")
+				if err != nil {
+					log.Printf("upOutputJSON marshalling error: %v", err)
+				} else {
+					fmt.Println(string(data))
+				}
+			} else {
+				fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+				if upArgs.qr {
+					q, err := qrcode.New(*url, qrcode.Medium)
+					if err != nil {
+						log.Printf("QR code error: %v", err)
+					} else {
+						fmt.Fprintf(Stderr, "%s\n", q.ToString(false))
+					}
+				}
 			}
 		}
 	})
@@ -543,6 +678,16 @@ func runUp(ctx context.Context, args []string) error {
 	}
 }
 
+func printUpDoneJSON(state ipn.State, errorString string) {
+	js := &upOutputJSON{BackendState: state.String(), Error: errorString}
+	data, err := json.MarshalIndent(js, "", "  ")
+	if err != nil {
+		log.Printf("printUpDoneJSON marshalling error: %v", err)
+	} else {
+		fmt.Println(string(data))
+	}
+}
+
 var (
 	prefsOfFlag = map[string][]string{} // "exit-node" => ExitNodeIP, ExitNodeID
 )
@@ -585,7 +730,7 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 // correspond to an ipn.Pref.
 func preflessFlag(flagName string) bool {
 	switch flagName {
-	case "authkey", "force-reauth", "reset", "qr":
+	case "authkey", "force-reauth", "reset", "qr", "json":
 		return true
 	}
 	return false
@@ -619,6 +764,7 @@ type upCheckEnv struct {
 	upArgs        upArgsT
 	backendState  string
 	curExitNodeIP netaddr.IP
+	distro        distro.Distro
 }
 
 // checkForAccidentalSettingReverts (the "up checker") checks for
@@ -669,6 +815,10 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck
 		if flagName == "login-server" && ipn.IsLoginServerSynonym(valCur) && ipn.IsLoginServerSynonym(valNew) {
 			continue
 		}
+		if flagName == "accept-routes" && valNew == false && env.goos == "linux" && env.distro == distro.Synology {
+			// Issue 3176. Old prefs had 'RouteAll: true' on disk, so ignore that.
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {

cmd/tailscale/cli/version.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -20,7 +19,7 @@ var versionCmd = &ffcli.Command{
 	ShortUsage: "version [flags]",
 	ShortHelp:  "Print Tailscale version",
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("version", flag.ExitOnError)
+		fs := newFlagSet("version")
 		fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version")
 		return fs
 	})(),
@@ -33,19 +32,19 @@ var versionArgs struct {
 
 func runVersion(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 	if !versionArgs.daemon {
-		fmt.Println(version.String())
+		outln(version.String())
 		return nil
 	}
 
-	fmt.Printf("Client: %s\n", version.String())
+	printf("Client: %s\n", version.String())
 
 	st, err := tailscale.StatusWithoutPeers(ctx)
 	if err != nil {
 		return err
 	}
-	fmt.Printf("Daemon: %s\n", st.Version)
+	printf("Daemon: %s\n", st.Version)
 	return nil
 }

cmd/tailscale/cli/web.go

@@ -76,7 +76,7 @@ Tailscale, as opposed to a CLI or a native app.
 `),
 
 	FlagSet: (func() *flag.FlagSet {
-		webf := flag.NewFlagSet("web", flag.ExitOnError)
+		webf := newFlagSet("web")
 		webf.StringVar(&webArgs.listen, "listen", "localhost:8088", "listen address; use port 0 for automatic")
 		webf.BoolVar(&webArgs.cgi, "cgi", false, "run as CGI script")
 		return webf
@@ -114,7 +114,7 @@ func tlsConfigFromEnvironment() *tls.Config {
 
 func runWeb(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 
 	if webArgs.cgi {

cmd/tailscale/depaware.txt

@@ -3,7 +3,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
+   L    github.com/klauspost/compress/flate                          from nhooyr.io/websocket
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli+
         github.com/peterbourgon/ff/v3                                from github.com/peterbourgon/ff/v3/ffcli
         github.com/peterbourgon/ff/v3/ffcli                          from tailscale.com/cmd/tailscale/cli
@@ -23,6 +25,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/cmd/tailscale/cli+
+   L    nhooyr.io/websocket                                          from tailscale.com/derp/derphttp+
+   L    nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+   L    nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
+        tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
@@ -30,6 +36,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
+   L    tailscale.com/derp/wsconn                                    from tailscale.com/derp/derphttp
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/hostinfo                                       from tailscale.com/net/interfaces
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
@@ -66,7 +73,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/structs                                  from tailscale.com/ipn+
-        tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
+        tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli
@@ -76,16 +83,16 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from crypto/tls+
         golang.org/x/crypto/hkdf                                     from crypto/tls
-        golang.org/x/crypto/nacl/box                                 from tailscale.com/derp+
+        golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/net/dns/dnsmessage                              from net
+        golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
@@ -97,7 +104,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from tailscale.com/net/netns+
    W    golang.org/x/sys/windows                                     from golang.org/x/sys/windows/registry+
-   W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg
+   W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
         golang.org/x/text/secure/bidirule                            from golang.org/x/net/idna
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
@@ -130,7 +137,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         crypto/tls                                                   from github.com/tcnksm/go-httpstat+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-        embed                                                        from tailscale.com/cmd/tailscale/cli
+        embed                                                        from tailscale.com/cmd/tailscale/cli+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+

cmd/tailscaled/debug.go

@@ -193,8 +193,8 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 		panic("unreachable")
 	}
 
-	priv1 := key.NewPrivate()
-	priv2 := key.NewPrivate()
+	priv1 := key.NewNode()
+	priv2 := key.NewNode()
 
 	c1 := derphttp.NewRegionClient(priv1, log.Printf, getRegion)
 	c2 := derphttp.NewRegionClient(priv2, log.Printf, getRegion)

cmd/tailscaled/depaware.txt

@@ -25,7 +25,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/aws-sdk-go-v2/credentials/stscreds            from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds                from github.com/aws/aws-sdk-go-v2/config+
    L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds
-   L    github.com/aws/aws-sdk-go-v2/internal/endpoints              from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
+   L    github.com/aws/aws-sdk-go-v2/internal/configsources          from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2           from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
    L    github.com/aws/aws-sdk-go-v2/internal/ini                    from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/internal/rand                   from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdk                    from github.com/aws/aws-sdk-go-v2/aws+
@@ -59,10 +60,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-        github.com/go-multierror/multierror                          from tailscale.com/cmd/tailscaled+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/google/btree                                      from inet.af/netstack/tcpip/header+
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
@@ -73,6 +74,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/wgengine/monitor
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/klauspost/compress                                from github.com/klauspost/compress/zstd
+   L    github.com/klauspost/compress/flate                          from nhooyr.io/websocket
         github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
@@ -91,32 +93,38 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/wgengine/router
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+   W 💣 golang.zx2c4.com/wintun                                      from golang.zx2c4.com/wireguard/tun
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
      💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
      💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
-   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+   W 💣 golang.zx2c4.com/wireguard/ipc/namedpipe                     from golang.zx2c4.com/wireguard/ipc
         golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
         golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device
      💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
-   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/cmd/tailscaled+
         inet.af/netaddr                                              from inet.af/wf+
         inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
      💣 inet.af/netstack/buffer                                      from inet.af/netstack/tcpip/stack
+        inet.af/netstack/context                                     from inet.af/netstack/refs+
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
         inet.af/netstack/linewriter                                  from inet.af/netstack/log
         inet.af/netstack/log                                         from inet.af/netstack/state+
         inet.af/netstack/rand                                        from inet.af/netstack/tcpip/network/hash+
+        inet.af/netstack/refs                                        from inet.af/netstack/refsvfs2
+        inet.af/netstack/refsvfs2                                    from inet.af/netstack/tcpip/stack
      💣 inet.af/netstack/sleep                                       from inet.af/netstack/tcpip/transport/tcp
      💣 inet.af/netstack/state                                       from inet.af/netstack/atomicbitops+
         inet.af/netstack/state/wire                                  from inet.af/netstack/state
@@ -127,6 +135,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/hash/jenkins                          from inet.af/netstack/tcpip/stack+
         inet.af/netstack/tcpip/header                                from inet.af/netstack/tcpip/header/parse+
         inet.af/netstack/tcpip/header/parse                          from inet.af/netstack/tcpip/network/ipv4+
+        inet.af/netstack/tcpip/internal/tcp                          from inet.af/netstack/tcpip/stack+
         inet.af/netstack/tcpip/link/channel                          from tailscale.com/wgengine/netstack
         inet.af/netstack/tcpip/network/hash                          from inet.af/netstack/tcpip/network/ipv4+
         inet.af/netstack/tcpip/network/internal/fragmentation        from inet.af/netstack/tcpip/network/ipv4+
@@ -136,7 +145,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/ports                                 from inet.af/netstack/tcpip/stack+
         inet.af/netstack/tcpip/seqnum                                from inet.af/netstack/tcpip/header+
      💣 inet.af/netstack/tcpip/stack                                 from inet.af/netstack/tcpip/adapters/gonet+
+        inet.af/netstack/tcpip/transport                             from inet.af/netstack/tcpip/transport/icmp+
         inet.af/netstack/tcpip/transport/icmp                        from tailscale.com/wgengine/netstack
+        inet.af/netstack/tcpip/transport/internal/network            from inet.af/netstack/tcpip/transport/icmp+
+        inet.af/netstack/tcpip/transport/internal/noop               from inet.af/netstack/tcpip/transport/raw
         inet.af/netstack/tcpip/transport/packet                      from inet.af/netstack/tcpip/transport/raw
         inet.af/netstack/tcpip/transport/raw                         from inet.af/netstack/tcpip/transport/icmp+
      💣 inet.af/netstack/tcpip/transport/tcp                         from inet.af/netstack/tcpip/adapters/gonet+
@@ -145,6 +157,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/waiter                                      from inet.af/netstack/tcpip+
         inet.af/peercred                                             from tailscale.com/ipn/ipnserver
    W 💣 inet.af/wf                                                   from tailscale.com/wf
+   L    nhooyr.io/websocket                                          from tailscale.com/derp/derphttp+
+   L    nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+   L    nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
+        tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
   LD    tailscale.com/chirp                                          from tailscale.com/cmd/tailscaled
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -153,6 +169,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/tailscaled+
+   L    tailscale.com/derp/wsconn                                    from tailscale.com/derp/derphttp
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/hostinfo                                       from tailscale.com/control/controlclient+
@@ -164,30 +181,31 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/ipn/store/aws                                  from tailscale.com/ipn/ipnserver
         tailscale.com/kube                                           from tailscale.com/ipn
-        tailscale.com/log/filelogger                                 from tailscale.com/ipn/ipnserver
+   W    tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled
-        tailscale.com/logtail                                        from tailscale.com/logpolicy
+        tailscale.com/logtail                                        from tailscale.com/logpolicy+
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
      💣 tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/net/dns/resolver                               from tailscale.com/net/dns+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
-        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
      💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock
-        tailscale.com/net/netknob                                    from tailscale.com/ipn/localapi+
+        tailscale.com/net/netknob                                    from tailscale.com/logpolicy+
         tailscale.com/net/netns                                      from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/netstat                                    from tailscale.com/ipn/ipnserver
         tailscale.com/net/packet                                     from tailscale.com/net/tstun+
         tailscale.com/net/portmapper                                 from tailscale.com/cmd/tailscaled+
-        tailscale.com/net/socks5                                     from tailscale.com/net/socks5/tssocks
-        tailscale.com/net/socks5/tssocks                             from tailscale.com/cmd/tailscaled
+        tailscale.com/net/proxymux                                   from tailscale.com/cmd/tailscaled
+        tailscale.com/net/socks5                                     from tailscale.com/cmd/tailscaled
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck+
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/tsdial                                     from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/cmd/tailscaled+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/paths                                          from tailscale.com/client/tailscale+
@@ -209,23 +227,25 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/netmap                                   from tailscale.com/control/controlclient+
         tailscale.com/types/nettype                                  from tailscale.com/wgengine/magicsock
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
-        tailscale.com/types/pad32                                    from tailscale.com/derp+
+        tailscale.com/types/pad32                                    from tailscale.com/derp
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
-        tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
+        tailscale.com/util/clientmetric                              from tailscale.com/ipn/localapi+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/dns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock
         tailscale.com/util/winutil                                   from tailscale.com/cmd/tailscaled+
+   W 💣 tailscale.com/util/winutil/vss                               from tailscale.com/util/winutil
         tailscale.com/version                                        from tailscale.com/client/tailscale+
         tailscale.com/version/distro                                 from tailscale.com/cmd/tailscaled+
    W    tailscale.com/wf                                             from tailscale.com/cmd/tailscaled
@@ -233,7 +253,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/cmd/tailscaled+
-        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled+
+        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -248,7 +268,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from crypto/tls+
         golang.org/x/crypto/hkdf                                     from crypto/tls
-        golang.org/x/crypto/nacl/box                                 from tailscale.com/derp+
+        golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
@@ -256,7 +276,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
-        golang.org/x/net/http2/hpack                                 from net/http
+        golang.org/x/net/http2                                       from golang.org/x/net/http2/h2c+
+        golang.org/x/net/http2/h2c                                   from tailscale.com/ipn/ipnlocal
+        golang.org/x/net/http2/hpack                                 from net/http+
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/ipv4                                        from golang.zx2c4.com/wireguard/device
         golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
@@ -278,7 +300,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/time/rate                                       from inet.af/netstack/tcpip/stack+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
-        compress/flate                                               from compress/gzip
+        compress/flate                                               from compress/gzip+
         compress/gzip                                                from internal/profile+
         container/heap                                               from inet.af/netstack/tcpip/transport/tcp
         container/list                                               from crypto/tls+
@@ -347,7 +369,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         path                                                         from github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
-        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints+
+        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints/v2+
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/klauspost/compress/zstd+
         runtime/pprof                                                from net/http/pprof+

cmd/tailscaled/tailscaled.go

@@ -20,6 +20,7 @@ import (
 	"net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
@@ -27,17 +28,23 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
+	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/netns"
-	"tailscale.com/net/socks5/tssocks"
+	"tailscale.com/net/proxymux"
+	"tailscale.com/net/socks5"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -78,6 +85,7 @@ var args struct {
 	debug          string
 	port           uint16
 	statepath      string
+	statedir       string
 	socketpath     string
 	birdSocketPath string
 	verbose        int
@@ -114,7 +122,8 @@ func main() {
 	flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM")
+	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "absolute path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM. If empty and --statedir is provided, the default is <statedir>/tailscaled.state")
+	flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -169,8 +178,7 @@ func main() {
 	osshare.SetFileSharingEnabled(false, logger.Discard)
 
 	if err != nil {
-		// No need to log; the func already did
-		os.Exit(1)
+		log.Fatal(err)
 	}
 }
 
@@ -202,6 +210,16 @@ func trySynologyMigration(p string) error {
 	return nil
 }
 
+func statePathOrDefault() string {
+	if args.statepath != "" {
+		return args.statepath
+	}
+	if args.statedir != "" {
+		return filepath.Join(args.statedir, "tailscaled.state")
+	}
+	return ""
+}
+
 func ipnServerOpts() (o ipnserver.Options) {
 	// Allow changing the OS-specific IPN behavior for tests
 	// so we can e.g. test Windows-specific behaviors on Linux.
@@ -210,9 +228,15 @@ func ipnServerOpts() (o ipnserver.Options) {
 		goos = runtime.GOOS
 	}
 
-	o.Port = 41112
-	o.StatePath = args.statepath
-	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+	o.VarRoot = args.statedir
+
+	// If an absolute --state is provided but not --statedir, try to derive
+	// a state directory.
+	if o.VarRoot == "" && filepath.IsAbs(args.statepath) {
+		if dir := filepath.Dir(args.statepath); strings.EqualFold(filepath.Base(dir), "tailscale") {
+			o.VarRoot = dir
+		}
+	}
 
 	switch goos {
 	default:
@@ -227,7 +251,7 @@ func ipnServerOpts() (o ipnserver.Options) {
 func run() error {
 	var err error
 
-	pol := logpolicy.New("tailnode.log.tailscale.io")
+	pol := logpolicy.New(logtail.CollectionNode)
 	pol.SetVerbosityLevel(args.verbose)
 	defer func() {
 		// Finish uploading logs after closing everything else.
@@ -261,10 +285,10 @@ func run() error {
 		return nil
 	}
 
-	if args.statepath == "" {
-		log.Fatalf("--state is required")
+	if args.statepath == "" && args.statedir == "" {
+		log.Fatalf("--statedir (or at least --state) is required")
 	}
-	if err := trySynologyMigration(args.statepath); err != nil {
+	if err := trySynologyMigration(statePathOrDefault()); err != nil {
 		log.Printf("error in synology migration: %v", err)
 	}
 
@@ -276,36 +300,55 @@ func run() error {
 
 	linkMon, err := monitor.New(logf)
 	if err != nil {
-		log.Fatalf("creating link monitor: %v", err)
+		return fmt.Errorf("monitor.New: %w", err)
 	}
 	pol.Logtail.SetLinkMonitor(linkMon)
 
-	socksListener := mustStartTCPListener("SOCKS5", args.socksAddr)
-	httpProxyListener := mustStartTCPListener("HTTP proxy", args.httpProxyAddr)
+	socksListener, httpProxyListener := mustStartProxyListeners(args.socksAddr, args.httpProxyAddr)
 
-	e, useNetstack, err := createEngine(logf, linkMon)
+	dialer := new(tsdial.Dialer) // mutated below (before used)
+	e, useNetstack, err := createEngine(logf, linkMon, dialer)
 	if err != nil {
-		logf("wgengine.New: %v", err)
-		return err
+		return fmt.Errorf("createEngine: %w", err)
+	}
+	if _, ok := e.(wgengine.ResolvingEngine).GetResolver(); !ok {
+		panic("internal error: exit node resolver not wired up")
 	}
 
-	var ns *netstack.Impl
-	if useNetstack || wrapNetstack {
-		onlySubnets := wrapNetstack && !useNetstack
-		ns = mustStartNetstack(logf, e, onlySubnets)
+	ns, err := newNetstack(logf, dialer, e)
+	if err != nil {
+		return fmt.Errorf("newNetstack: %w", err)
+	}
+	ns.ProcessLocalIPs = useNetstack
+	ns.ProcessSubnets = useNetstack || wrapNetstack
+	if err := ns.Start(); err != nil {
+		return fmt.Errorf("failed to start netstack: %w", err)
+	}
+
+	if useNetstack {
+		dialer.UseNetstackForIP = func(ip netaddr.IP) bool {
+			_, ok := e.PeerForIP(ip)
+			return ok
+		}
+		dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) {
+			return ns.DialContextTCP(ctx, dst)
+		}
 	}
 
 	if socksListener != nil || httpProxyListener != nil {
-		srv := tssocks.NewServer(logger.WithPrefix(logf, "socks5: "), e, ns)
 		if httpProxyListener != nil {
-			hs := &http.Server{Handler: httpProxyHandler(srv.Dialer)}
+			hs := &http.Server{Handler: httpProxyHandler(dialer.UserDial)}
 			go func() {
 				log.Fatalf("HTTP proxy exited: %v", hs.Serve(httpProxyListener))
 			}()
 		}
 		if socksListener != nil {
+			ss := &socks5.Server{
+				Logf:   logger.WithPrefix(logf, "socks5: "),
+				Dialer: dialer.UserDial,
+			}
 			go func() {
-				log.Fatalf("SOCKS5 server exited: %v", srv.Serve(socksListener))
+				log.Fatalf("SOCKS5 server exited: %v", ss.Serve(socksListener))
 			}()
 		}
 	}
@@ -332,32 +375,49 @@ func run() error {
 	}()
 
 	opts := ipnServerOpts()
-	opts.DebugMux = debugMux
-	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return fmt.Errorf("ipnserver.StateStore: %w", err)
+	}
+	srv, err := ipnserver.New(logf, pol.PublicID.String(), store, e, dialer, nil, opts)
+	if err != nil {
+		return fmt.Errorf("ipnserver.New: %w", err)
+	}
+
+	if debugMux != nil {
+		debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus)
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = srv.Run(ctx, ln)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
-		logf("ipnserver.Run: %v", err)
-		return err
+		return fmt.Errorf("ipnserver.Run: %w", err)
 	}
 
 	return nil
 }
 
-func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, useNetstack bool, err error) {
+func createEngine(logf logger.Logf, linkMon *monitor.Mon, dialer *tsdial.Dialer) (e wgengine.Engine, useNetstack bool, err error) {
 	if args.tunname == "" {
 		return nil, false, errors.New("no --tun value specified")
 	}
 	var errs []error
 	for _, name := range strings.Split(args.tunname, ",") {
 		logf("wgengine.NewUserspaceEngine(tun %q) ...", name)
-		e, useNetstack, err = tryEngine(logf, linkMon, name)
+		e, useNetstack, err = tryEngine(logf, linkMon, dialer, name)
 		if err == nil {
 			return e, useNetstack, nil
 		}
 		logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err)
 		errs = append(errs, err)
 	}
-	return nil, false, multierror.New(errs)
+	return nil, false, multierr.New(errs...)
 }
 
 var wrapNetstack = shouldWrapNetstack()
@@ -382,10 +442,11 @@ func shouldWrapNetstack() bool {
 	return false
 }
 
-func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, useNetstack bool, err error) {
+func tryEngine(logf logger.Logf, linkMon *monitor.Mon, dialer *tsdial.Dialer, name string) (e wgengine.Engine, useNetstack bool, err error) {
 	conf := wgengine.Config{
 		ListenPort:  args.port,
 		LinkMonitor: linkMon,
+		Dialer:      dialer,
 	}
 
 	useNetstack = name == "userspace-networking"
@@ -395,14 +456,14 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 		log.Printf("Connecting to BIRD at %s ...", args.birdSocketPath)
 		conf.BIRDClient, err = createBIRDClient(args.birdSocketPath)
 		if err != nil {
-			return nil, false, err
+			return nil, false, fmt.Errorf("createBIRDClient: %w", err)
 		}
 	}
 	if !useNetstack {
 		dev, devName, err := tstun.New(logf, name)
 		if err != nil {
 			tstun.Diagnose(logf, name)
-			return nil, false, err
+			return nil, false, fmt.Errorf("tstun.New(%q): %w", name, err)
 		}
 		conf.Tun = dev
 		if strings.HasPrefix(name, "tap:") {
@@ -414,11 +475,11 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
-			return nil, false, err
+			return nil, false, fmt.Errorf("creating router: %w", err)
 		}
 		d, err := dns.NewOSConfigurator(logf, devName)
 		if err != nil {
-			return nil, false, err
+			return nil, false, fmt.Errorf("dns.NewOSConfigurator: %w", err)
 		}
 		conf.DNS = d
 		conf.Router = r
@@ -435,6 +496,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 
 func newDebugMux() *http.ServeMux {
 	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/metrics", servePrometheusMetrics)
 	mux.HandleFunc("/debug/pprof/", pprof.Index)
 	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
@@ -443,6 +505,11 @@ func newDebugMux() *http.ServeMux {
 	return mux
 }
 
+func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 func runDebugServer(mux *http.ServeMux, addr string) {
 	srv := &http.Server{
 		Addr:    addr,
@@ -453,33 +520,54 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 	}
 }
 
-func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) *netstack.Impl {
+func newNetstack(logf logger.Logf, dialer *tsdial.Dialer, e wgengine.Engine) (*netstack.Impl, error) {
 	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
 	if !ok {
-		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+		return nil, fmt.Errorf("%T is not a wgengine.InternalsGetter", e)
 	}
-	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
-	if err != nil {
-		log.Fatalf("netstack.Create: %v", err)
-	}
-	if err := ns.Start(); err != nil {
-		log.Fatalf("failed to start netstack: %v", err)
-	}
-	return ns
+	return netstack.Create(logf, tunDev, e, magicConn, dialer)
 }
 
-func mustStartTCPListener(name, addr string) net.Listener {
-	if addr == "" {
-		return nil
+// mustStartProxyListeners creates listeners for local SOCKS and HTTP
+// proxies, if the respective addresses are not empty. socksAddr and
+// httpAddr can be the same, in which case socksListener will receive
+// connections that look like they're speaking SOCKS and httpListener
+// will receive everything else.
+//
+// socksListener and httpListener can be nil, if their respective
+// addrs are empty.
+func mustStartProxyListeners(socksAddr, httpAddr string) (socksListener, httpListener net.Listener) {
+	if socksAddr == httpAddr && socksAddr != "" && !strings.HasSuffix(socksAddr, ":0") {
+		ln, err := net.Listen("tcp", socksAddr)
+		if err != nil {
+			log.Fatalf("proxy listener: %v", err)
+		}
+		return proxymux.SplitSOCKSAndHTTP(ln)
 	}
-	ln, err := net.Listen("tcp", addr)
-	if err != nil {
-		log.Fatalf("%v listener: %v", name, err)
+
+	var err error
+	if socksAddr != "" {
+		socksListener, err = net.Listen("tcp", socksAddr)
+		if err != nil {
+			log.Fatalf("SOCKS5 listener: %v", err)
+		}
+		if strings.HasSuffix(socksAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("SOCKS5 listening on %v", socksListener.Addr())
+		}
 	}
-	if strings.HasSuffix(addr, ":0") {
-		// Log kernel-selected port number so integration tests
-		// can find it portably.
-		log.Printf("%v listening on %v", name, ln.Addr())
+	if httpAddr != "" {
+		httpListener, err = net.Listen("tcp", httpAddr)
+		if err != nil {
+			log.Fatalf("HTTP proxy listener: %v", err)
+		}
+		if strings.HasSuffix(httpAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("HTTP proxy listening on %v", httpListener.Addr())
+		}
 	}
-	return ln
+
+	return socksListener, httpListener
 }

cmd/tailscaled/tailscaled_windows.go

@@ -32,12 +32,15 @@ import (
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/net/tstun"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/wf"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
 )
@@ -74,7 +77,14 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	go func() {
 		defer close(doneCh)
 		args := []string{"/subproc", service.Policy.PublicID.String()}
-		ipnserver.BabysitProc(ctx, args, log.Printf)
+		// Make a logger without a date prefix, as filelogger
+		// and logtail both already add their own. All we really want
+		// from the log package is the automatic newline.
+		// We start with log.Default().Writer(), which is the logtail
+		// writer that logpolicy already installed as the global
+		// output.
+		logger := log.New(log.Default().Writer(), "", 0)
+		ipnserver.BabysitProc(ctx, args, logger.Printf)
 	}()
 
 	changes <- svc.Status{State: svc.Running, Accepts: svcAccepts}
@@ -109,6 +119,9 @@ func beWindowsSubprocess() bool {
 	}
 	logid := os.Args[2]
 
+	// Remove the date/time prefix; the logtail + file logggers add it.
+	log.SetFlags(0)
+
 	log.Printf("Program starting: v%v: %#v", version.Long, os.Args)
 	log.Printf("subproc mode: logid=%v", logid)
 
@@ -172,6 +185,12 @@ func beFirewallKillswitch() bool {
 func startIPNServer(ctx context.Context, logid string) error {
 	var logf logger.Logf = log.Printf
 
+	linkMon, err := monitor.New(logf)
+	if err != nil {
+		return err
+	}
+	dialer := new(tsdial.Dialer)
+
 	getEngineRaw := func() (wgengine.Engine, error) {
 		dev, devName, err := tstun.New(logf, "Tailscale")
 		if err != nil {
@@ -192,19 +211,26 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("DNS: %w", err)
 		}
 		eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
-			Tun:        dev,
-			Router:     r,
-			DNS:        d,
-			ListenPort: 41641,
+			Tun:         dev,
+			Router:      r,
+			DNS:         d,
+			ListenPort:  41641,
+			LinkMonitor: linkMon,
+			Dialer:      dialer,
 		})
 		if err != nil {
 			r.Close()
 			dev.Close()
 			return nil, fmt.Errorf("engine: %w", err)
 		}
-		onlySubnets := true
-		if wrapNetstack {
-			mustStartNetstack(logf, eng, onlySubnets)
+		ns, err := newNetstack(logf, dialer, eng)
+		if err != nil {
+			return nil, fmt.Errorf("newNetstack: %w", err)
+		}
+		ns.ProcessLocalIPs = false
+		ns.ProcessSubnets = wrapNetstack
+		if err := ns.Start(); err != nil {
+			return nil, fmt.Errorf("failed to start netstack: %w", err)
 		}
 		return wgengine.NewWatchdog(eng), nil
 	}
@@ -266,7 +292,18 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = ipnserver.Run(ctx, logf, ln, store, linkMon, dialer, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -157,8 +157,9 @@ func handleSSH(s ssh.Session) {
 		cmd.Process.Kill()
 		if err := cmd.Wait(); err != nil {
 			s.Exit(1)
+		} else {
+			s.Exit(0)
 		}
-		s.Exit(0)
 		return
 	}
 

control/controlclient/auto.go

@@ -14,11 +14,11 @@ import (
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/structs"
-	"tailscale.com/types/wgkey"
 )
 
 type LoginGoal struct {
@@ -281,7 +281,6 @@ func (c *Auto) authRoutine() {
 
 		report := func(err error, msg string) {
 			c.logf("[v1] %s: %v", msg, err)
-			err = fmt.Errorf("%s: %v", msg, err)
 			// don't send status updates for context errors,
 			// since context cancelation is always on purpose.
 			if ctx.Err() == nil {
@@ -340,11 +339,9 @@ func (c *Auto) authRoutine() {
 				continue
 			}
 			if url != "" {
-				if goal.url != "" {
-					err = fmt.Errorf("[unexpected] server required a new URL?")
-					report(err, "WaitLoginURL")
-				}
-
+				// goal.url ought to be empty here.
+				// However, not all control servers get this right,
+				// and logging about it here just generates noise.
 				c.mu.Lock()
 				c.loginGoal = &LoginGoal{
 					wantLoggedIn: true,
@@ -431,7 +428,7 @@ func (c *Auto) mapRoutine() {
 
 		report := func(err error, msg string) {
 			c.logf("[v1] %s: %v", msg, err)
-			err = fmt.Errorf("%s: %v", msg, err)
+			err = fmt.Errorf("%s: %w", msg, err)
 			// don't send status updates for context errors,
 			// since context cancelation is always on purpose.
 			if ctx.Err() == nil {
@@ -599,9 +596,7 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM
 		NetMap:         nm,
 		Hostinfo:       hi,
 		State:          state,
-	}
-	if err != nil {
-		new.Err = err.Error()
+		Err:            err,
 	}
 	if statusFunc != nil {
 		statusFunc(new)
@@ -702,7 +697,7 @@ func (c *Auto) Shutdown() {
 
 // NodePublicKey returns the node public key currently in use. This is
 // used exclusively in tests.
-func (c *Auto) TestOnlyNodePublicKey() wgkey.Key {
+func (c *Auto) TestOnlyNodePublicKey() key.NodePublic {
 	priv := c.direct.GetPersist()
 	return priv.PrivateNodeKey.Public()
 }

control/controlclient/client.go

@@ -20,6 +20,7 @@ type LoginFlags int
 const (
 	LoginDefault     = LoginFlags(0)
 	LoginInteractive = LoginFlags(1 << iota) // force user login and key refresh
+	LoginEphemeral                           // set RegisterRequest.Ephemeral
 )
 
 // Client represents a client connection to the control server.
@@ -78,3 +79,9 @@ type Client interface {
 	// requesting a DNS record be created or updated.
 	SetDNS(context.Context, *tailcfg.SetDNSRequest) error
 }
+
+// UserVisibleError is an error that should be shown to users.
+type UserVisibleError string
+
+func (e UserVisibleError) Error() string            { return string(e) }
+func (e UserVisibleError) UserVisibleError() string { return string(e) }

control/controlclient/direct.go

@@ -46,7 +46,7 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
-	"tailscale.com/types/wgkey"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/systemd"
 	"tailscale.com/wgengine/monitor"
 )
@@ -61,7 +61,7 @@ type Direct struct {
 	keepAlive              bool
 	logf                   logger.Logf
 	linkMon                *monitor.Mon // or nil
-	discoPubKey            tailcfg.DiscoKey
+	discoPubKey            key.DiscoPublic
 	getMachinePrivKey      func() (key.MachinePrivate, error)
 	debugFlags             []string
 	keepSharerAndUserSplit bool
@@ -72,7 +72,7 @@ type Direct struct {
 	serverKey    key.MachinePublic
 	persist      persist.Persist
 	authKey      string
-	tryingNewKey wgkey.Private
+	tryingNewKey key.NodePrivate
 	expiry       *time.Time
 	// hostinfo is mutated in-place while mu is held.
 	hostinfo      *tailcfg.Hostinfo // always non-nil
@@ -89,7 +89,7 @@ type Options struct {
 	AuthKey              string                             // optional node auth key for auto registration
 	TimeNow              func() time.Time                   // time.Now implementation used by Client
 	Hostinfo             *tailcfg.Hostinfo                  // non-nil passes ownership, nil means to use default using os.Hostname, etc
-	DiscoPublicKey       tailcfg.DiscoKey
+	DiscoPublicKey       key.DiscoPublic
 	NewDecompressor      func() (Decompressor, error)
 	KeepAlive            bool
 	Logf                 logger.Logf
@@ -147,13 +147,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	}
 
 	httpc := opts.HTTPTestClient
+	if httpc == nil && runtime.GOOS == "js" {
+		// In js/wasm, net/http.Transport (as of Go 1.18) will
+		// only use the browser's Fetch API if you're using
+		// the DefaultClient (or a client without dial hooks
+		// etc set).
+		httpc = http.DefaultClient
+	}
 	if httpc == nil {
 		dnsCache := &dnscache.Resolver{
 			Forward:          dnscache.Get().Forward, // use default cache's forwarder
 			UseLastGood:      true,
 			LookupIPFallback: dnsfallback.Lookup,
 		}
-		dialer := netns.NewDialer()
+		dialer := netns.NewDialer(opts.Logf)
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
@@ -285,8 +292,8 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -327,7 +334,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		c.mu.Unlock()
 	}
 
-	var oldNodeKey wgkey.Key
+	var oldNodeKey key.NodePublic
 	switch {
 	case opt.Logout:
 		tryingNewKey = persist.PrivateNodeKey
@@ -336,12 +343,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	case regen || persist.PrivateNodeKey.IsZero():
 		c.logf("Generating a new nodekey.")
 		persist.OldPrivateNodeKey = persist.PrivateNodeKey
-		key, err := wgkey.NewPrivate()
-		if err != nil {
-			c.logf("login keygen: %v", err)
-			return regen, opt.URL, err
-		}
-		tryingNewKey = key
+		tryingNewKey = key.NewNode()
 	default:
 		// Try refreshing the current key first
 		tryingNewKey = persist.PrivateNodeKey
@@ -363,11 +365,12 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	now := time.Now().Round(time.Second)
 	request := tailcfg.RegisterRequest{
 		Version:    1,
-		OldNodeKey: tailcfg.NodeKey(oldNodeKey),
-		NodeKey:    tailcfg.NodeKey(tryingNewKey.Public()),
-		Hostinfo:   hostinfo,
+		OldNodeKey: oldNodeKey,
+		NodeKey:    tryingNewKey.Public(),
+		Hostinfo:   hi,
 		Followup:   opt.URL,
 		Timestamp:  &now,
+		Ephemeral:  (opt.Flags & LoginEphemeral) != 0,
 	}
 	if opt.Logout {
 		request.Expiry = time.Unix(123, 0) // far in the past
@@ -435,6 +438,9 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	c.logf("RegisterReq: got response; nodeKeyExpired=%v, machineAuthorized=%v; authURL=%v",
 		resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
 
+	if resp.Error != "" {
+		return false, "", UserVisibleError(resp.Error)
+	}
 	if resp.NodeKeyExpired {
 		if regen {
 			return true, "", fmt.Errorf("weird: regen=true but server says NodeKeyExpired: %v", request.NodeKey)
@@ -553,12 +559,21 @@ const pollTimeout = 120 * time.Second
 
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	metricMapRequests.Add(1)
+	metricMapRequestsActive.Add(1)
+	defer metricMapRequestsActive.Add(-1)
+	if maxPolls == -1 {
+		metricMapRequestsPoll.Add(1)
+	} else {
+		metricMapRequestsLite.Add(1)
+	}
+
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	localPort := c.localPort
 	var epStrs []string
 	var epTypes []tailcfg.EndpointType
@@ -597,18 +612,18 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	request := &tailcfg.MapRequest{
 		Version:       tailcfg.CurrentMapRequestVersion,
 		KeepAlive:     c.keepAlive,
-		NodeKey:       tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
+		NodeKey:       persist.PrivateNodeKey.Public(),
 		DiscoKey:      c.discoPubKey,
 		Endpoints:     epStrs,
 		EndpointTypes: epTypes,
 		Stream:        allowStream,
-		Hostinfo:      hostinfo,
+		Hostinfo:      hi,
 		DebugFlags:    c.debugFlags,
 		OmitPeers:     cb == nil,
 	}
 	var extraDebugFlags []string
-	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
-		ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
+	if hi != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
+		ipForwardingBroken(hi.RoutableIPs, c.linkMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
 	if health.RouterHealth() != nil {
@@ -617,6 +632,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	if health.NetworkCategoryHealth() != nil {
 		extraDebugFlags = append(extraDebugFlags, "warn-network-category-unhealthy")
 	}
+	if hostinfo.DisabledEtcAptSource() {
+		extraDebugFlags = append(extraDebugFlags, "warn-etc-apt-source-disabled")
+	}
 	if len(extraDebugFlags) > 0 {
 		old := request.DebugFlags
 		request.DebugFlags = append(old[:len(old):len(old)], extraDebugFlags...)
@@ -739,11 +757,14 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return err
 		}
 
+		metricMapResponseMessages.Add(1)
+
 		if allowStream {
 			health.GotStreamedMapResponse()
 		}
 
 		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+			metricMapResponsePings.Add(1)
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -760,13 +781,23 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return ctx.Err()
 		}
 		if resp.KeepAlive {
+			metricMapResponseKeepAlives.Add(1)
 			continue
 		}
 
+		metricMapResponseMap.Add(1)
+		if i > 0 {
+			metricMapResponseMapDelta.Add(1)
+		}
+
 		hasDebug := resp.Debug != nil
 		// being conservative here, if Debug not present set to False
 		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
 		if hasDebug {
+			if code := resp.Debug.Exit; code != nil {
+				c.logf("exiting process with status %v per controlplane", *code)
+				os.Exit(*code)
+			}
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}
@@ -1169,7 +1200,13 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 
 // SetDNS sends the SetDNSRequest request to the control plane server,
 // requesting a DNS record be created or updated.
-func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) (err error) {
+	metricSetDNS.Add(1)
+	defer func() {
+		if err != nil {
+			metricSetDNSError.Add(1)
+		}
+	}()
 	c.mu.Lock()
 	serverKey := c.serverKey
 	c.mu.Unlock()
@@ -1269,3 +1306,20 @@ func postPingResult(now time.Time, logf logger.Logf, c *http.Client, pr *tailcfg
 	}
 	return nil
 }
+
+var (
+	metricMapRequestsActive = clientmetric.NewGauge("controlclient_map_requests_active")
+
+	metricMapRequests     = clientmetric.NewCounter("controlclient_map_requests")
+	metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite")
+	metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll")
+
+	metricMapResponseMessages   = clientmetric.NewCounter("controlclient_map_response_message") // any message type
+	metricMapResponsePings      = clientmetric.NewCounter("controlclient_map_response_ping")
+	metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive")
+	metricMapResponseMap        = clientmetric.NewCounter("controlclient_map_response_map")       // any non-keepalive map response
+	metricMapResponseMapDelta   = clientmetric.NewCounter("controlclient_map_response_map_delta") // 2nd+ non-keepalive map response
+
+	metricSetDNS      = clientmetric.NewCounter("controlclient_setdns")
+	metricSetDNSError = clientmetric.NewCounter("controlclient_setdns_error")
+)

control/controlclient/map.go

@@ -15,7 +15,6 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
-	"tailscale.com/types/wgkey"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -29,7 +28,7 @@ import (
 // one MapRequest).
 type mapSession struct {
 	// Immutable fields.
-	privateNodeKey         wgkey.Private
+	privateNodeKey         key.NodePrivate
 	logf                   logger.Logf
 	vlogf                  logger.Logf
 	machinePubKey          key.MachinePublic
@@ -51,7 +50,7 @@ type mapSession struct {
 	netMapBuilding *netmap.NetworkMap
 }
 
-func newMapSession(privateNodeKey wgkey.Private) *mapSession {
+func newMapSession(privateNodeKey key.NodePrivate) *mapSession {
 	ms := &mapSession{
 		privateNodeKey:  privateNodeKey,
 		logf:            logger.Discard,
@@ -111,7 +110,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 	}
 
 	nm := &netmap.NetworkMap{
-		NodeKey:         tailcfg.NodeKey(ms.privateNodeKey.Public()),
+		NodeKey:         ms.privateNodeKey.Public(),
 		PrivateKey:      ms.privateNodeKey,
 		MachineKey:      ms.machinePubKey,
 		Peers:           resp.Peers,

control/controlclient/map_test.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
-	"tailscale.com/types/wgkey"
 )
 
 func TestUndeltaPeers(t *testing.T) {
@@ -170,11 +170,7 @@ func formatNodes(nodes []*tailcfg.Node) string {
 }
 
 func newTestMapSession(t *testing.T) *mapSession {
-	k, err := wgkey.NewPrivate()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return newMapSession(k)
+	return newMapSession(key.NewNode())
 }
 
 func TestNetmapForResponse(t *testing.T) {

control/controlclient/status.go

@@ -67,7 +67,7 @@ type Status struct {
 	_              structs.Incomparable
 	LoginFinished  *empty.Message // nonempty when login finishes
 	LogoutFinished *empty.Message // nonempty when logout finishes
-	Err            string
+	Err            error
 	URL            string             // interactive URL to visit to finish logging in
 	NetMap         *netmap.NetworkMap // server-pushed configuration
 

control/noise/conn.go

@@ -0,0 +1,359 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package noise implements the base transport of the Tailscale 2021
+// control protocol.
+//
+// The base transport implements Noise IK, instantiated with
+// Curve25519, ChaCha20Poly1305 and BLAKE2s.
+package noise
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"tailscale.com/types/key"
+)
+
+const (
+	// maxMessageSize is the maximum size of a protocol frame on the
+	// wire, including header and payload.
+	maxMessageSize = 4096
+	// maxCiphertextSize is the maximum amount of ciphertext bytes
+	// that one protocol frame can carry, after framing.
+	maxCiphertextSize = maxMessageSize - 3
+	// maxPlaintextSize is the maximum amount of plaintext bytes that
+	// one protocol frame can carry, after encryption and framing.
+	maxPlaintextSize = maxCiphertextSize - chp.Overhead
+)
+
+// A Conn is a secured Noise connection. It implements the net.Conn
+// interface, with the unusual trait that any write error (including a
+// SetWriteDeadline induced i/o timeout) causes all future writes to
+// fail.
+type Conn struct {
+	conn          net.Conn
+	version       uint16
+	peer          key.MachinePublic
+	handshakeHash [blake2s.Size]byte
+	rx            rxState
+	tx            txState
+}
+
+// rxState is all the Conn state that Read uses.
+type rxState struct {
+	sync.Mutex
+	cipher    cipher.AEAD
+	nonce     nonce
+	buf       [maxMessageSize]byte
+	n         int    // number of valid bytes in buf
+	next      int    // offset of next undecrypted packet
+	plaintext []byte // slice into buf of decrypted bytes
+}
+
+// txState is all the Conn state that Write uses.
+type txState struct {
+	sync.Mutex
+	cipher cipher.AEAD
+	nonce  nonce
+	buf    [maxMessageSize]byte
+	err    error // records the first partial write error for all future calls
+}
+
+// ProtocolVersion returns the protocol version that was used to
+// establish this Conn.
+func (c *Conn) ProtocolVersion() int {
+	return int(c.version)
+}
+
+// HandshakeHash returns the Noise handshake hash for the connection,
+// which can be used to bind other messages to this connection
+// (i.e. to ensure that the message wasn't replayed from a different
+// connection).
+func (c *Conn) HandshakeHash() [blake2s.Size]byte {
+	return c.handshakeHash
+}
+
+// Peer returns the peer's long-term public key.
+func (c *Conn) Peer() key.MachinePublic {
+	return c.peer
+}
+
+// readNLocked reads into c.rx.buf until buf contains at least total
+// bytes. Returns a slice of the total bytes in rxBuf, or an
+// error if fewer than total bytes are available.
+func (c *Conn) readNLocked(total int) ([]byte, error) {
+	if total > maxMessageSize {
+		return nil, errReadTooBig{total}
+	}
+	for {
+		if total <= c.rx.n {
+			return c.rx.buf[:total], nil
+		}
+
+		n, err := c.conn.Read(c.rx.buf[c.rx.n:])
+		c.rx.n += n
+		if err != nil {
+			return nil, err
+		}
+	}
+}
+
+// decryptLocked decrypts msg (which is header+ciphertext) in-place
+// and sets c.rx.plaintext to the decrypted bytes.
+func (c *Conn) decryptLocked(msg []byte) (err error) {
+	if msgType := msg[0]; msgType != msgTypeRecord {
+		return fmt.Errorf("received message with unexpected type %d, want %d", msgType, msgTypeRecord)
+	}
+	// We don't check the length field here, because the caller
+	// already did in order to figure out how big the msg slice should
+	// be.
+	ciphertext := msg[headerLen:]
+
+	if !c.rx.nonce.Valid() {
+		return errCipherExhausted{}
+	}
+
+	c.rx.plaintext, err = c.rx.cipher.Open(ciphertext[:0], c.rx.nonce[:], ciphertext, nil)
+	c.rx.nonce.Increment()
+
+	if err != nil {
+		// Once a decryption has failed, our Conn is no longer
+		// synchronized with our peer. Nuke the cipher state to be
+		// safe, so that no further decryptions are attempted. Future
+		// read attempts will return net.ErrClosed.
+		c.rx.cipher = nil
+	}
+	return err
+}
+
+// encryptLocked encrypts plaintext into c.tx.buf (including the
+// packet header) and returns a slice of the ciphertext, or an error
+// if the cipher is exhausted (i.e. can no longer be used safely).
+func (c *Conn) encryptLocked(plaintext []byte) ([]byte, error) {
+	if !c.tx.nonce.Valid() {
+		// Received 2^64-1 messages on this cipher state. Connection
+		// is no longer usable.
+		return nil, errCipherExhausted{}
+	}
+
+	c.tx.buf[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(c.tx.buf[1:headerLen], uint16(len(plaintext)+chp.Overhead))
+	ret := c.tx.cipher.Seal(c.tx.buf[:headerLen], c.tx.nonce[:], plaintext, nil)
+	c.tx.nonce.Increment()
+
+	return ret, nil
+}
+
+// wholeMessageLocked returns a slice of one whole Noise transport
+// message from c.rx.buf, if one whole message is available, and
+// advances the read state to the next Noise message in the
+// buffer. Returns nil without advancing read state if there isn't one
+// whole message in c.rx.buf.
+func (c *Conn) wholeMessageLocked() []byte {
+	available := c.rx.n - c.rx.next
+	if available < headerLen {
+		return nil
+	}
+	bs := c.rx.buf[c.rx.next:c.rx.n]
+	totalSize := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	if len(bs) < totalSize {
+		return nil
+	}
+	c.rx.next += totalSize
+	return bs[:totalSize]
+}
+
+// decryptOneLocked decrypts one Noise transport message, reading from
+// c.conn as needed, and sets c.rx.plaintext to point to the decrypted
+// bytes. c.rx.plaintext is only valid if err == nil.
+func (c *Conn) decryptOneLocked() error {
+	c.rx.plaintext = nil
+
+	// Fast path: do we have one whole ciphertext frame buffered
+	// already?
+	if bs := c.wholeMessageLocked(); bs != nil {
+		return c.decryptLocked(bs)
+	}
+
+	if c.rx.next != 0 {
+		// To simplify the read logic, move the remainder of the
+		// buffered bytes back to the head of the buffer, so we can
+		// grow it without worrying about wraparound.
+		c.rx.n = copy(c.rx.buf[:], c.rx.buf[c.rx.next:c.rx.n])
+		c.rx.next = 0
+	}
+
+	bs, err := c.readNLocked(headerLen)
+	if err != nil {
+		return err
+	}
+	// The rest of the header (besides the length field) gets verified
+	// in decryptLocked, not here.
+	messageLen := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	bs, err = c.readNLocked(messageLen)
+	if err != nil {
+		return err
+	}
+
+	c.rx.next = len(bs)
+
+	return c.decryptLocked(bs)
+}
+
+// Read implements io.Reader.
+func (c *Conn) Read(bs []byte) (int, error) {
+	c.rx.Lock()
+	defer c.rx.Unlock()
+
+	if c.rx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+	// If no plaintext is buffered, decrypt incoming frames until we
+	// have some plaintext. Zero-byte Noise frames are allowed in this
+	// protocol, which is why we have to loop here rather than decrypt
+	// a single additional frame.
+	for len(c.rx.plaintext) == 0 {
+		if err := c.decryptOneLocked(); err != nil {
+			return 0, err
+		}
+	}
+	n := copy(bs, c.rx.plaintext)
+	c.rx.plaintext = c.rx.plaintext[n:]
+	return n, nil
+}
+
+// Write implements io.Writer.
+func (c *Conn) Write(bs []byte) (n int, err error) {
+	c.tx.Lock()
+	defer c.tx.Unlock()
+
+	if c.tx.err != nil {
+		return 0, c.tx.err
+	}
+	defer func() {
+		if err != nil {
+			// All write errors are fatal for this conn, so clear the
+			// cipher state whenever an error happens.
+			c.tx.cipher = nil
+		}
+		if c.tx.err == nil {
+			// Only set c.tx.err if not nil so that we can return one
+			// error on the first failure, and a different one for
+			// subsequent calls. See the error handling around Write
+			// below for why.
+			c.tx.err = err
+		}
+	}()
+
+	if c.tx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+
+	var sent int
+	for len(bs) > 0 {
+		toSend := bs
+		if len(toSend) > maxPlaintextSize {
+			toSend = bs[:maxPlaintextSize]
+		}
+		bs = bs[len(toSend):]
+
+		ciphertext, err := c.encryptLocked(toSend)
+		if err != nil {
+			return 0, err
+		}
+
+		n, err := c.conn.Write(ciphertext)
+		sent += n
+		if err != nil {
+			// Return the raw error on the Write that actually
+			// failed. For future writes, return that error wrapped in
+			// a desync error.
+			c.tx.err = errPartialWrite{err}
+			return sent, err
+		}
+	}
+	return sent, nil
+}
+
+// Close implements io.Closer.
+func (c *Conn) Close() error {
+	closeErr := c.conn.Close() // unblocks any waiting reads or writes
+
+	// Remove references to live cipher state. Strictly speaking this
+	// is unnecessary, but we want to try and hand the active cipher
+	// state to the garbage collector promptly, to preserve perfect
+	// forward secrecy as much as we can.
+	c.rx.Lock()
+	c.rx.cipher = nil
+	c.rx.Unlock()
+	c.tx.Lock()
+	c.tx.cipher = nil
+	c.tx.Unlock()
+	return closeErr
+}
+
+func (c *Conn) LocalAddr() net.Addr                { return c.conn.LocalAddr() }
+func (c *Conn) RemoteAddr() net.Addr               { return c.conn.RemoteAddr() }
+func (c *Conn) SetDeadline(t time.Time) error      { return c.conn.SetDeadline(t) }
+func (c *Conn) SetReadDeadline(t time.Time) error  { return c.conn.SetReadDeadline(t) }
+func (c *Conn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) }
+
+// errCipherExhausted is the error returned when we run out of nonces
+// on a cipher.
+type errCipherExhausted struct{}
+
+func (errCipherExhausted) Error() string {
+	return "cipher exhausted, no more nonces available for current key"
+}
+func (errCipherExhausted) Timeout() bool   { return false }
+func (errCipherExhausted) Temporary() bool { return false }
+
+// errPartialWrite is the error returned when the cipher state has
+// become unusable due to a past partial write.
+type errPartialWrite struct {
+	err error
+}
+
+func (e errPartialWrite) Error() string {
+	return fmt.Sprintf("cipher state desynchronized due to partial write (%v)", e.err)
+}
+func (e errPartialWrite) Unwrap() error   { return e.err }
+func (e errPartialWrite) Temporary() bool { return false }
+func (e errPartialWrite) Timeout() bool   { return false }
+
+// errReadTooBig is the error returned when the peer sent an
+// unacceptably large Noise frame.
+type errReadTooBig struct {
+	requested int
+}
+
+func (e errReadTooBig) Error() string {
+	return fmt.Sprintf("requested read of %d bytes exceeds max allowed Noise frame size", e.requested)
+}
+func (e errReadTooBig) Temporary() bool {
+	// permanent error because this error only occurs when our peer
+	// sends us a frame so large we're unwilling to ever decode it.
+	return false
+}
+func (e errReadTooBig) Timeout() bool { return false }
+
+type nonce [chp.NonceSize]byte
+
+func (n *nonce) Valid() bool {
+	return binary.BigEndian.Uint32(n[:4]) == 0 && binary.BigEndian.Uint64(n[4:]) != invalidNonce
+}
+
+func (n *nonce) Increment() {
+	if !n.Valid() {
+		panic("increment of invalid nonce")
+	}
+	binary.BigEndian.PutUint64(n[4:], 1+binary.BigEndian.Uint64(n[4:]))
+}

control/noise/conn_test.go

@@ -0,0 +1,339 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"testing/iotest"
+
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/net/nettest"
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestMessageSize(t *testing.T) {
+	// This test is a regression guard against someone looking at
+	// maxCiphertextSize, going "huh, we could be more efficient if it
+	// were larger, and accidentally violating the Noise spec. Do not
+	// change this max value, it's a deliberate limitation of the
+	// cryptographic protocol we use (see Section 3 "Message Format"
+	// of the Noise spec).
+	const max = 65535
+	if maxCiphertextSize > max {
+		t.Fatalf("max ciphertext size is %d, which is larger than the maximum noise message size %d", maxCiphertextSize, max)
+	}
+}
+
+func TestConnBasic(t *testing.T) {
+	client, server := pair(t)
+
+	sb := sinkReads(server)
+
+	want := "test"
+	if _, err := io.WriteString(client, want); err != nil {
+		t.Fatalf("client write failed: %v", err)
+	}
+	client.Close()
+
+	if got := sb.String(4); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatal("client close wasn't seen by server")
+	}
+	if sb.Total() != 4 {
+		t.Fatalf("wrong amount of bytes received: got %d, want 4", sb.Total())
+	}
+}
+
+// bufferedWriteConn wraps a net.Conn and gives control over how
+// Writes get batched out.
+type bufferedWriteConn struct {
+	net.Conn
+	w           *bufio.Writer
+	manualFlush bool
+}
+
+func (c *bufferedWriteConn) Write(bs []byte) (int, error) {
+	n, err := c.w.Write(bs)
+	if err == nil && !c.manualFlush {
+		err = c.w.Flush()
+	}
+	return n, err
+}
+
+// TestFastPath exercises the Read codepath that can receive multiple
+// Noise frames at once and decode each in turn without making another
+// syscall.
+func TestFastPath(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	b := &bufferedWriteConn{s1, bufio.NewWriterSize(s1, 10000), false}
+	client, server := pairWithConns(t, b, s2)
+
+	b.manualFlush = true
+
+	sb := sinkReads(server)
+
+	const packets = 10
+	s := "test"
+	for i := 0; i < packets; i++ {
+		// Many separate writes, to force separate Noise frames that
+		// all get buffered up and then all sent as a single slice to
+		// the server.
+		if _, err := io.WriteString(client, s); err != nil {
+			t.Fatalf("client write1 failed: %v", err)
+		}
+	}
+	if err := b.w.Flush(); err != nil {
+		t.Fatalf("client flush failed: %v", err)
+	}
+	client.Close()
+
+	want := strings.Repeat(s, packets)
+	if got := sb.String(len(want)); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatalf("client close wasn't seen by server")
+	}
+}
+
+// Writes things larger than a single Noise frame, to check the
+// chunking on the encoder and decoder.
+func TestBigData(t *testing.T) {
+	client, server := pair(t)
+
+	serverReads := sinkReads(server)
+	clientReads := sinkReads(client)
+
+	const sz = 15 * 1024 // 15KiB
+	clientStr := strings.Repeat("abcde", sz/5)
+	serverStr := strings.Repeat("fghij", sz/5*2)
+
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+	if _, err := io.WriteString(server, serverStr); err != nil {
+		t.Fatalf("writing server>client: %v", err)
+	}
+
+	if serverGot := serverReads.String(sz); serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+	if clientGot := clientReads.String(2 * sz); clientGot != serverStr {
+		t.Error("client didn't receive what server sent")
+	}
+
+	getNonce := func(n [chp.NonceSize]byte) uint64 {
+		if binary.BigEndian.Uint32(n[:4]) != 0 {
+			panic("unexpected nonce")
+		}
+		return binary.BigEndian.Uint64(n[4:])
+	}
+
+	// Reach into the Conns and verify the cipher nonces advanced as
+	// expected.
+	if getNonce(client.tx.nonce) != getNonce(server.rx.nonce) {
+		t.Error("desynchronized client tx nonce")
+	}
+	if getNonce(server.tx.nonce) != getNonce(client.rx.nonce) {
+		t.Error("desynchronized server tx nonce")
+	}
+	if n := getNonce(client.tx.nonce); n != 4 {
+		t.Errorf("wrong client tx nonce, got %d want 4", n)
+	}
+	if n := getNonce(server.tx.nonce); n != 8 {
+		t.Errorf("wrong client tx nonce, got %d want 8", n)
+	}
+}
+
+// readerConn wraps a net.Conn and routes its Reads through a separate
+// io.Reader.
+type readerConn struct {
+	net.Conn
+	r io.Reader
+}
+
+func (c readerConn) Read(bs []byte) (int, error) { return c.r.Read(bs) }
+
+// Check that the receiver can handle not being able to read an entire
+// frame in a single syscall.
+func TestDataTrickle(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	client, server := pairWithConns(t, s1, readerConn{s2, iotest.OneByteReader(s2)})
+	serverReads := sinkReads(server)
+
+	const sz = 10000
+	clientStr := strings.Repeat("abcde", sz/5)
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+
+	serverGot := serverReads.String(sz)
+	if serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+}
+
+func TestConnStd(t *testing.T) {
+	// You can run this test manually, and noise.Conn should pass all
+	// of them except for TestConn/PastTimeout,
+	// TestConn/FutureTimeout, TestConn/ConcurrentMethods, because
+	// those tests assume that write errors are recoverable, and
+	// they're not on our Conn due to cipher security.
+	t.Skip("not all tests can pass on this Conn, see https://github.com/golang/go/issues/46977")
+	nettest.TestConn(t, func() (c1 net.Conn, c2 net.Conn, stop func(), err error) {
+		s1, s2 := tsnettest.NewConn("noise", 4096)
+		controlKey := key.NewMachine()
+		machineKey := key.NewMachine()
+		serverErr := make(chan error, 1)
+		go func() {
+			var err error
+			c2, err = Server(context.Background(), s2, controlKey)
+			serverErr <- err
+		}()
+		c1, err = Client(context.Background(), s1, machineKey, controlKey.Public())
+		if err != nil {
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting client: %w", err)
+		}
+		if err := <-serverErr; err != nil {
+			c1.Close()
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting server: %w", err)
+		}
+		return c1, c2, func() {
+			c1.Close()
+			c2.Close()
+		}, nil
+	})
+}
+
+// mkConns creates synthetic Noise Conns wrapping the given net.Conns.
+// This function is for testing just the Conn transport logic without
+// having to muck about with Noise handshakes.
+func mkConns(s1, s2 net.Conn) (*Conn, *Conn) {
+	var k1, k2 [chp.KeySize]byte
+	if _, err := rand.Read(k1[:]); err != nil {
+		panic(err)
+	}
+	if _, err := rand.Read(k2[:]); err != nil {
+		panic(err)
+	}
+
+	ret1 := &Conn{
+		conn: s1,
+		tx:   txState{cipher: newCHP(k1)},
+		rx:   rxState{cipher: newCHP(k2)},
+	}
+	ret2 := &Conn{
+		conn: s2,
+		tx:   txState{cipher: newCHP(k2)},
+		rx:   rxState{cipher: newCHP(k1)},
+	}
+
+	return ret1, ret2
+}
+
+type readSink struct {
+	r io.Reader
+
+	cond *sync.Cond
+	sync.Mutex
+	bs  bytes.Buffer
+	err error
+}
+
+func sinkReads(r io.Reader) *readSink {
+	ret := &readSink{
+		r: r,
+	}
+	ret.cond = sync.NewCond(&ret.Mutex)
+	go func() {
+		var buf [4096]byte
+		for {
+			n, err := r.Read(buf[:])
+			ret.Lock()
+			ret.bs.Write(buf[:n])
+			if err != nil {
+				ret.err = err
+			}
+			ret.cond.Broadcast()
+			ret.Unlock()
+			if err != nil {
+				return
+			}
+		}
+	}()
+	return ret
+}
+
+func (s *readSink) String(total int) string {
+	s.Lock()
+	defer s.Unlock()
+	for s.bs.Len() < total && s.err == nil {
+		s.cond.Wait()
+	}
+	if s.err != nil {
+		total = s.bs.Len()
+	}
+	return string(s.bs.Bytes()[:total])
+}
+
+func (s *readSink) Error() error {
+	s.Lock()
+	defer s.Unlock()
+	for s.err == nil {
+		s.cond.Wait()
+	}
+	return s.err
+}
+
+func (s *readSink) Total() int {
+	s.Lock()
+	defer s.Unlock()
+	return s.bs.Len()
+}
+
+func pairWithConns(t *testing.T, clientConn, serverConn net.Conn) (*Conn, *Conn) {
+	var (
+		controlKey = key.NewMachine()
+		machineKey = key.NewMachine()
+		server     *Conn
+		serverErr  = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, controlKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, machineKey, controlKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+	return client, server
+}
+
+func pair(t *testing.T) (*Conn, *Conn) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	return pairWithConns(t, s1, s2)
+}

control/noise/handshake.go

@@ -0,0 +1,443 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	"go4.org/mem"
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+	"tailscale.com/types/key"
+)
+
+const (
+	// protocolName is the name of the specific instantiation of Noise
+	// that the control protocol uses. This string's value is fixed by
+	// the Noise spec, and shouldn't be changed unless we're updating
+	// the control protocol to use a different Noise instance.
+	protocolName = "Noise_IK_25519_ChaChaPoly_BLAKE2s"
+	// protocolVersion is the version of the control protocol that
+	// Client will use when initiating a handshake.
+	protocolVersion uint16 = 1
+	// protocolVersionPrefix is the name portion of the protocol
+	// name+version string that gets mixed into the handshake as a
+	// prologue.
+	//
+	// This mixing verifies that both clients agree that they're
+	// executing the control protocol at a specific version that
+	// matches the advertised version in the cleartext packet header.
+	protocolVersionPrefix = "Tailscale Control Protocol v"
+	invalidNonce          = ^uint64(0)
+)
+
+func protocolVersionPrologue(version uint16) []byte {
+	ret := make([]byte, 0, len(protocolVersionPrefix)+5) // 5 bytes is enough to encode all possible version numbers.
+	ret = append(ret, protocolVersionPrefix...)
+	return strconv.AppendUint(ret, uint64(version), 10)
+}
+
+// Client initiates a control client handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process. Any preexisting Conn deadline is removed.
+func Client(ctx context.Context, conn net.Conn, machineKey key.MachinePrivate, controlKey key.MachinePublic) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	// prologue
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	s.MixHash(controlKey.UntypedBytes())
+
+	// -> e, es, s, ss
+	init := mkInitiationMessage()
+	machineEphemeral := key.NewMachine()
+	machineEphemeralPub := machineEphemeral.Public()
+	copy(init.EphemeralPub(), machineEphemeralPub.UntypedBytes())
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(machineEphemeral, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	machineKeyPub := machineKey.Public()
+	s.EncryptAndHash(cipher, init.MachinePub(), machineKeyPub.UntypedBytes())
+	cipher, err = s.MixDH(machineKey, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	s.EncryptAndHash(cipher, init.Tag(), nil) // empty message payload
+
+	if _, err := conn.Write(init[:]); err != nil {
+		return nil, fmt.Errorf("writing initiation: %w", err)
+	}
+
+	// Read in the payload and look for errors/protocol violations from the server.
+	var resp responseMessage
+	if _, err := io.ReadFull(conn, resp.Header()); err != nil {
+		return nil, fmt.Errorf("reading response header: %w", err)
+	}
+	if resp.Type() != msgTypeResponse {
+		if resp.Type() != msgTypeError {
+			return nil, fmt.Errorf("unexpected response message type %d", resp.Type())
+		}
+		msg := make([]byte, resp.Length())
+		if _, err := io.ReadFull(conn, msg); err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("server error: %q", msg)
+	}
+	if resp.Length() != len(resp.Payload()) {
+		return nil, fmt.Errorf("wrong length %d received for handshake response", resp.Length())
+	}
+	if _, err := io.ReadFull(conn, resp.Payload()); err != nil {
+		return nil, err
+	}
+
+	// <- e, ee, se
+	controlEphemeralPub := key.MachinePublicFromRaw32(mem.B(resp.EphemeralPub()))
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err = s.MixDH(machineEphemeral, controlEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(machineKey, controlEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, resp.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting payload: %w", err)
+	}
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          controlKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c1,
+		},
+		rx: rxState{
+			cipher: c2,
+		},
+	}
+	return c, nil
+}
+
+// Server initiates a control server handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process.
+func Server(ctx context.Context, conn net.Conn, controlKey key.MachinePrivate) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	// Deliberately does not support formatting, so that we don't echo
+	// attacker-controlled input back to them.
+	sendErr := func(msg string) error {
+		if len(msg) >= 1<<16 {
+			msg = msg[:1<<16]
+		}
+		var hdr [headerLen]byte
+		hdr[0] = msgTypeError
+		binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg)))
+		if _, err := conn.Write(hdr[:]); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		if _, err := io.WriteString(conn, msg); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		return fmt.Errorf("refused client handshake: %q", msg)
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	var init initiationMessage
+	if _, err := io.ReadFull(conn, init.Header()); err != nil {
+		return nil, err
+	}
+	if init.Version() != protocolVersion {
+		return nil, sendErr("unsupported protocol version")
+	}
+	if init.Type() != msgTypeInitiation {
+		return nil, sendErr("unexpected handshake message type")
+	}
+	if init.Length() != len(init.Payload()) {
+		return nil, sendErr("wrong handshake initiation length")
+	}
+	if _, err := io.ReadFull(conn, init.Payload()); err != nil {
+		return nil, err
+	}
+
+	// prologue. Can only do this once we at least think the client is
+	// handshaking using a supported version.
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	controlKeyPub := controlKey.Public()
+	s.MixHash(controlKeyPub.UntypedBytes())
+
+	// -> e, es, s, ss
+	machineEphemeralPub := key.MachinePublicFromRaw32(mem.B(init.EphemeralPub()))
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(controlKey, machineEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	var machineKeyBytes [32]byte
+	if err := s.DecryptAndHash(cipher, machineKeyBytes[:], init.MachinePub()); err != nil {
+		return nil, fmt.Errorf("decrypting machine key: %w", err)
+	}
+	machineKey := key.MachinePublicFromRaw32(mem.B(machineKeyBytes[:]))
+	cipher, err = s.MixDH(controlKey, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, init.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting initiation tag: %w", err)
+	}
+
+	// <- e, ee, se
+	resp := mkResponseMessage()
+	controlEphemeral := key.NewMachine()
+	controlEphemeralPub := controlEphemeral.Public()
+	copy(resp.EphemeralPub(), controlEphemeralPub.UntypedBytes())
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err := s.MixDH(controlEphemeral, machineEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(controlEphemeral, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	s.EncryptAndHash(cipher, resp.Tag(), nil) // empty message payload
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	if _, err := conn.Write(resp[:]); err != nil {
+		return nil, err
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          machineKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c2,
+		},
+		rx: rxState{
+			cipher: c1,
+		},
+	}
+	return c, nil
+}
+
+// symmetricState contains the state of an in-flight handshake.
+type symmetricState struct {
+	finished bool
+
+	h  [blake2s.Size]byte // hash of currently-processed handshake state
+	ck [blake2s.Size]byte // chaining key used to construct session keys at the end of the handshake
+}
+
+func (s *symmetricState) checkFinished() {
+	if s.finished {
+		panic("attempted to use symmetricState after Split was called")
+	}
+}
+
+// Initialize sets s to the initial handshake state, prior to
+// processing any handshake messages.
+func (s *symmetricState) Initialize() {
+	s.checkFinished()
+	s.h = blake2s.Sum256([]byte(protocolName))
+	s.ck = s.h
+}
+
+// MixHash updates s.h to be BLAKE2s(s.h || data), where || is
+// concatenation.
+func (s *symmetricState) MixHash(data []byte) {
+	s.checkFinished()
+	h := newBLAKE2s()
+	h.Write(s.h[:])
+	h.Write(data)
+	h.Sum(s.h[:0])
+}
+
+// MixDH updates s.ck with the result of X25519(priv, pub) and returns
+// a singleUseCHP that can be used to encrypt or decrypt handshake
+// data.
+//
+// MixDH corresponds to MixKey(X25519(...))) in the spec. Implementing
+// it as a single function allows for strongly-typed arguments that
+// reduce the risk of error in the caller (e.g. invoking X25519 with
+// two private keys, or two public keys), and thus producing the wrong
+// calculation.
+func (s *symmetricState) MixDH(priv key.MachinePrivate, pub key.MachinePublic) (*singleUseCHP, error) {
+	s.checkFinished()
+	keyData, err := curve25519.X25519(priv.UntypedBytes(), pub.UntypedBytes())
+	if err != nil {
+		return nil, fmt.Errorf("computing X25519: %w", err)
+	}
+
+	r := hkdf.New(newBLAKE2s, keyData, s.ck[:], nil)
+	if _, err := io.ReadFull(r, s.ck[:]); err != nil {
+		return nil, fmt.Errorf("extracting ck: %w", err)
+	}
+	var k [chp.KeySize]byte
+	if _, err := io.ReadFull(r, k[:]); err != nil {
+		return nil, fmt.Errorf("extracting k: %w", err)
+	}
+	return newSingleUseCHP(k), nil
+}
+
+// EncryptAndHash encrypts plaintext into ciphertext (which must be
+// the correct size to hold the encrypted plaintext) using cipher,
+// mixes the ciphertext into s.h, and returns the ciphertext.
+func (s *symmetricState) EncryptAndHash(cipher *singleUseCHP, ciphertext, plaintext []byte) {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		panic("ciphertext is wrong size for given plaintext")
+	}
+	ret := cipher.Seal(ciphertext[:0], plaintext, s.h[:])
+	s.MixHash(ret)
+}
+
+// DecryptAndHash decrypts the given ciphertext into plaintext (which
+// must be the correct size to hold the decrypted ciphertext) using
+// cipher. If decryption is successful, it mixes the ciphertext into
+// s.h.
+func (s *symmetricState) DecryptAndHash(cipher *singleUseCHP, plaintext, ciphertext []byte) error {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		return errors.New("plaintext is wrong size for given ciphertext")
+	}
+	if _, err := cipher.Open(plaintext[:0], ciphertext, s.h[:]); err != nil {
+		return err
+	}
+	s.MixHash(ciphertext)
+	return nil
+}
+
+// Split returns two ChaCha20Poly1305 ciphers with keys derived from
+// the current handshake state. Methods on s cannot be used again
+// after calling Split.
+func (s *symmetricState) Split() (c1, c2 cipher.AEAD, err error) {
+	s.finished = true
+
+	var k1, k2 [chp.KeySize]byte
+	r := hkdf.New(newBLAKE2s, nil, s.ck[:], nil)
+	if _, err := io.ReadFull(r, k1[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k1: %w", err)
+	}
+	if _, err := io.ReadFull(r, k2[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k2: %w", err)
+	}
+	c1, err = chp.New(k1[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c1: %w", err)
+	}
+	c2, err = chp.New(k2[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c2: %w", err)
+	}
+	return c1, c2, nil
+}
+
+// newBLAKE2s returns a hash.Hash implementing BLAKE2s, or panics on
+// error.
+func newBLAKE2s() hash.Hash {
+	h, err := blake2s.New256(nil)
+	if err != nil {
+		// Should never happen, errors only happen when using BLAKE2s
+		// in MAC mode with a key.
+		panic(err)
+	}
+	return h
+}
+
+// newCHP returns a cipher.AEAD implementing ChaCha20Poly1305, or
+// panics on error.
+func newCHP(key [chp.KeySize]byte) cipher.AEAD {
+	aead, err := chp.New(key[:])
+	if err != nil {
+		// Can only happen if we passed a key of the wrong length. The
+		// function signature prevents that.
+		panic(err)
+	}
+	return aead
+}
+
+// singleUseCHP is an instance of ChaCha20Poly1305 that can be used
+// only once, either for encrypting or decrypting, but not both. The
+// chosen operation is always executed with an all-zeros
+// nonce. Subsequent calls to either Seal or Open panic.
+type singleUseCHP struct {
+	c cipher.AEAD
+}
+
+func newSingleUseCHP(key [chp.KeySize]byte) *singleUseCHP {
+	return &singleUseCHP{newCHP(key)}
+}
+
+func (c *singleUseCHP) Seal(dst, plaintext, additionalData []byte) []byte {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Seal(dst, nonce[:], plaintext, additionalData)
+}
+
+func (c *singleUseCHP) Open(dst, ciphertext, additionalData []byte) ([]byte, error) {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Open(dst, nonce[:], ciphertext, additionalData)
+}

control/noise/handshake_test.go

@@ -0,0 +1,299 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestHandshake(t *testing.T) {
+	var (
+		clientConn, serverConn = tsnettest.NewConn("noise", 128000)
+		serverKey              = key.NewMachine()
+		clientKey              = key.NewMachine()
+		server                 *Conn
+		serverErr              = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, serverKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+
+	if client.HandshakeHash() != server.HandshakeHash() {
+		t.Fatal("client and server disagree on handshake hash")
+	}
+
+	if client.ProtocolVersion() != int(protocolVersion) {
+		t.Fatalf("client reporting wrong protocol version %d, want %d", client.ProtocolVersion(), protocolVersion)
+	}
+	if client.ProtocolVersion() != server.ProtocolVersion() {
+		t.Fatalf("peers disagree on protocol version, client=%d server=%d", client.ProtocolVersion(), server.ProtocolVersion())
+	}
+	if client.Peer() != serverKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+	if server.Peer() != clientKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+}
+
+// Check that handshaking repeatedly with the same long-term keys
+// result in different handshake hashes and wire traffic.
+func TestNoReuse(t *testing.T) {
+	var (
+		hashes           = map[[32]byte]bool{}
+		clientHandshakes = map[[96]byte]bool{}
+		serverHandshakes = map[[48]byte]bool{}
+		packets          = map[[32]byte]bool{}
+	)
+	for i := 0; i < 10; i++ {
+		var (
+			clientRaw, serverRaw = tsnettest.NewConn("noise", 128000)
+			clientBuf, serverBuf bytes.Buffer
+			clientConn           = &readerConn{clientRaw, io.TeeReader(clientRaw, &clientBuf)}
+			serverConn           = &readerConn{serverRaw, io.TeeReader(serverRaw, &serverBuf)}
+			serverKey            = key.NewMachine()
+			clientKey            = key.NewMachine()
+			server               *Conn
+			serverErr            = make(chan error, 1)
+		)
+		go func() {
+			var err error
+			server, err = Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client connection failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed: %v", err)
+		}
+
+		var clientHS [96]byte
+		copy(clientHS[:], serverBuf.Bytes())
+		if clientHandshakes[clientHS] {
+			t.Fatal("client handshake seen twice")
+		}
+		clientHandshakes[clientHS] = true
+
+		var serverHS [48]byte
+		copy(serverHS[:], clientBuf.Bytes())
+		if serverHandshakes[serverHS] {
+			t.Fatal("server handshake seen twice")
+		}
+		serverHandshakes[serverHS] = true
+
+		clientBuf.Reset()
+		serverBuf.Reset()
+		cb := sinkReads(client)
+		sb := sinkReads(server)
+
+		if hashes[client.HandshakeHash()] {
+			t.Fatalf("handshake hash %v seen twice", client.HandshakeHash())
+		}
+		hashes[client.HandshakeHash()] = true
+
+		// Sending 14 bytes turns into 32 bytes on the wire (+16 for
+		// the chacha20poly1305 overhead, +2 length header)
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if _, err := io.WriteString(server, strings.Repeat("b", 14)); err != nil {
+			t.Fatalf("server>client write failed: %v", err)
+		}
+
+		// Wait for the bytes to be read, so we know they've traveled end to end
+		cb.String(14)
+		sb.String(14)
+
+		var clientWire, serverWire [32]byte
+		copy(clientWire[:], clientBuf.Bytes())
+		copy(serverWire[:], serverBuf.Bytes())
+
+		if packets[clientWire] {
+			t.Fatalf("client wire traffic seen twice")
+		}
+		packets[clientWire] = true
+		if packets[serverWire] {
+			t.Fatalf("server wire traffic seen twice")
+		}
+		packets[serverWire] = true
+
+		server.Close()
+		client.Close()
+	}
+}
+
+// tamperReader wraps a reader and mutates the Nth byte.
+type tamperReader struct {
+	r     io.Reader
+	n     int
+	total int
+}
+
+func (r *tamperReader) Read(bs []byte) (int, error) {
+	n, err := r.r.Read(bs)
+	if off := r.n - r.total; off >= 0 && off < n {
+		bs[off] += 1
+	}
+	r.total += n
+	return n, err
+}
+
+func TestTampering(t *testing.T) {
+	// Tamper with every byte of the client initiation message.
+	for i := 0; i < 101; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			// If the server failed, we have to close the Conn to
+			// unblock the client.
+			if err != nil {
+				serverConn.Close()
+			}
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatalf("server connection succeeded despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the server response message.
+	for i := 0; i < 51; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed despite no tampering: %v", err)
+		}
+	}
+
+	// Tamper with every byte of the first server>client transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, 51 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			_, err = io.WriteString(server, strings.Repeat("a", 14))
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		// The client needs a timeout if the tampering is hitting the length header.
+		if i == 1 || i == 2 {
+			client.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+		}
+
+		var bs [100]byte
+		n, err := client.Read(bs[:])
+		if err == nil {
+			t.Fatal("read succeeded despite tampering")
+		}
+		if n != 0 {
+			t.Fatal("conn yielded some bytes despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the first client>server transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, 101 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			var bs [100]byte
+			// The server needs a timeout if the tampering is hitting the length header.
+			if i == 1 || i == 2 {
+				server.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+			}
+			n, err := server.Read(bs[:])
+			if n != 0 {
+				panic("server got bytes despite tampering")
+			} else {
+				serverErr <- err
+			}
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatal("server successfully received bytes despite tampering")
+		}
+	}
+}

control/noise/interop_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"net"
+	"testing"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+// Can a reference Noise IK client talk to our server?
+func TestInteropClient(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		serverErr   = make(chan error, 2)
+		serverBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		server, err := Server(context.Background(), s2, controlKey)
+		serverErr <- err
+		if err != nil {
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(server, buf[:len(c2s)])
+		serverBytes <- buf[:len(c2s)]
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		_, err = server.Write([]byte(s2c))
+		serverErr <- err
+	}()
+
+	gotS2C, err := noiseExplorerClient(s1, controlKey.Public(), machineKey, []byte(c2s))
+	if err != nil {
+		t.Fatalf("failed client interop: %v", err)
+	}
+	if string(gotS2C) != s2c {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotS2C), s2c)
+	}
+
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server handshake failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server read/write failed: %v", err)
+	}
+	if got := string(<-serverBytes); got != c2s {
+		t.Fatalf("server received %q, want %q", got, c2s)
+	}
+}
+
+// Can our client talk to a reference Noise IK server?
+func TestInteropServer(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		clientErr   = make(chan error, 2)
+		clientBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		client, err := Client(context.Background(), s1, machineKey, controlKey.Public())
+		clientErr <- err
+		if err != nil {
+			return
+		}
+		_, err = client.Write([]byte(c2s))
+		if err != nil {
+			clientErr <- err
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(client, buf[:len(s2c)])
+		clientBytes <- buf[:len(s2c)]
+		clientErr <- err
+	}()
+
+	gotC2S, err := noiseExplorerServer(s2, controlKey, machineKey.Public(), []byte(s2c))
+	if err != nil {
+		t.Fatalf("failed server interop: %v", err)
+	}
+	if string(gotC2S) != c2s {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotC2S), c2s)
+	}
+
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client handshake failed: %v", err)
+	}
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client read/write failed: %v", err)
+	}
+	if got := string(<-clientBytes); got != s2c {
+		t.Fatalf("client received %q, want %q", got, s2c)
+	}
+}
+
+// noiseExplorerClient uses the Noise Explorer implementation of Noise
+// IK to handshake as a Noise client on conn, transmit payload, and
+// read+return a payload from the peer.
+func noiseExplorerClient(conn net.Conn, controlKey key.MachinePublic, machineKey key.MachinePrivate, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], machineKey.UntypedBytes())
+	copy(mk.public_key[:], machineKey.Public().UntypedBytes())
+	var peerKey [32]byte
+	copy(peerKey[:], controlKey.UntypedBytes())
+	session := InitSession(true, protocolVersionPrologue(protocolVersion), mk, peerKey)
+
+	_, msg1 := SendMessage(&session, nil)
+	var hdr [initiationHeaderLen]byte
+	binary.BigEndian.PutUint16(hdr[:2], protocolVersion)
+	hdr[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(hdr[3:5], 96)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ns); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ciphertext); err != nil {
+		return nil, err
+	}
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:51]); err != nil {
+		return nil, err
+	}
+	// ignore the header for this test, we're only checking the noise
+	// implementation.
+	msg2 := messagebuffer{
+		ciphertext: buf[35:51],
+	}
+	copy(msg2.ne[:], buf[3:35])
+	_, p, valid := RecvMessage(&session, &msg2)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg3 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg3.ciphertext)))
+	if _, err := conn.Write(hdr[:3]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg3.ciphertext); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	// Ignore all of the header except the payload length
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg4 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg4)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	return p, nil
+}
+
+func noiseExplorerServer(conn net.Conn, controlKey key.MachinePrivate, wantMachineKey key.MachinePublic, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], controlKey.UntypedBytes())
+	copy(mk.public_key[:], controlKey.Public().UntypedBytes())
+	session := InitSession(false, protocolVersionPrologue(protocolVersion), mk, [32]byte{})
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:101]); err != nil {
+		return nil, err
+	}
+	// Ignore the header, we're just checking the noise implementation.
+	msg1 := messagebuffer{
+		ns:         buf[37:85],
+		ciphertext: buf[85:101],
+	}
+	copy(msg1.ne[:], buf[5:37])
+	_, p, valid := RecvMessage(&session, &msg1)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg2 := SendMessage(&session, nil)
+	var hdr [headerLen]byte
+	hdr[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(hdr[1:3], 48)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ciphertext[:]); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg3 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg3)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	_, msg4 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg4.ciphertext)))
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg4.ciphertext); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}

control/noise/messages.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import "encoding/binary"
+
+const (
+	// msgTypeInitiation frames carry a Noise IK handshake initiation message.
+	msgTypeInitiation = 1
+	// msgTypeResponse frames carry a Noise IK handshake response message.
+	msgTypeResponse = 2
+	// msgTypeError frames carry an unauthenticated human-readable
+	// error message.
+	//
+	// Errors reported in this message type must be treated as public
+	// hints only. They are not encrypted or authenticated, and so can
+	// be seen and tampered with on the wire.
+	msgTypeError = 3
+	// msgTypeRecord frames carry session data bytes.
+	msgTypeRecord = 4
+
+	// headerLen is the size of the header on all messages except msgTypeInitiation.
+	headerLen = 3
+	// initiationHeaderLen is the size of the header on all msgTypeInitiation messages.
+	initiationHeaderLen = 5
+)
+
+// initiationMessage is the protocol message sent from a client
+// machine to a control server.
+//
+// 2b: protocol version
+// 1b: message type (0x01)
+// 2b: payload length (96)
+// 5b: header (see headerLen for fields)
+// 32b: client ephemeral public key (cleartext)
+// 48b: client machine public key (encrypted)
+// 16b: message tag (authenticates the whole message)
+type initiationMessage [101]byte
+
+func mkInitiationMessage() initiationMessage {
+	var ret initiationMessage
+	binary.BigEndian.PutUint16(ret[:2], uint16(protocolVersion))
+	ret[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(ret[3:5], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *initiationMessage) Header() []byte  { return m[:initiationHeaderLen] }
+func (m *initiationMessage) Payload() []byte { return m[initiationHeaderLen:] }
+
+func (m *initiationMessage) Version() uint16 { return binary.BigEndian.Uint16(m[:2]) }
+func (m *initiationMessage) Type() byte      { return m[2] }
+func (m *initiationMessage) Length() int     { return int(binary.BigEndian.Uint16(m[3:5])) }
+
+func (m *initiationMessage) EphemeralPub() []byte {
+	return m[initiationHeaderLen : initiationHeaderLen+32]
+}
+func (m *initiationMessage) MachinePub() []byte {
+	return m[initiationHeaderLen+32 : initiationHeaderLen+32+48]
+}
+func (m *initiationMessage) Tag() []byte { return m[initiationHeaderLen+32+48:] }
+
+// responseMessage is the protocol message sent from a control server
+// to a client machine.
+//
+// 1b: message type (0x02)
+// 2b: payload length (48)
+// 32b: control ephemeral public key (cleartext)
+// 16b: message tag (authenticates the whole message)
+type responseMessage [51]byte
+
+func mkResponseMessage() responseMessage {
+	var ret responseMessage
+	ret[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(ret[1:], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *responseMessage) Header() []byte  { return m[:headerLen] }
+func (m *responseMessage) Payload() []byte { return m[headerLen:] }
+
+func (m *responseMessage) Type() byte  { return m[0] }
+func (m *responseMessage) Length() int { return int(binary.BigEndian.Uint16(m[1:3])) }
+
+func (m *responseMessage) EphemeralPub() []byte { return m[headerLen : headerLen+32] }
+func (m *responseMessage) Tag() []byte          { return m[headerLen+32:] }

control/noise/noiseexplorer_test.go

@@ -0,0 +1,475 @@
+// This file contains the implementation of Noise IK from
+// https://noiseexplorer.com/ . Unlike the rest of this repository,
+// this file is licensed under the terms of the GNU GPL v3. See
+// https://source.symbolic.software/noiseexplorer/noiseexplorer for
+// more information.
+//
+// This file is used here to verify that Tailscale's implementation of
+// Noise IK is interoperable with another implementation.
+//lint:file-ignore SA4006 not our code.
+
+/*
+IK:
+  <- s
+  ...
+  -> e, es, s, ss
+  <- e, ee, se
+  ->
+  <-
+*/
+
+// Implementation Version: 1.0.2
+
+/* ---------------------------------------------------------------- *
+ * PARAMETERS                                                       *
+ * ---------------------------------------------------------------- */
+
+package noise
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/binary"
+	"hash"
+	"io"
+	"math"
+
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+)
+
+/* ---------------------------------------------------------------- *
+ * TYPES                                                            *
+ * ---------------------------------------------------------------- */
+
+type keypair struct {
+	public_key  [32]byte
+	private_key [32]byte
+}
+
+type messagebuffer struct {
+	ne         [32]byte
+	ns         []byte
+	ciphertext []byte
+}
+
+type cipherstate struct {
+	k [32]byte
+	n uint32
+}
+
+type symmetricstate struct {
+	cs cipherstate
+	ck [32]byte
+	h  [32]byte
+}
+
+type handshakestate struct {
+	ss  symmetricstate
+	s   keypair
+	e   keypair
+	rs  [32]byte
+	re  [32]byte
+	psk [32]byte
+}
+
+type noisesession struct {
+	hs  handshakestate
+	h   [32]byte
+	cs1 cipherstate
+	cs2 cipherstate
+	mc  uint64
+	i   bool
+}
+
+/* ---------------------------------------------------------------- *
+ * CONSTANTS                                                        *
+ * ---------------------------------------------------------------- */
+
+var emptyKey = [32]byte{
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+}
+
+var minNonce = uint32(0)
+
+/* ---------------------------------------------------------------- *
+ * UTILITY FUNCTIONS                                                *
+ * ---------------------------------------------------------------- */
+
+func getPublicKey(kp *keypair) [32]byte {
+	return kp.public_key
+}
+
+func isEmptyKey(k [32]byte) bool {
+	return subtle.ConstantTimeCompare(k[:], emptyKey[:]) == 1
+}
+
+func validatePublicKey(k []byte) bool {
+	forbiddenCurveValues := [12][]byte{
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{224, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 0},
+		{95, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 87},
+		{236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{237, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{205, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 128},
+		{76, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 215},
+		{217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{218, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{219, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25},
+	}
+
+	for _, testValue := range forbiddenCurveValues {
+		if subtle.ConstantTimeCompare(k[:], testValue[:]) == 1 {
+			panic("Invalid public key")
+		}
+	}
+	return true
+}
+
+/* ---------------------------------------------------------------- *
+ * PRIMITIVES                                                       *
+ * ---------------------------------------------------------------- */
+
+func incrementNonce(n uint32) uint32 {
+	return n + 1
+}
+
+func dh(private_key [32]byte, public_key [32]byte) [32]byte {
+	var ss [32]byte
+	curve25519.ScalarMult(&ss, &private_key, &public_key)
+	return ss
+}
+
+func generateKeypair() keypair {
+	var public_key [32]byte
+	var private_key [32]byte
+	_, _ = rand.Read(private_key[:])
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	if validatePublicKey(public_key[:]) {
+		return keypair{public_key, private_key}
+	}
+	return generateKeypair()
+}
+
+func generatePublicKey(private_key [32]byte) [32]byte {
+	var public_key [32]byte
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	return public_key
+}
+
+func encrypt(k [32]byte, n uint32, ad []byte, plaintext []byte) []byte {
+	var nonce [12]byte
+	var ciphertext []byte
+	enc, _ := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	ciphertext = enc.Seal(nil, nonce[:], plaintext, ad)
+	return ciphertext
+}
+
+func decrypt(k [32]byte, n uint32, ad []byte, ciphertext []byte) (bool, []byte, []byte) {
+	var nonce [12]byte
+	var plaintext []byte
+	enc, err := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	plaintext, err = enc.Open(nil, nonce[:], ciphertext, ad)
+	return (err == nil), ad, plaintext
+}
+
+func getHash(a []byte, b []byte) [32]byte {
+	return blake2s.Sum256(append(a, b...))
+}
+
+func hashProtocolName(protocolName []byte) [32]byte {
+	var h [32]byte
+	if len(protocolName) <= 32 {
+		copy(h[:], protocolName)
+	} else {
+		h = getHash(protocolName, []byte{})
+	}
+	return h
+}
+
+func blake2HkdfInterface() hash.Hash {
+	h, _ := blake2s.New256([]byte{})
+	return h
+}
+
+func getHkdf(ck [32]byte, ikm []byte) ([32]byte, [32]byte, [32]byte) {
+	var k1 [32]byte
+	var k2 [32]byte
+	var k3 [32]byte
+	output := hkdf.New(blake2HkdfInterface, ikm[:], ck[:], []byte{})
+	io.ReadFull(output, k1[:])
+	io.ReadFull(output, k2[:])
+	io.ReadFull(output, k3[:])
+	return k1, k2, k3
+}
+
+/* ---------------------------------------------------------------- *
+ * STATE MANAGEMENT                                                 *
+ * ---------------------------------------------------------------- */
+
+/* CipherState */
+func initializeKey(k [32]byte) cipherstate {
+	return cipherstate{k, minNonce}
+}
+
+func hasKey(cs *cipherstate) bool {
+	return !isEmptyKey(cs.k)
+}
+
+func setNonce(cs *cipherstate, newNonce uint32) *cipherstate {
+	cs.n = newNonce
+	return cs
+}
+
+func encryptWithAd(cs *cipherstate, ad []byte, plaintext []byte) (*cipherstate, []byte) {
+	e := encrypt(cs.k, cs.n, ad, plaintext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, e
+}
+
+func decryptWithAd(cs *cipherstate, ad []byte, ciphertext []byte) (*cipherstate, []byte, bool) {
+	valid, ad, plaintext := decrypt(cs.k, cs.n, ad, ciphertext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, plaintext, valid
+}
+
+func reKey(cs *cipherstate) *cipherstate {
+	e := encrypt(cs.k, math.MaxUint32, []byte{}, emptyKey[:])
+	copy(cs.k[:], e)
+	return cs
+}
+
+/* SymmetricState */
+
+func initializeSymmetric(protocolName []byte) symmetricstate {
+	h := hashProtocolName(protocolName)
+	ck := h
+	cs := initializeKey(emptyKey)
+	return symmetricstate{cs, ck, h}
+}
+
+func mixKey(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	ck, tempK, _ := getHkdf(ss.ck, ikm[:])
+	ss.cs = initializeKey(tempK)
+	ss.ck = ck
+	return ss
+}
+
+func mixHash(ss *symmetricstate, data []byte) *symmetricstate {
+	ss.h = getHash(ss.h[:], data)
+	return ss
+}
+
+func mixKeyAndHash(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	var tempH [32]byte
+	var tempK [32]byte
+	ss.ck, tempH, tempK = getHkdf(ss.ck, ikm[:])
+	ss = mixHash(ss, tempH[:])
+	ss.cs = initializeKey(tempK)
+	return ss
+}
+
+func getHandshakeHash(ss *symmetricstate) [32]byte {
+	return ss.h
+}
+
+func encryptAndHash(ss *symmetricstate, plaintext []byte) (*symmetricstate, []byte) {
+	var ciphertext []byte
+	if hasKey(&ss.cs) {
+		_, ciphertext = encryptWithAd(&ss.cs, ss.h[:], plaintext)
+	} else {
+		ciphertext = plaintext
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, ciphertext
+}
+
+func decryptAndHash(ss *symmetricstate, ciphertext []byte) (*symmetricstate, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if hasKey(&ss.cs) {
+		_, plaintext, valid = decryptWithAd(&ss.cs, ss.h[:], ciphertext)
+	} else {
+		plaintext, valid = ciphertext, true
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, plaintext, valid
+}
+
+func split(ss *symmetricstate) (cipherstate, cipherstate) {
+	tempK1, tempK2, _ := getHkdf(ss.ck, []byte{})
+	cs1 := initializeKey(tempK1)
+	cs2 := initializeKey(tempK2)
+	return cs1, cs2
+}
+
+/* HandshakeState */
+
+func initializeInitiator(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, rs[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func initializeResponder(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, s.public_key[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func writeMessageA(hs *handshakestate, payload []byte) (*handshakestate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	spk := make([]byte, len(hs.s.public_key))
+	copy(spk[:], hs.s.public_key[:])
+	_, ns = encryptAndHash(&hs.ss, spk)
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return hs, messageBuffer
+}
+
+func writeMessageB(hs *handshakestate, payload []byte) ([32]byte, messagebuffer, cipherstate, cipherstate) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, messageBuffer, cs1, cs2
+}
+
+func writeMessageRegular(cs *cipherstate, payload []byte) (*cipherstate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	cs, ciphertext = encryptWithAd(cs, []byte{}, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return cs, messageBuffer
+}
+
+func readMessageA(hs *handshakestate, message *messagebuffer) (*handshakestate, []byte, bool) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, ns, valid1 := decryptAndHash(&hs.ss, message.ns)
+	if valid1 && len(ns) == 32 && validatePublicKey(message.ns[:]) {
+		copy(hs.rs[:], ns)
+	}
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	return hs, plaintext, (valid1 && valid2)
+}
+
+func readMessageB(hs *handshakestate, message *messagebuffer) ([32]byte, []byte, bool, cipherstate, cipherstate) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, plaintext, (valid1 && valid2), cs1, cs2
+}
+
+func readMessageRegular(cs *cipherstate, message *messagebuffer) (*cipherstate, []byte, bool) {
+	/* No encrypted keys */
+	_, plaintext, valid2 := decryptWithAd(cs, []byte{}, message.ciphertext)
+	return cs, plaintext, valid2
+}
+
+/* ---------------------------------------------------------------- *
+ * PROCESSES                                                        *
+ * ---------------------------------------------------------------- */
+
+func InitSession(initiator bool, prologue []byte, s keypair, rs [32]byte) noisesession {
+	var session noisesession
+	psk := emptyKey
+	if initiator {
+		session.hs = initializeInitiator(prologue, s, rs, psk)
+	} else {
+		session.hs = initializeResponder(prologue, s, rs, psk)
+	}
+	session.i = initiator
+	session.mc = 0
+	return session
+}
+
+func SendMessage(session *noisesession, message []byte) (*noisesession, messagebuffer) {
+	var messageBuffer messagebuffer
+	if session.mc == 0 {
+		_, messageBuffer = writeMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, messageBuffer, session.cs1, session.cs2 = writeMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, messageBuffer = writeMessageRegular(&session.cs1, message)
+		} else {
+			_, messageBuffer = writeMessageRegular(&session.cs2, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, messageBuffer
+}
+
+func RecvMessage(session *noisesession, message *messagebuffer) (*noisesession, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if session.mc == 0 {
+		_, plaintext, valid = readMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, plaintext, valid, session.cs1, session.cs2 = readMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, plaintext, valid = readMessageRegular(&session.cs2, message)
+		} else {
+			_, plaintext, valid = readMessageRegular(&session.cs1, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, plaintext, valid
+}
+
+func main() {}

derp/derp_client.go

@@ -6,7 +6,6 @@ package derp
 
 import (
 	"bufio"
-	crand "crypto/rand"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
@@ -15,7 +14,7 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/crypto/nacl/box"
+	"go4.org/mem"
 	"golang.org/x/time/rate"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -23,9 +22,9 @@ import (
 
 // Client is a DERP client.
 type Client struct {
-	serverKey   key.Public // of the DERP server; not a machine or node key
-	privateKey  key.Private
-	publicKey   key.Public // of privateKey
+	serverKey   key.NodePublic // of the DERP server; not a machine or node key
+	privateKey  key.NodePrivate
+	publicKey   key.NodePublic // of privateKey
 	logf        logger.Logf
 	nc          Conn
 	br          *bufio.Reader
@@ -54,7 +53,7 @@ func (f clientOptFunc) update(o *clientOpt) { f(o) }
 // clientOpt are the options passed to newClient.
 type clientOpt struct {
 	MeshKey     string
-	ServerPub   key.Public
+	ServerPub   key.NodePublic
 	CanAckPings bool
 	IsProber    bool
 }
@@ -71,7 +70,7 @@ func IsProber(v bool) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.Is
 
 // ServerPublicKey returns a ClientOpt to declare that the server's DERP public key is known.
 // If key is the zero value, the returned ClientOpt is a no-op.
-func ServerPublicKey(key key.Public) ClientOpt {
+func ServerPublicKey(key key.NodePublic) ClientOpt {
 	return clientOptFunc(func(o *clientOpt) { o.ServerPub = key })
 }
 
@@ -81,7 +80,7 @@ func CanAckPings(v bool) ClientOpt {
 	return clientOptFunc(func(o *clientOpt) { o.CanAckPings = v })
 }
 
-func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opts ...ClientOpt) (*Client, error) {
+func NewClient(privateKey key.NodePrivate, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opts ...ClientOpt) (*Client, error) {
 	var opt clientOpt
 	for _, o := range opts {
 		if o == nil {
@@ -92,7 +91,7 @@ func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logg
 	return newClient(privateKey, nc, brw, logf, opt)
 }
 
-func newClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opt clientOpt) (*Client, error) {
+func newClient(privateKey key.NodePrivate, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opt clientOpt) (*Client, error) {
 	c := &Client{
 		privateKey:  privateKey,
 		publicKey:   privateKey.Public(),
@@ -130,7 +129,7 @@ func (c *Client) recvServerKey() error {
 	if flen < uint32(len(buf)) || t != frameServerKey || string(buf[:len(magic)]) != magic {
 		return errors.New("invalid server greeting")
 	}
-	copy(c.serverKey[:], buf[len(magic):])
+	c.serverKey = key.NodePublicFromRaw32(mem.B(buf[len(magic):]))
 	return nil
 }
 
@@ -143,13 +142,9 @@ func (c *Client) parseServerInfo(b []byte) (*serverInfo, error) {
 	if fl > maxLength {
 		return nil, fmt.Errorf("long serverInfo frame")
 	}
-	// TODO: add a read-nonce-and-box helper
-	var nonce [nonceLen]byte
-	copy(nonce[:], b)
-	msgbox := b[nonceLen:]
-	msg, ok := box.Open(nil, msgbox, &nonce, c.serverKey.B32(), c.privateKey.B32())
+	msg, ok := c.privateKey.OpenFrom(c.serverKey, b)
 	if !ok {
-		return nil, fmt.Errorf("failed to open naclbox from server key %x", c.serverKey[:])
+		return nil, fmt.Errorf("failed to open naclbox from server key %s", c.serverKey)
 	}
 	info := new(serverInfo)
 	if err := json.Unmarshal(msg, info); err != nil {
@@ -176,10 +171,6 @@ type clientInfo struct {
 }
 
 func (c *Client) sendClientKey() error {
-	var nonce [nonceLen]byte
-	if _, err := crand.Read(nonce[:]); err != nil {
-		return err
-	}
 	msg, err := json.Marshal(clientInfo{
 		Version:     ProtocolVersion,
 		MeshKey:     c.meshKey,
@@ -189,24 +180,23 @@ func (c *Client) sendClientKey() error {
 	if err != nil {
 		return err
 	}
-	msgbox := box.Seal(nil, msg, &nonce, c.serverKey.B32(), c.privateKey.B32())
+	msgbox := c.privateKey.SealTo(c.serverKey, msg)
 
-	buf := make([]byte, 0, nonceLen+keyLen+len(msgbox))
-	buf = append(buf, c.publicKey[:]...)
-	buf = append(buf, nonce[:]...)
+	buf := make([]byte, 0, keyLen+len(msgbox))
+	buf = c.publicKey.AppendTo(buf)
 	buf = append(buf, msgbox...)
 	return writeFrame(c.bw, frameClientInfo, buf)
 }
 
 // ServerPublicKey returns the server's public key.
-func (c *Client) ServerPublicKey() key.Public { return c.serverKey }
+func (c *Client) ServerPublicKey() key.NodePublic { return c.serverKey }
 
 // Send sends a packet to the Tailscale node identified by dstKey.
 //
 // It is an error if the packet is larger than 64KB.
-func (c *Client) Send(dstKey key.Public, pkt []byte) error { return c.send(dstKey, pkt) }
+func (c *Client) Send(dstKey key.NodePublic, pkt []byte) error { return c.send(dstKey, pkt) }
 
-func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
+func (c *Client) send(dstKey key.NodePublic, pkt []byte) (ret error) {
 	defer func() {
 		if ret != nil {
 			ret = fmt.Errorf("derp.Send: %w", ret)
@@ -220,15 +210,15 @@ func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
 	if c.rate != nil {
-		pktLen := frameHeaderLen + len(dstKey) + len(pkt)
+		pktLen := frameHeaderLen + key.NodePublicRawLen + len(pkt)
 		if !c.rate.AllowN(time.Now(), pktLen) {
 			return nil // drop
 		}
 	}
-	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(len(dstKey)+len(pkt))); err != nil {
+	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(key.NodePublicRawLen+len(pkt))); err != nil {
 		return err
 	}
-	if _, err := c.bw.Write(dstKey[:]); err != nil {
+	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(pkt); err != nil {
@@ -237,7 +227,7 @@ func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
 	return c.bw.Flush()
 }
 
-func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error) {
+func (c *Client) ForwardPacket(srcKey, dstKey key.NodePublic, pkt []byte) (err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("derp.ForwardPacket: %w", err)
@@ -257,10 +247,10 @@ func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error
 	if err := writeFrameHeader(c.bw, frameForwardPacket, uint32(keyLen*2+len(pkt))); err != nil {
 		return err
 	}
-	if _, err := c.bw.Write(srcKey[:]); err != nil {
+	if _, err := c.bw.Write(srcKey.AppendTo(nil)); err != nil {
 		return err
 	}
-	if _, err := c.bw.Write(dstKey[:]); err != nil {
+	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(pkt); err != nil {
@@ -322,10 +312,10 @@ func (c *Client) WatchConnectionChanges() error {
 
 // ClosePeer asks the server to close target's TCP connection.
 // It's a fatal error if the client wasn't created using MeshKey.
-func (c *Client) ClosePeer(target key.Public) error {
+func (c *Client) ClosePeer(target key.NodePublic) error {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
-	return writeFrame(c.bw, frameClosePeer, target[:])
+	return writeFrame(c.bw, frameClosePeer, target.AppendTo(nil))
 }
 
 // ReceivedMessage represents a type returned by Client.Recv. Unless
@@ -338,7 +328,7 @@ type ReceivedMessage interface {
 
 // ReceivedPacket is a ReceivedMessage representing an incoming packet.
 type ReceivedPacket struct {
-	Source key.Public
+	Source key.NodePublic
 	// Data is the received packet bytes. It aliases the memory
 	// passed to Client.Recv.
 	Data []byte
@@ -349,13 +339,13 @@ func (ReceivedPacket) msg() {}
 // PeerGoneMessage is a ReceivedMessage that indicates that the client
 // identified by the underlying public key had previously sent you a
 // packet but has now disconnected from the server.
-type PeerGoneMessage key.Public
+type PeerGoneMessage key.NodePublic
 
 func (PeerGoneMessage) msg() {}
 
 // PeerPresentMessage is a ReceivedMessage that indicates that the client
 // is connected to the server. (Only used by trusted mesh clients)
-type PeerPresentMessage key.Public
+type PeerPresentMessage key.NodePublic
 
 func (PeerPresentMessage) msg() {}
 
@@ -516,8 +506,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 				c.logf("[unexpected] dropping short peerGone frame from DERP server")
 				continue
 			}
-			var pg PeerGoneMessage
-			copy(pg[:], b[:keyLen])
+			pg := PeerGoneMessage(key.NodePublicFromRaw32(mem.B(b[:keyLen])))
 			return pg, nil
 
 		case framePeerPresent:
@@ -525,8 +514,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 				c.logf("[unexpected] dropping short peerPresent frame from DERP server")
 				continue
 			}
-			var pg PeerPresentMessage
-			copy(pg[:], b[:keyLen])
+			pg := PeerPresentMessage(key.NodePublicFromRaw32(mem.B(b[:keyLen])))
 			return pg, nil
 
 		case frameRecvPacket:
@@ -535,7 +523,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 				c.logf("[unexpected] dropping short packet from DERP server")
 				continue
 			}
-			copy(rp.Source[:], b[:keyLen])
+			rp.Source = key.NodePublicFromRaw32(mem.B(b[:keyLen]))
 			rp.Data = b[keyLen:n]
 			return rp, nil
 

derp/derp_server.go

@@ -34,7 +34,6 @@ import (
 	"time"
 
 	"go4.org/mem"
-	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/time/rate"
 	"inet.af/netaddr"
@@ -52,7 +51,7 @@ var debug, _ = strconv.ParseBool(os.Getenv("DERP_DEBUG_LOGS"))
 
 // verboseDropKeys is the set of destination public keys that should
 // verbosely log whenever DERP drops a packet.
-var verboseDropKeys = map[key.Public]bool{}
+var verboseDropKeys = map[key.NodePublic]bool{}
 
 func init() {
 	keys := os.Getenv("TS_DEBUG_VERBOSE_DROPS")
@@ -60,7 +59,7 @@ func init() {
 		return
 	}
 	for _, keyStr := range strings.Split(keys, ",") {
-		k, err := key.NewPublicFromHexMem(mem.S(keyStr))
+		k, err := key.ParseNodePublicUntyped(mem.S(keyStr))
 		if err != nil {
 			log.Printf("ignoring invalid debug key %q: %v", keyStr, err)
 		} else {
@@ -99,8 +98,8 @@ type Server struct {
 	// before failing when writing to a client.
 	WriteTimeout time.Duration
 
-	privateKey  key.Private
-	publicKey   key.Public
+	privateKey  key.NodePrivate
+	publicKey   key.NodePublic
 	logf        logger.Logf
 	memSys0     uint64 // runtime.MemStats.Sys at start (or early-ish)
 	meshKey     string
@@ -146,22 +145,22 @@ type Server struct {
 	mu       sync.Mutex
 	closed   bool
 	netConns map[Conn]chan struct{} // chan is closed when conn closes
-	clients  map[key.Public]clientSet
+	clients  map[key.NodePublic]clientSet
 	watchers map[*sclient]bool // mesh peer -> true
 	// clientsMesh tracks all clients in the cluster, both locally
 	// and to mesh peers.  If the value is nil, that means the
 	// peer is only local (and thus in the clients Map, but not
 	// remote). If the value is non-nil, it's remote (+ maybe also
 	// local).
-	clientsMesh map[key.Public]PacketForwarder
+	clientsMesh map[key.NodePublic]PacketForwarder
 	// sentTo tracks which peers have sent to which other peers,
 	// and at which connection number. This isn't on sclient
 	// because it includes intra-region forwarded packets as the
 	// src.
-	sentTo map[key.Public]map[key.Public]int64 // src => dst => dst's latest sclient.connNum
+	sentTo map[key.NodePublic]map[key.NodePublic]int64 // src => dst => dst's latest sclient.connNum
 
 	// maps from netaddr.IPPort to a client's public key
-	keyOfAddr map[netaddr.IPPort]key.Public
+	keyOfAddr map[netaddr.IPPort]key.NodePublic
 }
 
 // clientSet represents 1 or more *sclients.
@@ -277,7 +276,7 @@ func (s *dupClientSet) removeClient(c *sclient) bool {
 // is a multiForwarder, which this package creates as needed if a
 // public key gets more than one PacketForwarder registered for it.
 type PacketForwarder interface {
-	ForwardPacket(src, dst key.Public, payload []byte) error
+	ForwardPacket(src, dst key.NodePublic, payload []byte) error
 }
 
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
@@ -294,7 +293,7 @@ type Conn interface {
 
 // NewServer returns a new DERP server. It doesn't listen on its own.
 // Connections are given to it via Server.Accept.
-func NewServer(privateKey key.Private, logf logger.Logf) *Server {
+func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 	var ms runtime.MemStats
 	runtime.ReadMemStats(&ms)
 
@@ -306,14 +305,14 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		packetsRecvByKind:    metrics.LabelMap{Label: "kind"},
 		packetsDroppedReason: metrics.LabelMap{Label: "reason"},
 		packetsDroppedType:   metrics.LabelMap{Label: "type"},
-		clients:              map[key.Public]clientSet{},
-		clientsMesh:          map[key.Public]PacketForwarder{},
+		clients:              map[key.NodePublic]clientSet{},
+		clientsMesh:          map[key.NodePublic]PacketForwarder{},
 		netConns:             map[Conn]chan struct{}{},
 		memSys0:              ms.Sys,
 		watchers:             map[*sclient]bool{},
-		sentTo:               map[key.Public]map[key.Public]int64{},
+		sentTo:               map[key.NodePublic]map[key.NodePublic]int64{},
 		avgQueueDuration:     new(uint64),
-		keyOfAddr:            map[netaddr.IPPort]key.Public{},
+		keyOfAddr:            map[netaddr.IPPort]key.NodePublic{},
 	}
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
@@ -353,10 +352,10 @@ func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
 func (s *Server) MeshKey() string { return s.meshKey }
 
 // PrivateKey returns the server's private key.
-func (s *Server) PrivateKey() key.Private { return s.privateKey }
+func (s *Server) PrivateKey() key.NodePrivate { return s.privateKey }
 
 // PublicKey returns the server's public key.
-func (s *Server) PublicKey() key.Public { return s.publicKey }
+func (s *Server) PublicKey() key.NodePublic { return s.publicKey }
 
 // Close closes the server and waits for the connections to disconnect.
 func (s *Server) Close() error {
@@ -447,7 +446,7 @@ func (s *Server) initMetacert() {
 	tmpl := &x509.Certificate{
 		SerialNumber: big.NewInt(ProtocolVersion),
 		Subject: pkix.Name{
-			CommonName: fmt.Sprintf("derpkey%x", s.publicKey[:]),
+			CommonName: fmt.Sprintf("derpkey%s", s.publicKey.UntypedHexString()),
 		},
 		// Windows requires NotAfter and NotBefore set:
 		NotAfter:  time.Now().Add(30 * 24 * time.Hour),
@@ -515,7 +514,7 @@ func (s *Server) registerClient(c *sclient) {
 // presence changed.
 //
 // s.mu must be held.
-func (s *Server) broadcastPeerStateChangeLocked(peer key.Public, present bool) {
+func (s *Server) broadcastPeerStateChangeLocked(peer key.NodePublic, present bool) {
 	for w := range s.watchers {
 		w.peerStateChange = append(w.peerStateChange, peerConnState{peer: peer, present: present})
 		go w.requestMeshUpdate()
@@ -577,7 +576,7 @@ func (s *Server) unregisterClient(c *sclient) {
 // key has sent to previously (whether those sends were from a local
 // client or forwarded).  It must only be called after the key has
 // been removed from clientsMesh.
-func (s *Server) notePeerGoneFromRegionLocked(key key.Public) {
+func (s *Server) notePeerGoneFromRegionLocked(key key.NodePublic) {
 	if _, ok := s.clientsMesh[key]; ok {
 		panic("usage")
 	}
@@ -663,7 +662,7 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		connectedAt:    time.Now(),
 		sendQueue:      make(chan pkt, perClientSendQueueDepth),
 		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
-		peerGone:       make(chan key.Public),
+		peerGone:       make(chan key.NodePublic),
 		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
 	}
 
@@ -774,8 +773,8 @@ func (c *sclient) handleFrameClosePeer(ft frameType, fl uint32) error {
 	if !c.canMesh {
 		return fmt.Errorf("insufficient permissions")
 	}
-	var targetKey key.Public
-	if _, err := io.ReadFull(c.br, targetKey[:]); err != nil {
+	var targetKey key.NodePublic
+	if err := targetKey.ReadRawWithoutAllocating(c.br); err != nil {
 		return err
 	}
 	s := c.s
@@ -845,10 +844,10 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 // notePeerSendLocked records that src sent to dst.  We keep track of
 // that so when src disconnects, we can tell dst (if it's still
 // around) that src is gone (a peerGone frame).
-func (s *Server) notePeerSendLocked(src key.Public, dst *sclient) {
+func (s *Server) notePeerSendLocked(src key.NodePublic, dst *sclient) {
 	m, ok := s.sentTo[src]
 	if !ok {
-		m = map[key.Public]int64{}
+		m = map[key.NodePublic]int64{}
 		s.sentTo[src] = m
 	}
 	m[dst.key] = dst.connNum
@@ -919,7 +918,7 @@ const (
 	dropReasonDupClient                          // the public key is connected 2+ times (active/active, fighting)
 )
 
-func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.Public, reason dropReason) {
+func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) {
 	s.packetsDropped.Add(1)
 	s.packetsDroppedReasonCounters[reason].Add(1)
 	if disco.LooksLikeDiscoWrapper(packetBytes) {
@@ -982,7 +981,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 // requestPeerGoneWrite sends a request to write a "peer gone" frame
 // that the provided peer has disconnected. It blocks until either the
 // write request is scheduled, or the client has closed.
-func (c *sclient) requestPeerGoneWrite(peer key.Public) {
+func (c *sclient) requestPeerGoneWrite(peer key.NodePublic) {
 	select {
 	case c.peerGone <- peer:
 	case <-c.done:
@@ -999,7 +998,7 @@ func (c *sclient) requestMeshUpdate() {
 	}
 }
 
-func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
+func (s *Server) verifyClient(clientKey key.NodePublic, info *clientInfo) error {
 	if !s.verifyClients {
 		return nil
 	}
@@ -1018,9 +1017,9 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 }
 
 func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
-	buf := make([]byte, 0, len(magic)+len(s.publicKey))
+	buf := make([]byte, 0, len(magic)+key.NodePublicRawLen)
 	buf = append(buf, magic...)
-	buf = append(buf, s.publicKey[:]...)
+	buf = s.publicKey.AppendTo(buf)
 	err := writeFrame(lw.bw(), frameServerKey, buf)
 	lw.Flush() // redundant (no-op) flush to release bufio.Writer
 	return err
@@ -1084,21 +1083,14 @@ type serverInfo struct {
 	TokenBucketBytesBurst     int `json:",omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
-	var nonce [24]byte
-	if _, err := crand.Read(nonce[:]); err != nil {
-		return err
-	}
+func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.NodePublic) error {
 	msg, err := json.Marshal(serverInfo{Version: ProtocolVersion})
 	if err != nil {
 		return err
 	}
 
-	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
-		return err
-	}
-	if _, err := bw.Write(nonce[:]); err != nil {
+	msgbox := s.privateKey.SealTo(clientKey, msg)
+	if err := writeFrameHeader(bw.bw(), frameServerInfo, uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(msgbox); err != nil {
@@ -1110,7 +1102,7 @@ func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error
 // recvClientKey reads the frameClientInfo frame from the client (its
 // proof of identity) upon its initial connection. It should be
 // considered especially untrusted at this point.
-func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *clientInfo, err error) {
+func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.NodePublic, info *clientInfo, err error) {
 	fl, err := readFrameTypeHeader(br, frameClientInfo)
 	if err != nil {
 		return zpub, nil, err
@@ -1124,21 +1116,17 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *cl
 	if fl > 256<<10 {
 		return zpub, nil, errors.New("long client info")
 	}
-	if _, err := io.ReadFull(br, clientKey[:]); err != nil {
+	if err := clientKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, nil, err
 	}
-	var nonce [24]byte
-	if _, err := io.ReadFull(br, nonce[:]); err != nil {
-		return zpub, nil, fmt.Errorf("nonce: %v", err)
-	}
-	msgLen := int(fl - minLen)
+	msgLen := int(fl - keyLen)
 	msgbox := make([]byte, msgLen)
 	if _, err := io.ReadFull(br, msgbox); err != nil {
 		return zpub, nil, fmt.Errorf("msgbox: %v", err)
 	}
-	msg, ok := box.Open(nil, msgbox, &nonce, (*[32]byte)(&clientKey), s.privateKey.B32())
+	msg, ok := s.privateKey.OpenFrom(clientKey, msgbox)
 	if !ok {
-		return zpub, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %x", msgLen, clientKey[:])
+		return zpub, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %s", msgLen, clientKey)
 	}
 	info = new(clientInfo)
 	if err := json.Unmarshal(msg, info); err != nil {
@@ -1147,11 +1135,11 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *cl
 	return clientKey, info, nil
 }
 
-func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.Public, contents []byte, err error) {
+func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.NodePublic, contents []byte, err error) {
 	if frameLen < keyLen {
 		return zpub, nil, errors.New("short send packet frame")
 	}
-	if err := readPublicKey(br, &dstKey); err != nil {
+	if err := dstKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, nil, err
 	}
 	packetLen := frameLen - keyLen
@@ -1172,17 +1160,17 @@ func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.Publi
 	return dstKey, contents, nil
 }
 
-// zpub is the key.Public zero value.
-var zpub key.Public
+// zpub is the key.NodePublic zero value.
+var zpub key.NodePublic
 
-func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, dstKey key.Public, contents []byte, err error) {
+func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, dstKey key.NodePublic, contents []byte, err error) {
 	if frameLen < keyLen*2 {
 		return zpub, zpub, nil, errors.New("short send packet frame")
 	}
-	if _, err := io.ReadFull(br, srcKey[:]); err != nil {
+	if err := srcKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, zpub, nil, err
 	}
-	if _, err := io.ReadFull(br, dstKey[:]); err != nil {
+	if err := dstKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, zpub, nil, err
 	}
 	packetLen := frameLen - keyLen*2
@@ -1206,19 +1194,19 @@ type sclient struct {
 	connNum        int64 // process-wide unique counter, incremented each Accept
 	s              *Server
 	nc             Conn
-	key            key.Public
+	key            key.NodePublic
 	info           clientInfo
 	logf           logger.Logf
-	done           <-chan struct{}  // closed when connection closes
-	remoteAddr     string           // usually ip:port from net.Conn.RemoteAddr().String()
-	remoteIPPort   netaddr.IPPort   // zero if remoteAddr is not ip:port.
-	sendQueue      chan pkt         // packets queued to this client; never closed
-	discoSendQueue chan pkt         // important packets queued to this client; never closed
-	peerGone       chan key.Public  // write request that a previous sender has disconnected (not used by mesh peers)
-	meshUpdate     chan struct{}    // write request to write peerStateChange
-	canMesh        bool             // clientInfo had correct mesh token for inter-region routing
-	isDup          syncs.AtomicBool // whether more than 1 sclient for key is connected
-	isDisabled     syncs.AtomicBool // whether sends to this peer are disabled due to active/active dups
+	done           <-chan struct{}     // closed when connection closes
+	remoteAddr     string              // usually ip:port from net.Conn.RemoteAddr().String()
+	remoteIPPort   netaddr.IPPort      // zero if remoteAddr is not ip:port.
+	sendQueue      chan pkt            // packets queued to this client; never closed
+	discoSendQueue chan pkt            // important packets queued to this client; never closed
+	peerGone       chan key.NodePublic // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate     chan struct{}       // write request to write peerStateChange
+	canMesh        bool                // clientInfo had correct mesh token for inter-region routing
+	isDup          syncs.AtomicBool    // whether more than 1 sclient for key is connected
+	isDisabled     syncs.AtomicBool    // whether sends to this peer are disabled due to active/active dups
 
 	// replaceLimiter controls how quickly two connections with
 	// the same client key can kick each other off the server by
@@ -1245,14 +1233,14 @@ type sclient struct {
 // peerConnState represents whether a peer is connected to the server
 // or not.
 type peerConnState struct {
-	peer    key.Public
+	peer    key.NodePublic
 	present bool
 }
 
 // pkt is a request to write a data frame to an sclient.
 type pkt struct {
 	// src is the who's the sender of the packet.
-	src key.Public
+	src key.NodePublic
 
 	// enqueuedAt is when a packet was put onto a queue before it was sent,
 	// and is used for reporting metrics on the duration of packets in the queue.
@@ -1397,23 +1385,23 @@ func (c *sclient) sendKeepAlive() error {
 }
 
 // sendPeerGone sends a peerGone frame, without flushing.
-func (c *sclient) sendPeerGone(peer key.Public) error {
+func (c *sclient) sendPeerGone(peer key.NodePublic) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
 	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
 		return err
 	}
-	_, err := c.bw.Write(peer[:])
+	_, err := c.bw.Write(peer.AppendTo(nil))
 	return err
 }
 
 // sendPeerPresent sends a peerPresent frame, without flushing.
-func (c *sclient) sendPeerPresent(peer key.Public) error {
+func (c *sclient) sendPeerPresent(peer key.NodePublic) error {
 	c.setWriteDeadline()
 	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
 		return err
 	}
-	_, err := c.bw.Write(peer[:])
+	_, err := c.bw.Write(peer.AppendTo(nil))
 	return err
 }
 
@@ -1465,7 +1453,7 @@ func (c *sclient) sendMeshUpdates() error {
 // DERPv2. The bytes of contents are only valid until this function
 // returns, do not retain slices.
 // It does not flush its bufio.Writer.
-func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
+func (c *sclient) sendPacket(srcKey key.NodePublic, contents []byte) (err error) {
 	defer func() {
 		// Stats update.
 		if err != nil {
@@ -1481,14 +1469,13 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	withKey := !srcKey.IsZero()
 	pktLen := len(contents)
 	if withKey {
-		pktLen += len(srcKey)
+		pktLen += key.NodePublicRawLen
 	}
 	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw.bw(), &srcKey)
-		if err != nil {
+		if err := srcKey.WriteRawWithoutAllocating(c.bw.bw()); err != nil {
 			return err
 		}
 	}
@@ -1498,7 +1485,7 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 
 // AddPacketForwarder registers fwd as a packet forwarder for dst.
 // fwd must be comparable.
-func (s *Server) AddPacketForwarder(dst key.Public, fwd PacketForwarder) {
+func (s *Server) AddPacketForwarder(dst key.NodePublic, fwd PacketForwarder) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if prev, ok := s.clientsMesh[dst]; ok {
@@ -1530,7 +1517,7 @@ func (s *Server) AddPacketForwarder(dst key.Public, fwd PacketForwarder) {
 
 // RemovePacketForwarder removes fwd as a packet forwarder for dst.
 // fwd must be comparable.
-func (s *Server) RemovePacketForwarder(dst key.Public, fwd PacketForwarder) {
+func (s *Server) RemovePacketForwarder(dst key.NodePublic, fwd PacketForwarder) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	v, ok := s.clientsMesh[dst]
@@ -1592,7 +1579,7 @@ func (m multiForwarder) maxVal() (max uint8) {
 	return
 }
 
-func (m multiForwarder) ForwardPacket(src, dst key.Public, payload []byte) error {
+func (m multiForwarder) ForwardPacket(src, dst key.NodePublic, payload []byte) error {
 	var fwd PacketForwarder
 	var lowest uint8
 	for k, v := range m {
@@ -1692,37 +1679,6 @@ func (s *Server) ConsistencyCheck() error {
 	return errors.New(strings.Join(errs, ", "))
 }
 
-// readPublicKey reads key from br.
-// It is ~4x slower than io.ReadFull(br, key),
-// but it prevents key from escaping and thus being allocated.
-// If io.ReadFull(br, key) does not cause key to escape, use that instead.
-func readPublicKey(br *bufio.Reader, key *key.Public) error {
-	// Do io.ReadFull(br, key), but one byte at a time, to avoid allocation.
-	for i := range key {
-		b, err := br.ReadByte()
-		if err != nil {
-			return err
-		}
-		key[i] = b
-	}
-	return nil
-}
-
-// writePublicKey writes key to bw.
-// It is ~3x slower than bw.Write(key[:]),
-// but it prevents key from escaping and thus being allocated.
-// If bw.Write(key[:]) does not cause key to escape, use that instead.
-func writePublicKey(bw *bufio.Writer, key *key.Public) error {
-	// Do bw.Write(key[:]), but one byte at a time to avoid allocation.
-	for _, b := range key {
-		err := bw.WriteByte(b)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 const minTimeBetweenLogs = 2 * time.Second
 
 // BytesSentRecv records the number of bytes that have been sent since the last traffic check
@@ -1731,7 +1687,7 @@ type BytesSentRecv struct {
 	Sent uint64
 	Recv uint64
 	// Key is the public key of the client which sent/received these bytes.
-	Key key.Public
+	Key key.NodePublic
 }
 
 // parseSSOutput parses the output from the specific call to ss in ServeDebugTraffic.

derp/derp_test.go

@@ -8,7 +8,6 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	crand "crypto/rand"
 	"crypto/x509"
 	"encoding/json"
 	"errors"
@@ -23,20 +22,13 @@ import (
 	"testing"
 	"time"
 
+	"go4.org/mem"
 	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
 
-func newPrivateKey(tb testing.TB) (k key.Private) {
-	tb.Helper()
-	if _, err := crand.Read(k[:]); err != nil {
-		tb.Fatal(err)
-	}
-	return
-}
-
 func TestClientInfoUnmarshal(t *testing.T) {
 	for i, in := range []string{
 		`{"Version":5,"MeshKey":"abc"}`,
@@ -54,15 +46,15 @@ func TestClientInfoUnmarshal(t *testing.T) {
 }
 
 func TestSendRecv(t *testing.T) {
-	serverPrivateKey := newPrivateKey(t)
+	serverPrivateKey := key.NewNode()
 	s := NewServer(serverPrivateKey, t.Logf)
 	defer s.Close()
 
 	const numClients = 3
-	var clientPrivateKeys []key.Private
-	var clientKeys []key.Public
+	var clientPrivateKeys []key.NodePrivate
+	var clientKeys []key.NodePublic
 	for i := 0; i < numClients; i++ {
-		priv := newPrivateKey(t)
+		priv := key.NewNode()
 		clientPrivateKeys = append(clientPrivateKeys, priv)
 		clientKeys = append(clientKeys, priv.Public())
 	}
@@ -225,7 +217,7 @@ func TestSendRecv(t *testing.T) {
 }
 
 func TestSendFreeze(t *testing.T) {
-	serverPrivateKey := newPrivateKey(t)
+	serverPrivateKey := key.NewNode()
 	s := NewServer(serverPrivateKey, t.Logf)
 	defer s.Close()
 	s.WriteTimeout = 100 * time.Millisecond
@@ -238,7 +230,7 @@ func TestSendFreeze(t *testing.T) {
 	// Then cathy stops processing messsages.
 	// That should not interfere with alice talking to bob.
 
-	newClient := func(name string, k key.Private) (c *Client, clientConn nettest.Conn) {
+	newClient := func(name string, k key.NodePrivate) (c *Client, clientConn nettest.Conn) {
 		t.Helper()
 		c1, c2 := nettest.NewConn(name, 1024)
 		go s.Accept(c1, bufio.NewReadWriter(bufio.NewReader(c1), bufio.NewWriter(c1)), name)
@@ -252,13 +244,13 @@ func TestSendFreeze(t *testing.T) {
 		return c, c2
 	}
 
-	aliceKey := newPrivateKey(t)
+	aliceKey := key.NewNode()
 	aliceClient, aliceConn := newClient("alice", aliceKey)
 
-	bobKey := newPrivateKey(t)
+	bobKey := key.NewNode()
 	bobClient, bobConn := newClient("bob", bobKey)
 
-	cathyKey := newPrivateKey(t)
+	cathyKey := key.NewNode()
 	cathyClient, cathyConn := newClient("cathy", cathyKey)
 
 	var (
@@ -427,7 +419,7 @@ type testServer struct {
 	logf logger.Logf
 
 	mu      sync.Mutex
-	pubName map[key.Public]string
+	pubName map[key.NodePublic]string
 	clients map[*testClient]bool
 }
 
@@ -437,14 +429,14 @@ func (ts *testServer) addTestClient(c *testClient) {
 	ts.clients[c] = true
 }
 
-func (ts *testServer) addKeyName(k key.Public, name string) {
+func (ts *testServer) addKeyName(k key.NodePublic, name string) {
 	ts.mu.Lock()
 	defer ts.mu.Unlock()
 	ts.pubName[k] = name
 	ts.logf("test adding named key %q for %x", name, k)
 }
 
-func (ts *testServer) keyName(k key.Public) string {
+func (ts *testServer) keyName(k key.NodePublic) string {
 	ts.mu.Lock()
 	defer ts.mu.Unlock()
 	if name, ok := ts.pubName[k]; ok {
@@ -465,7 +457,7 @@ func (ts *testServer) close(t *testing.T) error {
 func newTestServer(t *testing.T) *testServer {
 	t.Helper()
 	logf := logger.WithPrefix(t.Logf, "derp-server: ")
-	s := NewServer(newPrivateKey(t), logf)
+	s := NewServer(key.NewNode(), logf)
 	s.SetMeshKey("mesh-key")
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -491,7 +483,7 @@ func newTestServer(t *testing.T) *testServer {
 		ln:      ln,
 		logf:    logf,
 		clients: map[*testClient]bool{},
-		pubName: map[key.Public]string{},
+		pubName: map[key.NodePublic]string{},
 	}
 }
 
@@ -499,20 +491,20 @@ type testClient struct {
 	name   string
 	c      *Client
 	nc     net.Conn
-	pub    key.Public
+	pub    key.NodePublic
 	ts     *testServer
 	closed bool
 }
 
-func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net.Conn, key.Private, logger.Logf) (*Client, error)) *testClient {
+func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net.Conn, key.NodePrivate, logger.Logf) (*Client, error)) *testClient {
 	t.Helper()
 	nc, err := net.Dial("tcp", ts.ln.Addr().String())
 	if err != nil {
 		t.Fatal(err)
 	}
-	key := newPrivateKey(t)
-	ts.addKeyName(key.Public(), name)
-	c, err := newClient(nc, key, logger.WithPrefix(t.Logf, "client-"+name+": "))
+	k := key.NewNode()
+	ts.addKeyName(k.Public(), name)
+	c, err := newClient(nc, k, logger.WithPrefix(t.Logf, "client-"+name+": "))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -521,14 +513,14 @@ func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net
 		nc:   nc,
 		c:    c,
 		ts:   ts,
-		pub:  key.Public(),
+		pub:  k.Public(),
 	}
 	ts.addTestClient(tc)
 	return tc
 }
 
 func newRegularClient(t *testing.T, ts *testServer, name string) *testClient {
-	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.NodePrivate, logf logger.Logf) (*Client, error) {
 		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
 		c, err := NewClient(priv, nc, brw, logf)
 		if err != nil {
@@ -541,7 +533,7 @@ func newRegularClient(t *testing.T, ts *testServer, name string) *testClient {
 }
 
 func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient {
-	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.NodePrivate, logf logger.Logf) (*Client, error) {
 		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
 		c, err := NewClient(priv, nc, brw, logf, MeshKey("mesh-key"))
 		if err != nil {
@@ -555,9 +547,9 @@ func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient {
 	})
 }
 
-func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
+func (tc *testClient) wantPresent(t *testing.T, peers ...key.NodePublic) {
 	t.Helper()
-	want := map[key.Public]bool{}
+	want := map[key.NodePublic]bool{}
 	for _, k := range peers {
 		want[k] = true
 	}
@@ -569,7 +561,7 @@ func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
 		}
 		switch m := m.(type) {
 		case PeerPresentMessage:
-			got := key.Public(m)
+			got := key.NodePublic(m)
 			if !want[got] {
 				t.Fatalf("got peer present for %v; want present for %v", tc.ts.keyName(got), logger.ArgWriter(func(bw *bufio.Writer) {
 					for _, pub := range peers {
@@ -587,7 +579,7 @@ func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
 	}
 }
 
-func (tc *testClient) wantGone(t *testing.T, peer key.Public) {
+func (tc *testClient) wantGone(t *testing.T, peer key.NodePublic) {
 	t.Helper()
 	m, err := tc.c.recvTimeout(time.Second)
 	if err != nil {
@@ -595,7 +587,7 @@ func (tc *testClient) wantGone(t *testing.T, peer key.Public) {
 	}
 	switch m := m.(type) {
 	case PeerGoneMessage:
-		got := key.Public(m)
+		got := key.NodePublic(m)
 		if peer != got {
 			t.Errorf("got gone message for %v; want gone for %v", tc.ts.keyName(got), tc.ts.keyName(peer))
 		}
@@ -654,21 +646,24 @@ func TestWatch(t *testing.T) {
 
 type testFwd int
 
-func (testFwd) ForwardPacket(key.Public, key.Public, []byte) error { panic("not called in tests") }
+func (testFwd) ForwardPacket(key.NodePublic, key.NodePublic, []byte) error {
+	panic("not called in tests")
+}
 
-func pubAll(b byte) (ret key.Public) {
-	for i := range ret {
-		ret[i] = b
+func pubAll(b byte) (ret key.NodePublic) {
+	var bs [32]byte
+	for i := range bs {
+		bs[i] = b
 	}
-	return
+	return key.NodePublicFromRaw32(mem.B(bs[:]))
 }
 
 func TestForwarderRegistration(t *testing.T) {
 	s := &Server{
-		clients:     make(map[key.Public]clientSet),
-		clientsMesh: map[key.Public]PacketForwarder{},
+		clients:     make(map[key.NodePublic]clientSet),
+		clientsMesh: map[key.NodePublic]PacketForwarder{},
 	}
-	want := func(want map[key.Public]PacketForwarder) {
+	want := func(want map[key.NodePublic]PacketForwarder) {
 		t.Helper()
 		if got := s.clientsMesh; !reflect.DeepEqual(got, want) {
 			t.Fatalf("mismatch\n got: %v\nwant: %v\n", got, want)
@@ -687,28 +682,28 @@ func TestForwarderRegistration(t *testing.T) {
 
 	s.AddPacketForwarder(u1, testFwd(1))
 	s.AddPacketForwarder(u2, testFwd(2))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 		u2: testFwd(2),
 	})
 
 	// Verify a remove of non-registered forwarder is no-op.
 	s.RemovePacketForwarder(u2, testFwd(999))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 		u2: testFwd(2),
 	})
 
 	// Verify a remove of non-registered user is no-op.
 	s.RemovePacketForwarder(u3, testFwd(1))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 		u2: testFwd(2),
 	})
 
 	// Actual removal.
 	s.RemovePacketForwarder(u2, testFwd(2))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 	})
 
@@ -716,7 +711,7 @@ func TestForwarderRegistration(t *testing.T) {
 	wantCounter(&s.multiForwarderCreated, 0)
 	s.AddPacketForwarder(u1, testFwd(100))
 	s.AddPacketForwarder(u1, testFwd(100)) // dup to trigger dup path
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: multiForwarder{
 			testFwd(1):   1,
 			testFwd(100): 2,
@@ -726,7 +721,7 @@ func TestForwarderRegistration(t *testing.T) {
 
 	// Removing a forwarder in a multi set that doesn't exist; does nothing.
 	s.RemovePacketForwarder(u1, testFwd(55))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: multiForwarder{
 			testFwd(1):   1,
 			testFwd(100): 2,
@@ -737,7 +732,7 @@ func TestForwarderRegistration(t *testing.T) {
 	// from being a multiForwarder.
 	wantCounter(&s.multiForwarderDeleted, 0)
 	s.RemovePacketForwarder(u1, testFwd(1))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(100),
 	})
 	wantCounter(&s.multiForwarderDeleted, 1)
@@ -750,18 +745,18 @@ func TestForwarderRegistration(t *testing.T) {
 	}
 	s.clients[u1] = singleClient{u1c}
 	s.RemovePacketForwarder(u1, testFwd(100))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: nil,
 	})
 
 	// But once that client disconnects, it should go away.
 	s.unregisterClient(u1c)
-	want(map[key.Public]PacketForwarder{})
+	want(map[key.NodePublic]PacketForwarder{})
 
 	// But if it already has a forwarder, it's not removed.
 	s.AddPacketForwarder(u1, testFwd(2))
 	s.unregisterClient(u1c)
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(2),
 	})
 
@@ -770,17 +765,17 @@ func TestForwarderRegistration(t *testing.T) {
 	// from nil to the new one, not a multiForwarder.
 	s.clients[u1] = singleClient{u1c}
 	s.clientsMesh[u1] = nil
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: nil,
 	})
 	s.AddPacketForwarder(u1, testFwd(3))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(3),
 	})
 }
 
 func TestMetaCert(t *testing.T) {
-	priv := newPrivateKey(t)
+	priv := key.NewNode()
 	pub := priv.Public()
 	s := NewServer(priv, t.Logf)
 
@@ -792,7 +787,7 @@ func TestMetaCert(t *testing.T) {
 	if fmt.Sprint(cert.SerialNumber) != fmt.Sprint(ProtocolVersion) {
 		t.Errorf("serial = %v; want %v", cert.SerialNumber, ProtocolVersion)
 	}
-	if g, w := cert.Subject.CommonName, fmt.Sprintf("derpkey%x", pub[:]); g != w {
+	if g, w := cert.Subject.CommonName, fmt.Sprintf("derpkey%s", pub.UntypedHexString()); g != w {
 		t.Errorf("CommonName = %q; want %q", g, w)
 	}
 }
@@ -882,10 +877,10 @@ func TestClientSendPong(t *testing.T) {
 }
 
 func TestServerDupClients(t *testing.T) {
-	serverPriv := newPrivateKey(t)
+	serverPriv := key.NewNode()
 	var s *Server
 
-	clientPriv := newPrivateKey(t)
+	clientPriv := key.NewNode()
 	clientPub := clientPriv.Public()
 
 	var c1, c2, c3 *sclient
@@ -1141,12 +1136,12 @@ func BenchmarkSendRecv(b *testing.B) {
 }
 
 func benchmarkSendRecvSize(b *testing.B, packetSize int) {
-	serverPrivateKey := newPrivateKey(b)
+	serverPrivateKey := key.NewNode()
 	s := NewServer(serverPrivateKey, logger.Discard)
 	defer s.Close()
 
-	key := newPrivateKey(b)
-	clientKey := key.Public()
+	k := key.NewNode()
+	clientKey := k.Public()
 
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -1170,7 +1165,7 @@ func benchmarkSendRecvSize(b *testing.B, packetSize int) {
 	go s.Accept(connIn, brwServer, "test-client")
 
 	brw := bufio.NewReadWriter(bufio.NewReader(connOut), bufio.NewWriter(connOut))
-	client, err := NewClient(key, connOut, brw, logger.Discard)
+	client, err := NewClient(k, connOut, brw, logger.Discard)
 	if err != nil {
 		b.Fatalf("client: %v", err)
 	}
@@ -1279,7 +1274,7 @@ func TestClientSendRateLimiting(t *testing.T) {
 	c.setSendRateLimiter(ServerInfoMessage{})
 
 	pkt := make([]byte, 1000)
-	if err := c.send(key.Public{}, pkt); err != nil {
+	if err := c.send(key.NodePublic{}, pkt); err != nil {
 		t.Fatal(err)
 	}
 	writes1, bytes1 := cw.Stats()
@@ -1290,7 +1285,7 @@ func TestClientSendRateLimiting(t *testing.T) {
 	// Flood should all succeed.
 	cw.ResetStats()
 	for i := 0; i < 1000; i++ {
-		if err := c.send(key.Public{}, pkt); err != nil {
+		if err := c.send(key.NodePublic{}, pkt); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -1309,7 +1304,7 @@ func TestClientSendRateLimiting(t *testing.T) {
 		TokenBucketBytesBurst:     int(bytes1 * 2),
 	})
 	for i := 0; i < 1000; i++ {
-		if err := c.send(key.Public{}, pkt); err != nil {
+		if err := c.send(key.NodePublic{}, pkt); err != nil {
 			t.Fatal(err)
 		}
 	}

derp/derphttp/derphttp_client.go

@@ -22,6 +22,9 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -50,7 +53,7 @@ type Client struct {
 	MeshKey   string             // optional; for trusted clients
 	IsProber  bool               // optional; for probers to optional declare themselves as such
 
-	privateKey key.Private
+	privateKey key.NodePrivate
 	logf       logger.Logf
 	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
@@ -68,12 +71,12 @@ type Client struct {
 	netConn      io.Closer
 	client       *derp.Client
 	connGen      int // incremented once per new connection; valid values are >0
-	serverPubKey key.Public
+	serverPubKey key.NodePublic
 }
 
 // NewRegionClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
-func NewRegionClient(privateKey key.Private, logf logger.Logf, getRegion func() *tailcfg.DERPRegion) *Client {
+func NewRegionClient(privateKey key.NodePrivate, logf logger.Logf, getRegion func() *tailcfg.DERPRegion) *Client {
 	ctx, cancel := context.WithCancel(context.Background())
 	c := &Client{
 		privateKey: privateKey,
@@ -93,7 +96,7 @@ func NewNetcheckClient(logf logger.Logf) *Client {
 
 // NewClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
-func NewClient(privateKey key.Private, serverURL string, logf logger.Logf) (*Client, error) {
+func NewClient(privateKey key.NodePrivate, serverURL string, logf logger.Logf) (*Client, error) {
 	u, err := url.Parse(serverURL)
 	if err != nil {
 		return nil, fmt.Errorf("derphttp.NewClient: %v", err)
@@ -124,14 +127,14 @@ func (c *Client) Connect(ctx context.Context) error {
 //
 // It only returns a non-zero value once a connection has succeeded
 // from an earlier call.
-func (c *Client) ServerPublicKey() key.Public {
+func (c *Client) ServerPublicKey() key.NodePublic {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.serverPubKey
 }
 
 // SelfPublicKey returns our own public key.
-func (c *Client) SelfPublicKey() key.Public {
+func (c *Client) SelfPublicKey() key.NodePublic {
 	return c.privateKey.Public()
 }
 
@@ -177,6 +180,20 @@ func (c *Client) urlString(node *tailcfg.DERPNode) string {
 	return fmt.Sprintf("https://%s/derp", node.HostName)
 }
 
+// dialWebsocketFunc is non-nil (set by websocket.go's init) when compiled in.
+var dialWebsocketFunc func(ctx context.Context, urlStr string) (net.Conn, error)
+
+func useWebsockets() bool {
+	if runtime.GOOS == "js" {
+		return true
+	}
+	if dialWebsocketFunc != nil {
+		v, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_DERP_WS_CLIENT"))
+		return v
+	}
+	return false
+}
+
 func (c *Client) connect(ctx context.Context, caller string) (client *derp.Client, connGen int, err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -229,10 +246,44 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	}()
 
 	var node *tailcfg.DERPNode // nil when using c.url to dial
-	if c.url != nil {
+	switch {
+	case useWebsockets():
+		var urlStr string
+		if c.url != nil {
+			urlStr = c.url.String()
+		} else {
+			urlStr = c.urlString(reg.Nodes[0])
+		}
+		c.logf("%s: connecting websocket to %v", caller, urlStr)
+		conn, err := dialWebsocketFunc(ctx, urlStr)
+		if err != nil {
+			c.logf("%s: websocket to %v error: %v", caller, urlStr, err)
+			return nil, 0, err
+		}
+		brw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
+		derpClient, err := derp.NewClient(c.privateKey, conn, brw, c.logf,
+			derp.MeshKey(c.MeshKey),
+			derp.CanAckPings(c.canAckPings),
+			derp.IsProber(c.IsProber),
+		)
+		if err != nil {
+			return nil, 0, err
+		}
+		if c.preferred {
+			if err := derpClient.NotePreferred(true); err != nil {
+				go conn.Close()
+				return nil, 0, err
+			}
+		}
+		c.serverPubKey = derpClient.ServerPublicKey()
+		c.client = derpClient
+		c.netConn = tcpConn
+		c.connGen++
+		return c.client, c.connGen, nil
+	case c.url != nil:
 		c.logf("%s: connecting to %v", caller, c.url)
 		tcpConn, err = c.dialURL(ctx)
-	} else {
+	default:
 		c.logf("%s: connecting to derp-%d (%v)", caller, reg.RegionID, reg.RegionCode)
 		tcpConn, node, err = c.dialRegion(ctx, reg)
 	}
@@ -264,8 +315,8 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 		}
 	}()
 
-	var httpConn net.Conn    // a TCP conn or a TLS conn; what we speak HTTP to
-	var serverPub key.Public // or zero if unknown (if not using TLS or TLS middlebox eats it)
+	var httpConn net.Conn        // a TCP conn or a TLS conn; what we speak HTTP to
+	var serverPub key.NodePublic // or zero if unknown (if not using TLS or TLS middlebox eats it)
 	var serverProtoVersion int
 	if c.useHTTPS() {
 		tlsConn := c.tlsClient(tcpConn, node)
@@ -378,7 +429,7 @@ func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
 	}
 	hostOrIP := host
-	dialer := netns.NewDialer()
+	dialer := netns.NewDialer(c.logf)
 
 	if c.DNSCache != nil {
 		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
@@ -468,7 +519,7 @@ func (c *Client) DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (tl
 }
 
 func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) {
-	return netns.NewDialer().DialContext(ctx, proto, addr)
+	return netns.NewDialer(c.logf).DialContext(ctx, proto, addr)
 }
 
 // shouldDialProto reports whether an explicitly provided IPv4 or IPv6
@@ -636,7 +687,7 @@ func (c *Client) dialNodeUsingProxy(ctx context.Context, n *tailcfg.DERPNode, pr
 	return proxyConn, nil
 }
 
-func (c *Client) Send(dstKey key.Public, b []byte) error {
+func (c *Client) Send(dstKey key.NodePublic, b []byte) error {
 	client, _, err := c.connect(context.TODO(), "derphttp.Client.Send")
 	if err != nil {
 		return err
@@ -647,7 +698,7 @@ func (c *Client) Send(dstKey key.Public, b []byte) error {
 	return err
 }
 
-func (c *Client) ForwardPacket(from, to key.Public, b []byte) error {
+func (c *Client) ForwardPacket(from, to key.NodePublic, b []byte) error {
 	client, _, err := c.connect(context.TODO(), "derphttp.Client.ForwardPacket")
 	if err != nil {
 		return err
@@ -728,7 +779,7 @@ func (c *Client) WatchConnectionChanges() error {
 // ClosePeer asks the server to close target's TCP connection.
 //
 // Only trusted connections (using MeshKey) are allowed to use this.
-func (c *Client) ClosePeer(target key.Public) error {
+func (c *Client) ClosePeer(target key.NodePublic) error {
 	client, _, err := c.connect(context.TODO(), "derphttp.Client.ClosePeer")
 	if err != nil {
 		return err
@@ -812,15 +863,15 @@ func (c *Client) closeForReconnect(brokenClient *derp.Client) {
 
 var ErrClientClosed = errors.New("derphttp.Client closed")
 
-func parseMetaCert(certs []*x509.Certificate) (serverPub key.Public, serverProtoVersion int) {
+func parseMetaCert(certs []*x509.Certificate) (serverPub key.NodePublic, serverProtoVersion int) {
 	for _, cert := range certs {
 		if cn := cert.Subject.CommonName; strings.HasPrefix(cn, "derpkey") {
 			var err error
-			serverPub, err = key.NewPublicFromHexMem(mem.S(strings.TrimPrefix(cn, "derpkey")))
+			serverPub, err = key.ParseNodePublicUntyped(mem.S(strings.TrimPrefix(cn, "derpkey")))
 			if err == nil && cert.SerialNumber.BitLen() <= 8 { // supports up to version 255
 				return serverPub, int(cert.SerialNumber.Int64())
 			}
 		}
 	}
-	return key.Public{}, 0
+	return key.NodePublic{}, 0
 }

derp/derphttp/derphttp_server.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strings"
 
 	"tailscale.com/derp"
 )
@@ -20,10 +21,15 @@ const fastStartHeader = "Derp-Fast-Start"
 
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if p := r.Header.Get("Upgrade"); p != "WebSocket" && p != "DERP" {
+		up := strings.ToLower(r.Header.Get("Upgrade"))
+		if up != "websocket" && up != "derp" {
+			if up != "" {
+				log.Printf("Weird upgrade: %q", up)
+			}
 			http.Error(w, "DERP requires connection upgrade", http.StatusUpgradeRequired)
 			return
 		}
+
 		fastStart := r.Header.Get(fastStartHeader) == "1"
 
 		h, ok := w.(http.Hijacker)
@@ -45,9 +51,9 @@ func Handler(s *derp.Server) http.Handler {
 				"Upgrade: DERP\r\n"+
 				"Connection: Upgrade\r\n"+
 				"Derp-Version: %v\r\n"+
-				"Derp-Public-Key: %x\r\n\r\n",
+				"Derp-Public-Key: %s\r\n\r\n",
 				derp.ProtocolVersion,
-				pubKey[:])
+				pubKey.UntypedHexString())
 		}
 
 		s.Accept(netConn, conn, netConn.RemoteAddr().String())

derp/derphttp/derphttp_test.go

@@ -18,13 +18,13 @@ import (
 )
 
 func TestSendRecv(t *testing.T) {
-	serverPrivateKey := key.NewPrivate()
+	serverPrivateKey := key.NewNode()
 
 	const numClients = 3
-	var clientPrivateKeys []key.Private
-	var clientKeys []key.Public
+	var clientPrivateKeys []key.NodePrivate
+	var clientKeys []key.NodePublic
 	for i := 0; i < numClients; i++ {
-		priv := key.NewPrivate()
+		priv := key.NewNode()
 		clientPrivateKeys = append(clientPrivateKeys, priv)
 		clientKeys = append(clientKeys, priv.Public())
 	}

derp/derphttp/mesh_client.go

@@ -27,7 +27,7 @@ import (
 //
 // To force RunWatchConnectionLoop to return quickly, its ctx needs to
 // be closed, and c itself needs to be closed.
-func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.Public, infoLogf logger.Logf, add, remove func(key.Public)) {
+func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.NodePublic, infoLogf logger.Logf, add, remove func(key.NodePublic)) {
 	if infoLogf == nil {
 		infoLogf = logger.Discard
 	}
@@ -36,7 +36,7 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 	const statusInterval = 10 * time.Second
 	var (
 		mu              sync.Mutex
-		present         = map[key.Public]bool{}
+		present         = map[key.NodePublic]bool{}
 		loggedConnected = false
 	)
 	clear := func() {
@@ -49,7 +49,7 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 		for k := range present {
 			remove(k)
 		}
-		present = map[key.Public]bool{}
+		present = map[key.NodePublic]bool{}
 	}
 	lastConnGen := 0
 	lastStatus := time.Now()
@@ -69,7 +69,7 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 	})
 	defer timer.Stop()
 
-	updatePeer := func(k key.Public, isPresent bool) {
+	updatePeer := func(k key.NodePublic, isPresent bool) {
 		if isPresent {
 			add(k)
 		} else {
@@ -127,9 +127,9 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 			}
 			switch m := m.(type) {
 			case derp.PeerPresentMessage:
-				updatePeer(key.Public(m), true)
+				updatePeer(key.NodePublic(m), true)
 			case derp.PeerGoneMessage:
-				updatePeer(key.Public(m), false)
+				updatePeer(key.NodePublic(m), false)
 			default:
 				continue
 			}

derp/derphttp/websocket.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux || js
+// +build linux js
+
+package derphttp
+
+import (
+	"context"
+	"log"
+	"net"
+
+	"nhooyr.io/websocket"
+	"tailscale.com/derp/wsconn"
+)
+
+func init() {
+	dialWebsocketFunc = dialWebsocket
+}
+
+func dialWebsocket(ctx context.Context, urlStr string) (net.Conn, error) {
+	c, res, err := websocket.Dial(ctx, urlStr, &websocket.DialOptions{
+		Subprotocols: []string{"derp"},
+	})
+	if err != nil {
+		log.Printf("websocket Dial: %v, %+v", err, res)
+		return nil, err
+	}
+	log.Printf("websocket: connected to %v", urlStr)
+	return wsconn.New(c), nil
+}

derp/wsconn/wsconn.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package wsconn contains an adapter type that turns
+// a websocket connection into a net.Conn.
+package wsconn
+
+import (
+	"context"
+	"net"
+	"sync"
+	"time"
+
+	"nhooyr.io/websocket"
+)
+
+// New returns a net.Conn wrapper around c,
+// using c to send and receive binary messages with
+// chunks of bytes with no defined framing, effectively
+// discarding all WebSocket-level message framing.
+func New(c *websocket.Conn) net.Conn {
+	return &websocketConn{c: c}
+}
+
+// websocketConn implements derp.Conn around a *websocket.Conn,
+// treating a websocket.Conn as a byte stream, ignoring the WebSocket
+// frame/message boundaries.
+type websocketConn struct {
+	c *websocket.Conn
+
+	// rextra are extra bytes owned by the reader.
+	rextra []byte
+
+	mu         sync.Mutex
+	rdeadline  time.Time
+	cancelRead context.CancelFunc
+}
+
+func (wc *websocketConn) LocalAddr() net.Addr  { return addr{} }
+func (wc *websocketConn) RemoteAddr() net.Addr { return addr{} }
+
+type addr struct{}
+
+func (addr) Network() string { return "websocket" }
+func (addr) String() string  { return "websocket" }
+
+func (wc *websocketConn) Read(p []byte) (n int, err error) {
+	// Drain any leftover from previously.
+	n = copy(p, wc.rextra)
+	if n > 0 {
+		wc.rextra = wc.rextra[n:]
+		return n, nil
+	}
+
+	var ctx context.Context
+	var cancel context.CancelFunc
+
+	wc.mu.Lock()
+	if dl := wc.rdeadline; !dl.IsZero() {
+		ctx, cancel = context.WithDeadline(context.Background(), wc.rdeadline)
+	} else {
+		ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(30*24*time.Hour))
+		wc.rdeadline = time.Time{}
+	}
+	wc.cancelRead = cancel
+	wc.mu.Unlock()
+	defer cancel()
+
+	_, buf, err := wc.c.Read(ctx)
+	n = copy(p, buf)
+	wc.rextra = buf[n:]
+	return n, err
+}
+
+func (wc *websocketConn) Write(p []byte) (n int, err error) {
+	err = wc.c.Write(context.Background(), websocket.MessageBinary, p)
+	if err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
+func (wc *websocketConn) Close() error { return wc.c.Close(websocket.StatusNormalClosure, "close") }
+
+func (wc *websocketConn) SetDeadline(t time.Time) error {
+	wc.SetReadDeadline(t)
+	wc.SetWriteDeadline(t)
+	return nil
+}
+
+func (wc *websocketConn) SetReadDeadline(t time.Time) error {
+	wc.mu.Lock()
+	defer wc.mu.Unlock()
+	if !t.IsZero() && (wc.rdeadline.IsZero() || t.Before(wc.rdeadline)) && wc.cancelRead != nil {
+		wc.cancelRead()
+	}
+	wc.rdeadline = t
+	return nil
+}
+
+func (wc *websocketConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}

disco/disco.go

@@ -25,7 +25,9 @@ import (
 	"fmt"
 	"net"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
+	"tailscale.com/types/key"
 )
 
 // Magic is the 6 byte header of all discovery messages.
@@ -106,12 +108,28 @@ func appendMsgHeader(b []byte, t MessageType, ver uint8, dataLen int) (all, data
 }
 
 type Ping struct {
+	// TxID is a random client-generated per-ping transaction ID.
 	TxID [12]byte
+
+	// NodeKey is allegedly the ping sender's wireguard public key.
+	// Old clients (~1.16.0 and earlier) don't send this field.
+	// It shouldn't be trusted by itself, but can be combined with
+	// netmap data to reduce the discokey:nodekey relation from 1:N to
+	// 1:1.
+	NodeKey key.NodePublic
 }
 
 func (m *Ping) AppendMarshal(b []byte) []byte {
-	ret, d := appendMsgHeader(b, TypePing, v0, 12)
-	copy(d, m.TxID[:])
+	dataLen := 12
+	hasKey := !m.NodeKey.IsZero()
+	if hasKey {
+		dataLen += key.NodePublicRawLen
+	}
+	ret, d := appendMsgHeader(b, TypePing, v0, dataLen)
+	n := copy(d, m.TxID[:])
+	if hasKey {
+		m.NodeKey.AppendTo(d[:n])
+	}
 	return ret
 }
 
@@ -120,7 +138,12 @@ func parsePing(ver uint8, p []byte) (m *Ping, err error) {
 		return nil, errShort
 	}
 	m = new(Ping)
-	copy(m.TxID[:], p)
+	p = p[copy(m.TxID[:], p):]
+	// Deliberately lax on longer-than-expected messages, for future
+	// compatibility.
+	if len(p) >= key.NodePublicRawLen {
+		m.NodeKey = key.NodePublicFromRaw32(mem.B(p[:key.NodePublicRawLen]))
+	}
 	return m, nil
 }
 

disco/disco_test.go

@@ -10,7 +10,9 @@ import (
 	"strings"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
+	"tailscale.com/types/key"
 )
 
 func TestMarshalAndParse(t *testing.T) {
@@ -26,6 +28,14 @@ func TestMarshalAndParse(t *testing.T) {
 			},
 			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c",
 		},
+		{
+			name: "ping_with_nodekey_src",
+			m: &Ping{
+				TxID:    [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				NodeKey: key.NodePublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})),
+			},
+			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f",
+		},
 		{
 			name: "pong",
 			m: &Pong{

docs/k8s/Makefile

@@ -32,3 +32,7 @@ userspace-sidecar:
 proxy:
 	@kubectl delete -f proxy.yaml --ignore-not-found --grace-period=0
 	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" proxy.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | sed -e "s;{{DEST_IP}};$(DEST_IP);g" | kubectl create -f-
+
+subnet-router:
+	@kubectl delete -f subnet.yaml --ignore-not-found --grace-period=0
+	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" subnet.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | sed -e "s;{{ROUTES}};$(ROUTES);g" | kubectl create -f-

docs/k8s/README.md

@@ -27,12 +27,12 @@ There are quite a few ways of running Tailscale inside a Kubernetes Cluster, som
    Configure RBAC to allow the Tailscale pod to read/write the `tailscale` secret.
    ```bash
    export SA_NAME=tailscale
-   export KUBE_SECRET=tailscale
+   export KUBE_SECRET=tailscale-auth
    make rbac
    ```
 
 ### Sample Sidecar
-Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
+Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivity between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
 
 1. Create and login to the sample nginx pod with a Tailscale sidecar
 
@@ -107,4 +107,41 @@ Running a Tailscale proxy allows you to provide inbound connectivity to a Kubern
 
    ```bash
    curl "http://$(tailscale ip -4 proxy)"
-   ```
+   ```
+
+### Subnet Router
+
+Running a Tailscale [subnet router](https://tailscale.com/kb/1019/subnets/) allows you to access
+the entire Kubernetes cluster network (assuming NetworkPolicies allow) over Tailscale.
+
+1. Identify the Pod/Service CIDRs that cover your Kubernetes cluster.  These will vary depending on [which CNI](https://kubernetes.io/docs/concepts/cluster-administration/networking/) you are using and on the Cloud Provider you are using. Add these to the `ROUTES` variable as comma-separated values.
+
+   ```bash
+   SERVICE_CIDR=10.20.0.0/16
+   POD_CIDR=10.42.0.0/15
+   export ROUTES=$SERVICE_CIDR,$POD_CIDR
+   ```
+
+1. Deploy the subnet-router pod.
+
+   ```bash
+   make subnet-router
+   # If not using an auth key, authenticate by grabbing the Login URL here:
+   kubectl logs subnet-router
+   ```
+
+1. In the [Tailscale admin console](https://login.tailscale.com/admin/machines), ensure that the
+routes for the subnet-router are enabled.
+
+1. Make sure that any client you want to connect from has `--accept-routes` enabled.
+
+1. Check if you can connect to a `ClusterIP` or a `PodIP` over Tailscale:
+
+   ```bash
+   # Get the Service IP
+   INTERNAL_IP="$(kubectl get svc <SVC_NAME> -o=jsonpath='{.spec.clusterIP}')"
+   # or, the Pod IP 
+   # INTERNAL_IP="$(kubectl get po <POD_NAME> -o=jsonpath='{.status.podIP}')" 
+   INTERNAL_PORT=8080 
+   curl http://$INTERNAL_IP:$INTERNAL_PORT
+   ```

docs/k8s/run.sh

@@ -53,7 +53,7 @@ tailscale --socket=/tmp/tailscaled.sock up ${UP_ARGS}
 
 if [[ ! -z "${DEST_IP}" ]]; then
   echo "Adding iptables rule for DNAT"
-  iptables -t nat -I PREROUTING -d "$(tailscale ip -4)" -j DNAT --to-destination "${DEST_IP}"
+  iptables -t nat -I PREROUTING -d "$(tailscale --socket=/tmp/tailscaled.sock ip -4)" -j DNAT --to-destination "${DEST_IP}"
 fi
 
-wait ${PID}
+wait ${PID}

docs/k8s/subnet.yaml

@@ -0,0 +1,32 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: subnet-router
+  labels:
+    app: tailscale
+spec:
+  serviceAccountName: "{{SA_NAME}}"
+  containers:
+  - name: tailscale
+    imagePullPolicy: Always
+    image: "{{IMAGE_TAG}}"
+    env:
+    # Store the state in a k8s secret
+    - name: KUBE_SECRET
+      value: "{{KUBE_SECRET}}"
+    - name: USERSPACE
+      value: "true"
+    - name: AUTH_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tailscale-auth
+          key: AUTH_KEY
+          optional: true
+    - name: ROUTES
+      value: "{{ROUTES}}"
+    securityContext:
+      runAsUser: 1000
+      runAsGroup: 1000

go.mod

@@ -4,206 +4,260 @@ go 1.17
 
 require (
 	filippo.io/mkcert v1.4.3
+	github.com/akutz/memconn v0.1.0
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go v1.38.52
-	github.com/aws/aws-sdk-go-v2 v1.9.2
-	github.com/aws/aws-sdk-go-v2/config v1.8.3
-	github.com/aws/aws-sdk-go-v2/service/ssm v1.12.0
+	github.com/aws/aws-sdk-go-v2 v1.11.2
+	github.com/aws/aws-sdk-go-v2/config v1.11.0
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.7.4
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.21.0
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1
 	github.com/coreos/go-iptables v0.6.0
-	github.com/creack/pty v1.1.16
+	github.com/creack/pty v1.1.17
 	github.com/dave/jennifer v1.4.1
-	github.com/frankban/quicktest v1.13.1
+	github.com/frankban/quicktest v1.14.0
 	github.com/gliderlabs/ssh v0.3.3
-	github.com/go-multierror/multierror v1.0.2
-	github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1
-	github.com/godbus/dbus/v5 v5.0.5
+	github.com/go-ole/go-ole v1.2.6
+	github.com/godbus/dbus/v5 v5.0.6
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.3.0
 	github.com/goreleaser/nfpm v1.10.3
 	github.com/iancoleman/strcase v0.2.0
-	github.com/insomniacslk/dhcp v0.0.0-20210621130208-1cac67f12b1e
-	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
+	github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd
+	github.com/jsimonetti/rtnetlink v0.0.0-20211203074127-fd9a11f42291
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.13.6
-	github.com/mdlayher/netlink v1.4.1
+	github.com/mdlayher/netlink v1.4.2
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.43
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/pborman/getopt v1.1.0
-	github.com/peterbourgon/ff/v3 v3.1.0
+	github.com/peterbourgon/ff/v3 v3.1.2
 	github.com/pkg/sftp v1.13.4
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
-	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
+	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
-	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/tailscale/hujson v0.0.0-20211105212140-3a0adc019d83
+	github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
-	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20210927181540-4e4d966f7476
+	github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6
+	go4.org/mem v0.0.0-20210711025021-927187094b94
+	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
+	golang.org/x/net v0.0.0-20211205041911-012df41ee64c
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
-	golang.org/x/tools v0.1.7
-	golang.zx2c4.com/wireguard v0.0.0-20211012062646-82d2aa87aa62
+	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
+	golang.org/x/tools v0.1.8
+	golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45
 	golang.zx2c4.com/wireguard/windows v0.4.10
-	honnef.co/go/tools v0.2.1
-	inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1
-	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
-	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
-	inet.af/wf v0.0.0-20210516214145-a5343001b756
+	honnef.co/go/tools v0.2.2
+	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
+	inet.af/netstack v0.0.0-20211120045802-8aa80cf23d3c
+	inet.af/peercred v0.0.0-20210906144145-0893ea02156a
+	inet.af/wf v0.0.0-20211204062712-86aaea0a7310
+	nhooyr.io/websocket v1.8.7
 )
 
 require (
-	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
-	github.com/BurntSushi/toml v0.3.1 // indirect
+	4d63.com/gochecknoglobals v0.1.0 // indirect
+	github.com/Antonboom/errname v0.1.5 // indirect
+	github.com/Antonboom/nilnil v0.1.0 // indirect
+	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/Djarvur/go-err113 v0.1.0 // indirect
-	github.com/Masterminds/goutils v1.1.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
-	github.com/Microsoft/go-winio v0.4.16 // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 // indirect
-	github.com/aws/smithy-go v1.8.0 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/ashanbrown/forbidigo v1.2.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20210520155254-b6261585ddde // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect
+	github.com/aws/smithy-go v1.9.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
 	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect
-	github.com/bombsimon/wsl/v3 v3.1.0 // indirect
+	github.com/blizzy78/varnamelen v0.5.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.3.0 // indirect
+	github.com/breml/bidichk v0.2.1 // indirect
+	github.com/butuzov/ireturn v0.1.1 // indirect
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e // indirect
-	github.com/daixiang0/gci v0.2.7 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/charithe/durationcheck v0.0.9 // indirect
+	github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect
+	github.com/daixiang0/gci v0.2.9 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/denis-tingajkin/go-header v0.3.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
-	github.com/fatih/color v1.10.0 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-critic/go-critic v0.5.2 // indirect
+	github.com/esimonov/ifshort v1.0.3 // indirect
+	github.com/ettle/strcase v0.1.1 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/go-critic/go-critic v0.6.1 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
-	github.com/go-git/go-billy/v5 v5.0.0 // indirect
-	github.com/go-git/go-git/v5 v5.2.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-toolsmith/astcast v1.0.0 // indirect
 	github.com/go-toolsmith/astcopy v1.0.0 // indirect
-	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.1 // indirect
 	github.com/go-toolsmith/astfmt v1.0.0 // indirect
 	github.com/go-toolsmith/astp v1.0.0 // indirect
 	github.com/go-toolsmith/strparse v1.0.0 // indirect
 	github.com/go-toolsmith/typep v1.0.2 // indirect
-	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
 	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
 	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
-	github.com/golangci/golangci-lint v1.33.0 // indirect
-	github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc // indirect
+	github.com/golangci/golangci-lint v1.43.0 // indirect
 	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
 	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
 	github.com/golangci/misspell v0.3.5 // indirect
-	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 // indirect
-	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/golangci/revgrep v0.0.0-20210930125155-c22e5001d4f2 // indirect
 	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
 	github.com/google/btree v1.0.1 // indirect
-	github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect
+	github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 // indirect
 	github.com/google/rpmpack v0.0.0-20201206194719-59e495f2b7e1 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 // indirect
 	github.com/goreleaser/chglog v0.1.2 // indirect
 	github.com/goreleaser/fileglob v0.3.1 // indirect
-	github.com/gostaticanalysis/analysisutil v0.6.1 // indirect
-	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.2 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 // indirect
-	github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a // indirect
+	github.com/jgautheron/goconst v1.5.1 // indirect
+	github.com/jingyugao/rowserrcheck v1.1.1 // indirect
 	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/julz/importas v0.0.0-20210922140945-27e0a5d4dee2 // indirect
+	github.com/kevinburke/ssh_config v1.1.0 // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
 	github.com/kisielk/gotool v1.0.0 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.3 // indirect
 	github.com/kyoh86/exportloopref v0.1.8 // indirect
-	github.com/magiconair/properties v1.8.4 // indirect
+	github.com/ldez/gomoddirectives v0.2.2 // indirect
+	github.com/ldez/tagliatelle v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/maratori/testpackage v1.0.1 // indirect
-	github.com/matoous/godox v0.0.0-20200801072554-4fb83dc2941e // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mbilski/exhaustivestruct v1.1.0 // indirect
-	github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00 // indirect
-	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb // indirect
+	github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 // indirect
+	github.com/mgechev/revive v1.1.2 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.4.0 // indirect
-	github.com/mitchellh/reflectwalk v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moricho/tparallel v0.2.1 // indirect
-	github.com/nakabonne/nestif v0.3.0 // indirect
-	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
-	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nakabonne/nestif v0.3.1 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
+	github.com/nishanths/exhaustive v0.7.11 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
-	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
-	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
+	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f // indirect
-	github.com/quasilyte/go-ruleguard v0.2.1 // indirect
-	github.com/quasilyte/regex/syntax v0.0.0-20200805063351-8f842688393c // indirect
-	github.com/rogpeppe/go-internal v1.6.2 // indirect
-	github.com/ryancurrah/gomodguard v1.1.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20211125173453-6d6d39c5bb8b // indirect
+	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.13 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect
+	github.com/ryancurrah/gomodguard v1.2.3 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect
 	github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b // indirect
-	github.com/securego/gosec/v2 v2.5.0 // indirect
-	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/securego/gosec/v2 v2.9.3 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
-	github.com/sirupsen/logrus v1.7.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sivchari/tenv v1.4.7 // indirect
 	github.com/sonatard/noctx v0.0.1 // indirect
 	github.com/sourcegraph/go-diff v0.6.1 // indirect
-	github.com/spf13/afero v1.5.1 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.1.1 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cobra v1.2.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/spf13/viper v1.9.0 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
-	github.com/tetafro/godot v1.3.2 // indirect
-	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
-	github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 // indirect
-	github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa // indirect
+	github.com/sylvia7788/contextcheck v1.0.4 // indirect
+	github.com/tdakkota/asciicheck v0.1.1 // indirect
+	github.com/tetafro/godot v1.4.11 // indirect
+	github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 // indirect
+	github.com/tomarrell/wrapcheck/v2 v2.4.0 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.4.0 // indirect
 	github.com/u-root/uio v0.0.0-20210528114334-82958018845c // indirect
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.4 // indirect
-	github.com/uudashr/gocognit v1.0.1 // indirect
-	github.com/xanzy/ssh-agent v0.3.0 // indirect
-	go4.org/intern v0.0.0-20210108033219-3eb7198706b2 // indirect
-	go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	github.com/uudashr/gocognit v1.0.5 // indirect
+	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	github.com/xanzy/ssh-agent v0.3.1 // indirect
+	github.com/yeya24/promlinter v0.1.0 // indirect
+	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
+	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
+	golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
-	mvdan.cc/gofumpt v0.0.0-20201129102820-5c11c50e9475 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	howett.net/plist v1.0.0 // indirect
+	mvdan.cc/gofumpt v0.2.0 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
 	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
-	mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 // indirect
-	software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237 // indirect
+	mvdan.cc/unparam v0.0.0-20211002134041-24922b6997ca // indirect
+	software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 // indirect
 )

health/health.go

@@ -10,13 +10,14 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"runtime"
 	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/multierr"
 )
 
 var (
@@ -57,6 +58,9 @@ const (
 	// SysDNS is the name of the net/dns subsystem.
 	SysDNS = Subsystem("dns")
 
+	// SysDNSOS is the name of the net/dns OSConfigurator subsystem.
+	SysDNSOS = Subsystem("dns-os")
+
 	// SysNetworkCategory is the name of the subsystem that sets
 	// the Windows network adapter's "category" (public, private, domain).
 	// If it's unhealthy, the Windows firewall rules won't match.
@@ -100,6 +104,12 @@ func SetDNSHealth(err error) { set(SysDNS, err) }
 // DNSHealth returns the net/dns.Manager error state.
 func DNSHealth() error { return get(SysDNS) }
 
+// SetDNSOSHealth sets the state of the net/dns.OSConfigurator
+func SetDNSOSHealth(err error) { set(SysDNSOS, err) }
+
+// DNSOSHealth returns the net/dns.OSConfigurator error state.
+func DNSOSHealth() error { return get(SysDNSOS) }
+
 // SetNetworkCategoryHealth sets the state of setting the network adaptor's category.
 // This only applies on Windows.
 func SetNetworkCategoryHealth(err error) { set(SysNetworkCategory, err) }
@@ -267,7 +277,7 @@ func selfCheckLocked() {
 // OverallError returns a summary of the health state.
 //
 // If there are multiple problems, the error will be of type
-// multierror.MultipleErrors.
+// multierr.Error.
 func OverallError() error {
 	mu.Lock()
 	defer mu.Unlock()
@@ -336,7 +346,7 @@ func overallErrorLocked() error {
 		// Not super efficient (stringifying these in a sort), but probably max 2 or 3 items.
 		return errs[i].Error() < errs[j].Error()
 	})
-	return multierror.New(errs)
+	return multierr.New(errs...)
 }
 
 var (
@@ -347,6 +357,12 @@ var (
 	receiveFuncs = []*ReceiveFuncStats{&ReceiveIPv4, &ReceiveIPv6, &ReceiveDERP}
 )
 
+func init() {
+	if runtime.GOOS == "js" {
+		receiveFuncs = receiveFuncs[2:] // ignore IPv4 and IPv6
+	}
+}
+
 // ReceiveFuncStats tracks the calls made to a wireguard-go receive func.
 type ReceiveFuncStats struct {
 	// name is the name of the receive func.

hostinfo/hostinfo.go

@@ -7,11 +7,14 @@
 package hostinfo
 
 import (
+	"bufio"
 	"io"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync/atomic"
+	"time"
 
 	"go4.org/mem"
 	"tailscale.com/tailcfg"
@@ -28,7 +31,7 @@ func New() *tailcfg.Hostinfo {
 		IPNVersion:  version.Long,
 		Hostname:    hostname,
 		OS:          version.OS(),
-		OSVersion:   getOSVersion(),
+		OSVersion:   GetOSVersion(),
 		Package:     packageType(),
 		GoArch:      runtime.GOARCH,
 		DeviceModel: deviceModel(),
@@ -37,7 +40,8 @@ func New() *tailcfg.Hostinfo {
 
 var osVersion func() string // non-nil on some platforms
 
-func getOSVersion() string {
+// GetOSVersion returns the OSVersion of current host if available.
+func GetOSVersion() string {
 	if s, _ := osVersionAtomic.Load().(string); s != "" {
 		return s
 	}
@@ -82,6 +86,8 @@ const (
 	AzureAppService = EnvType("az")
 	AWSFargate      = EnvType("fg")
 	FlyDotIo        = EnvType("fly")
+	Kubernetes      = EnvType("k8s")
+	DockerDesktop   = EnvType("dde")
 )
 
 var envType atomic.Value // of EnvType
@@ -136,6 +142,12 @@ func getEnvType() EnvType {
 	if inFlyDotIo() {
 		return FlyDotIo
 	}
+	if inKubernetes() {
+		return Kubernetes
+	}
+	if inDockerDesktop() {
+		return DockerDesktop
+	}
 	return ""
 }
 
@@ -212,3 +224,69 @@ func inFlyDotIo() bool {
 	}
 	return false
 }
+
+func inKubernetes() bool {
+	if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
+		return true
+	}
+	return false
+}
+
+func inDockerDesktop() bool {
+	if os.Getenv("TS_HOST_ENV") == "dde" {
+		return true
+	}
+	return false
+}
+
+type etcAptSrcResult struct {
+	mod      time.Time
+	disabled bool
+}
+
+var etcAptSrcCache atomic.Value // of etcAptSrcResult
+
+// DisabledEtcAptSource reports whether Ubuntu (or similar) has disabled
+// the /etc/apt/sources.list.d/tailscale.list file contents upon upgrade
+// to a new release of the distro.
+//
+// See https://github.com/tailscale/tailscale/issues/3177
+func DisabledEtcAptSource() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	const path = "/etc/apt/sources.list.d/tailscale.list"
+	fi, err := os.Stat(path)
+	if err != nil || !fi.Mode().IsRegular() {
+		return false
+	}
+	mod := fi.ModTime()
+	if c, ok := etcAptSrcCache.Load().(etcAptSrcResult); ok && c.mod == mod {
+		return c.disabled
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	v := etcAptSourceFileIsDisabled(f)
+	etcAptSrcCache.Store(etcAptSrcResult{mod: mod, disabled: v})
+	return v
+}
+
+func etcAptSourceFileIsDisabled(r io.Reader) bool {
+	bs := bufio.NewScanner(r)
+	disabled := false // did we find the "disabled on upgrade" comment?
+	for bs.Scan() {
+		line := strings.TrimSpace(bs.Text())
+		if strings.Contains(line, "# disabled on upgrade") {
+			disabled = true
+		}
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		// Well, it has some contents in it at least.
+		return false
+	}
+	return disabled
+}

hostinfo/hostinfo_test.go

@@ -6,6 +6,7 @@ package hostinfo
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 )
 
@@ -27,3 +28,25 @@ func TestOSVersion(t *testing.T) {
 	}
 	t.Logf("Got: %#q", osVersion())
 }
+
+func TestEtcAptSourceFileIsDisabled(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want bool
+	}{
+		{"empty", "", false},
+		{"normal", "deb foo\n", false},
+		{"normal-commented", "# deb foo\n", false},
+		{"normal-disabled-by-ubuntu", "# deb foo # disabled on upgrade to dingus\n", true},
+		{"normal-disabled-then-uncommented", "deb foo # disabled on upgrade to dingus\n", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := etcAptSourceFileIsDisabled(strings.NewReader(tt.in))
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

hostinfo/hostinfo_windows.go

@@ -5,10 +5,11 @@
 package hostinfo
 
 import (
-	"os/exec"
-	"strings"
+	"fmt"
 	"sync/atomic"
-	"syscall"
+
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/registry"
 )
 
 func init() {
@@ -21,19 +22,37 @@ func osVersionWindows() string {
 	if s, ok := winVerCache.Load().(string); ok {
 		return s
 	}
-	cmd := exec.Command("cmd", "/c", "ver")
-	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
-	out, _ := cmd.Output() // "\nMicrosoft Windows [Version 10.0.19041.388]\n\n"
-	s := strings.TrimSpace(string(out))
-	s = strings.TrimPrefix(s, "Microsoft Windows [")
-	s = strings.TrimSuffix(s, "]")
-
-	// "Version 10.x.y.z", with "Version" localized. Keep only stuff after the space.
-	if sp := strings.Index(s, " "); sp != -1 {
-		s = s[sp+1:]
+	major, minor, build := windows.RtlGetNtVersionNumbers()
+	s := fmt.Sprintf("%d.%d.%d", major, minor, build)
+	// Windows 11 still uses 10 as its major number internally
+	if major == 10 {
+		if ubr, err := getUBR(); err == nil {
+			s += fmt.Sprintf(".%d", ubr)
+		}
 	}
 	if s != "" {
 		winVerCache.Store(s)
 	}
 	return s // "10.0.19041.388", ideally
 }
+
+// getUBR obtains a fourth version field, the "Update Build Revision",
+// from the registry. This field is only available beginning with Windows 10.
+func getUBR() (uint32, error) {
+	key, err := registry.OpenKey(registry.LOCAL_MACHINE,
+		`SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE|registry.WOW64_64KEY)
+	if err != nil {
+		return 0, err
+	}
+	defer key.Close()
+
+	val, valType, err := key.GetIntegerValue("UBR")
+	if err != nil {
+		return 0, err
+	}
+	if valType != registry.DWORD {
+		return 0, registry.ErrUnexpectedType
+	}
+
+	return uint32(val), nil
+}

ipn/backend.go

@@ -12,6 +12,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/structs"
 )
@@ -48,7 +49,7 @@ type EngineStatus struct {
 	RBytes, WBytes int64
 	NumLive        int
 	LiveDERPs      int // number of active DERP connections
-	LivePeers      map[tailcfg.NodeKey]ipnstate.PeerStatusLite
+	LivePeers      map[key.NodePublic]ipnstate.PeerStatusLite
 }
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend

ipn/ipnlocal/dnsconfig_test.go

@@ -232,32 +232,11 @@ func TestDNSConfigForNetmap(t *testing.T) {
 			},
 		},
 		{
-			name: "android_does_need_fallbacks",
-			os:   "android",
-			nm: &netmap.NetworkMap{
-				DNS: tailcfg.DNSConfig{
-					FallbackResolvers: []dnstype.Resolver{
-						{Addr: "8.8.4.4"},
-					},
-					Routes: map[string][]dnstype.Resolver{
-						"foo.com.": {{Addr: "1.2.3.4"}},
-					},
-				},
-			},
-			prefs: &ipn.Prefs{
-				CorpDNS: true,
-			},
-			want: &dns.Config{
-				Hosts: map[dnsname.FQDN][]netaddr.IP{},
-				DefaultResolvers: []dnstype.Resolver{
-					{Addr: "8.8.4.4:53"},
-				},
-				Routes: map[dnsname.FQDN][]dnstype.Resolver{
-					"foo.com.": {{Addr: "1.2.3.4:53"}},
-				},
-			},
-		},
-		{
+			// Prior to fixing https://github.com/tailscale/tailscale/issues/2116,
+			// Android had cases where it needed FallbackResolvers. This was the
+			// negative test for the case where Override-local-DNS was set, so the
+			// fallback resolvers did not need to be used. This test is still valid
+			// so we keep it, but the fallback test has been removed.
 			name: "android_does_NOT_need_fallbacks",
 			os:   "android",
 			nm: &netmap.NetworkMap{
@@ -344,3 +323,48 @@ func TestDNSConfigForNetmap(t *testing.T) {
 		})
 	}
 }
+
+func TestAllowExitNodeDNSProxyToServeName(t *testing.T) {
+	b := &LocalBackend{}
+	if b.allowExitNodeDNSProxyToServeName("google.com") {
+		t.Fatal("unexpected true on backend with nil NetMap")
+	}
+
+	b.netMap = &netmap.NetworkMap{
+		DNS: tailcfg.DNSConfig{
+			ExitNodeFilteredSet: []string{
+				".ts.net",
+				"some.exact.bad",
+			},
+		},
+	}
+	tests := []struct {
+		name string
+		want bool
+	}{
+		// Allow by default:
+		{"google.com", true},
+		{"GOOGLE.com", true},
+
+		// Rejected by suffix:
+		{"foo.TS.NET", false},
+		{"foo.ts.net", false},
+
+		// Suffix doesn't match
+		{"ts.net", true},
+
+		// Rejected by exact match:
+		{"some.exact.bad", false},
+		{"SOME.EXACT.BAD", false},
+
+		// But a prefix is okay.
+		{"prefix-okay.some.exact.bad", true},
+	}
+	for _, tt := range tests {
+		got := b.allowExitNodeDNSProxyToServeName(tt.name)
+		if got != tt.want {
+			t.Errorf("for %q = %v; want %v", tt.name, got, tt.want)
+		}
+	}
+
+}

ipn/ipnlocal/local.go

@@ -22,10 +22,8 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
-	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
@@ -37,6 +35,7 @@ import (
 	"tailscale.com/net/dns"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/paths"
 	"tailscale.com/portlist"
 	"tailscale.com/tailcfg"
@@ -47,9 +46,9 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
-	"tailscale.com/types/wgkey"
 	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
@@ -89,6 +88,7 @@ type LocalBackend struct {
 	statsLogf             logger.Logf        // for printing peers stats on change
 	e                     wgengine.Engine
 	store                 ipn.StateStore
+	dialer                *tsdial.Dialer // non-nil
 	backendLogID          string
 	unregisterLinkMon     func()
 	unregisterHealthWatch func()
@@ -97,9 +97,12 @@ type LocalBackend struct {
 	gotPortPollRes        chan struct{}    // closed upon first readPoller result
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
+	varRoot               string // or empty if SetVarRoot never called
 
 	filterHash deephash.Sum
 
+	filterAtomic atomic.Value // of *filter.Filter
+
 	// The mutex protects the following elements.
 	mu             sync.Mutex
 	httpTestClient *http.Client // for controlclient. nil by default, used by tests.
@@ -122,6 +125,7 @@ type LocalBackend struct {
 	engineStatus     ipn.EngineStatus
 	endpoints        []tailcfg.Endpoint
 	blocked          bool
+	keyExpired       bool
 	authURL          string // cleared on Notify
 	authURLSticky    string // not cleared on Notify
 	interact         bool
@@ -138,7 +142,11 @@ type LocalBackend struct {
 	// same as the Network Extension lifetime and we can thus avoid
 	// double-copying files by writing them to the right location
 	// immediately.
-	directFileRoot string
+	// It's also used on Synology & TrueNAS, but in that case DoFinalRename
+	// is also set true, which moves the *.partial file to its final
+	// name on completion.
+	directFileRoot          string
+	directFileDoFinalRename bool // false on macOS, true on Synology & TrueNAS
 
 	// statusLock must be held before calling statusChanged.Wait() or
 	// statusChanged.Broadcast().
@@ -152,16 +160,18 @@ type clientGen func(controlclient.Options) (controlclient.Client, error)
 
 // NewLocalBackend returns a new LocalBackend that is ready to run,
 // but is not actually running.
-func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wgengine.Engine) (*LocalBackend, error) {
+//
+// If dialer is nil, a new one is made.
+func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, dialer *tsdial.Dialer, e wgengine.Engine) (*LocalBackend, error) {
 	if e == nil {
-		panic("ipn.NewLocalBackend: wgengine must not be nil")
+		panic("ipn.NewLocalBackend: engine must not be nil")
+	}
+	if dialer == nil {
+		dialer = new(tsdial.Dialer)
 	}
 
 	osshare.SetFileSharingEnabled(false, logf)
 
-	// Default filter blocks everything and logs nothing, until Start() is called.
-	e.SetFilter(filter.NewAllowNone(logf, &netaddr.IPSet{}))
-
 	ctx, cancel := context.WithCancel(context.Background())
 	portpoll, err := portlist.NewPoller()
 	if err != nil {
@@ -176,11 +186,16 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		statsLogf:      logger.LogOnChange(logf, 5*time.Minute, time.Now),
 		e:              e,
 		store:          store,
+		dialer:         dialer,
 		backendLogID:   logid,
 		state:          ipn.NoState,
 		portpoll:       portpoll,
 		gotPortPollRes: make(chan struct{}),
 	}
+
+	// Default filter blocks everything and logs nothing, until Start() is called.
+	b.setFilter(filter.NewAllowNone(logf, &netaddr.IPSet{}))
+
 	b.statusChanged = sync.NewCond(&b.statusLock)
 	b.e.SetStatusCallback(b.setWgengineStatus)
 
@@ -207,6 +222,11 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 	return b, nil
 }
 
+// Dialer returns the backend's dialer.
+func (b *LocalBackend) Dialer() *tsdial.Dialer {
+	return b.dialer
+}
+
 // SetDirectFileRoot sets the directory to download files to directly,
 // without buffering them through an intermediate daemon-owned
 // tailcfg.UserID-specific directory.
@@ -218,6 +238,17 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
+// SetDirectFileDoFinalRename sets whether the peerapi file server should rename
+// a received "name.partial" file to "name" when the download is complete.
+//
+// This only applies when SetDirectFileRoot is non-empty.
+// The default is false.
+func (b *LocalBackend) SetDirectFileDoFinalRename(v bool) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.directFileDoFinalRename = v
+}
+
 // b.mu must be held.
 func (b *LocalBackend) maybePauseControlClientLocked() {
 	if b.cc == nil {
@@ -294,8 +325,8 @@ func (b *LocalBackend) Prefs() *ipn.Prefs {
 	p := b.prefs.Clone()
 	if p != nil && p.Persist != nil {
 		p.Persist.LegacyFrontendPrivateMachineKey = key.MachinePrivate{}
-		p.Persist.PrivateNodeKey = wgkey.Private{}
-		p.Persist.OldPrivateNodeKey = wgkey.Private{}
+		p.Persist.PrivateNodeKey = key.NodePrivate{}
+		p.Persist.OldPrivateNodeKey = key.NodePrivate{}
 	}
 	return p
 }
@@ -335,8 +366,8 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 
 		if err := health.OverallError(); err != nil {
 			switch e := err.(type) {
-			case multierror.MultipleErrors:
-				for _, err := range e {
+			case multierr.Error:
+				for _, err := range e.Errors() {
 					s.Health = append(s.Health, err.Error())
 				}
 			default:
@@ -349,8 +380,18 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		}
 	})
 	sb.MutateSelfStatus(func(ss *ipnstate.PeerStatus) {
-		if b.netMap != nil && b.netMap.SelfNode != nil {
-			ss.ID = b.netMap.SelfNode.StableID
+		if b.netMap != nil {
+			ss.HostName = b.netMap.Hostinfo.Hostname
+			ss.DNSName = b.netMap.Name
+			ss.UserID = b.netMap.User
+			if sn := b.netMap.SelfNode; sn != nil {
+				ss.ID = sn.StableID
+				if c := sn.Capabilities; len(c) > 0 {
+					ss.Capabilities = append([]string(nil), c...)
+				}
+			}
+		} else {
+			ss.HostName, _ = os.Hostname()
 		}
 		for _, pln := range b.peerAPIListeners {
 			ss.PeerAPIURL = append(ss.PeerAPIURL, pln.urlStr)
@@ -376,33 +417,30 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 		if p.LastSeen != nil {
 			lastSeen = *p.LastSeen
 		}
-		var tailAddr4 string
 		var tailscaleIPs = make([]netaddr.IP, 0, len(p.Addresses))
 		for _, addr := range p.Addresses {
 			if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.IP()) {
-				if addr.IP().Is4() && tailAddr4 == "" {
-					// The peer struct previously only allowed a single
-					// Tailscale IP address. For compatibility for a few releases starting
-					// with 1.8, keep it pulled out as IPv4-only for a bit.
-					tailAddr4 = addr.IP().String()
-				}
 				tailscaleIPs = append(tailscaleIPs, addr.IP())
 			}
 		}
-		sb.AddPeer(key.Public(p.Key), &ipnstate.PeerStatus{
-			InNetworkMap:       true,
-			ID:                 p.StableID,
-			UserID:             p.User,
-			TailAddrDeprecated: tailAddr4,
-			TailscaleIPs:       tailscaleIPs,
-			HostName:           p.Hostinfo.Hostname,
-			DNSName:            p.Name,
-			OS:                 p.Hostinfo.OS,
-			KeepAlive:          p.KeepAlive,
-			Created:            p.Created,
-			LastSeen:           lastSeen,
-			ShareeNode:         p.Hostinfo.ShareeNode,
-			ExitNode:           p.StableID != "" && p.StableID == b.prefs.ExitNodeID,
+		exitNodeOption := tsaddr.PrefixesContainsFunc(p.AllowedIPs, func(r netaddr.IPPrefix) bool {
+			return r.Bits() == 0
+		})
+		sb.AddPeer(p.Key, &ipnstate.PeerStatus{
+			InNetworkMap:   true,
+			ID:             p.StableID,
+			UserID:         p.User,
+			TailscaleIPs:   tailscaleIPs,
+			HostName:       p.Hostinfo.Hostname,
+			DNSName:        p.Name,
+			OS:             p.Hostinfo.OS,
+			KeepAlive:      p.KeepAlive,
+			Created:        p.Created,
+			LastSeen:       lastSeen,
+			Online:         p.Online != nil && *p.Online,
+			ShareeNode:     p.Hostinfo.ShareeNode,
+			ExitNode:       p.StableID != "" && p.StableID == b.prefs.ExitNodeID,
+			ExitNodeOption: exitNodeOption,
 		})
 	}
 }
@@ -448,20 +486,39 @@ func (b *LocalBackend) SetDecompressor(fn func() (controlclient.Decompressor, er
 // Among other things, this is where we update the netmap, packet filters, DNS and DERP maps.
 func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 	// The following do not depend on any data for which we need to lock b.
-	if st.Err != "" {
+	if st.Err != nil {
 		// TODO(crawshaw): display in the UI.
-		if st.Err == "EOF" {
+		if errors.Is(st.Err, io.EOF) {
 			b.logf("[v1] Received error: EOF")
-		} else {
-			b.logf("Received error: %v", st.Err)
+			return
+		}
+		b.logf("Received error: %v", st.Err)
+		var uerr controlclient.UserVisibleError
+		if errors.As(st.Err, &uerr) {
+			s := uerr.UserVisibleError()
+			b.send(ipn.Notify{ErrMessage: &s})
 		}
 		return
 	}
 
 	b.mu.Lock()
 	wasBlocked := b.blocked
+	keyExpiryExtended := false
+	if st.NetMap != nil {
+		wasExpired := b.keyExpired
+		isExpired := !st.NetMap.Expiry.IsZero() && st.NetMap.Expiry.Before(time.Now())
+		if wasExpired && !isExpired {
+			keyExpiryExtended = true
+		}
+		b.keyExpired = isExpired
+	}
 	b.mu.Unlock()
 
+	if keyExpiryExtended && wasBlocked {
+		// Key extended, unblock the engine
+		b.blockEngineUpdates(false)
+	}
+
 	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
@@ -568,6 +625,11 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 // findExitNodeIDLocked updates b.prefs to reference an exit node by ID,
 // rather than by IP. It returns whether prefs was mutated.
 func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged bool) {
+	if nm == nil {
+		// No netmap, can't resolve anything.
+		return false
+	}
+
 	// If we have a desired IP on file, try to find the corresponding
 	// node.
 	if b.prefs.ExitNodeIP.IsZero() {
@@ -845,7 +907,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		})
 	}
 
-	var discoPublic tailcfg.DiscoKey
+	var discoPublic key.DiscoPublic
 	if controlclient.Debug.Disco {
 		discoPublic = b.e.DiscoPublicKey()
 	}
@@ -981,20 +1043,25 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 
 	if !haveNetmap {
 		b.logf("netmap packet filter: (not ready yet)")
-		b.e.SetFilter(filter.NewAllowNone(b.logf, logNets))
+		b.setFilter(filter.NewAllowNone(b.logf, logNets))
 		return
 	}
 
 	oldFilter := b.e.GetFilter()
 	if shieldsUp {
 		b.logf("netmap packet filter: (shields up)")
-		b.e.SetFilter(filter.NewShieldsUpFilter(localNets, logNets, oldFilter, b.logf))
+		b.setFilter(filter.NewShieldsUpFilter(localNets, logNets, oldFilter, b.logf))
 	} else {
 		b.logf("netmap packet filter: %v filters", len(packetFilter))
-		b.e.SetFilter(filter.New(packetFilter, localNets, logNets, oldFilter, b.logf))
+		b.setFilter(filter.New(packetFilter, localNets, logNets, oldFilter, b.logf))
 	}
 }
 
+func (b *LocalBackend) setFilter(f *filter.Filter) {
+	b.filterAtomic.Store(f)
+	b.e.SetFilter(f)
+}
+
 var removeFromDefaultRoute = []netaddr.IPPrefix{
 	// RFC1918 LAN ranges
 	netaddr.MustParseIPPrefix("192.168.0.0/16"),
@@ -1022,14 +1089,29 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 // Given that "internal" routes don't leave the device, we choose to
 // trust them more, allowing access to them when an Exit Node is enabled.
 func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
-	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
+	il, err := interfaces.GetList()
+	if err != nil {
+		return nil, nil, err
+	}
+	return internalAndExternalInterfacesFrom(il, runtime.GOOS)
+}
+
+func internalAndExternalInterfacesFrom(il interfaces.List, goos string) (internal, external []netaddr.IPPrefix, err error) {
+	// We use an IPSetBuilder here to canonicalize the prefixes
+	// and to remove any duplicate entries.
+	var internalBuilder, externalBuilder netaddr.IPSetBuilder
+	if err := il.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
 		if tsaddr.IsTailscaleIP(pfx.IP()) {
 			return
 		}
 		if pfx.IsSingleIP() {
 			return
 		}
-		if runtime.GOOS == "windows" {
+		if iface.IsLoopback() {
+			internalBuilder.AddPrefix(pfx)
+			return
+		}
+		if goos == "windows" {
 			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
 			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
 			//
@@ -1040,16 +1122,24 @@ func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err
 			// configuration breaks WSL2 DNS without this.
 			mac := iface.Interface.HardwareAddr
 			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
-				internal = append(internal, pfx)
+				internalBuilder.AddPrefix(pfx)
 				return
 			}
 		}
-		external = append(external, pfx)
+		externalBuilder.AddPrefix(pfx)
 	}); err != nil {
 		return nil, nil, err
 	}
+	iSet, err := internalBuilder.IPSet()
+	if err != nil {
+		return nil, nil, err
+	}
+	eSet, err := externalBuilder.IPSet()
+	if err != nil {
+		return nil, nil, err
+	}
 
-	return internal, external, nil
+	return iSet.Prefixes(), eSet.Prefixes(), nil
 }
 
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
@@ -1199,7 +1289,7 @@ func (b *LocalBackend) send(n ipn.Notify) {
 		return
 	}
 
-	if apiSrv != nil && apiSrv.hasFilesWaiting() {
+	if apiSrv.hasFilesWaiting() {
 		n.FilesWaiting = &empty.Message{}
 	}
 
@@ -1483,7 +1573,15 @@ func (b *LocalBackend) StartLoginInteractive() {
 	if url != "" {
 		b.popBrowserAuthNow()
 	} else {
-		cc.Login(nil, controlclient.LoginInteractive)
+		flags := controlclient.LoginInteractive
+		if runtime.GOOS == "js" {
+			// The js/wasm client has no state storage so for now
+			// treat all interactive logins as ephemeral.
+			// TODO(bradfitz): if we start using browser LocalStorage
+			// or something, then rethink this.
+			flags |= controlclient.LoginEphemeral
+		}
+		cc.Login(nil, flags)
 	}
 }
 
@@ -1529,7 +1627,7 @@ func (b *LocalBackend) parseWgStatusLocked(s *wgengine.Status) (ret ipn.EngineSt
 	var peerStats, peerKeys strings.Builder
 
 	ret.LiveDERPs = s.DERPs
-	ret.LivePeers = map[tailcfg.NodeKey]ipnstate.PeerStatusLite{}
+	ret.LivePeers = map[key.NodePublic]ipnstate.PeerStatusLite{}
 	for _, p := range s.Peers {
 		if !p.LastHandshake.IsZero() {
 			fmt.Fprintf(&peerStats, "%d/%d ", p.RxBytes, p.TxBytes)
@@ -1600,7 +1698,7 @@ func (b *LocalBackend) SetPrefs(newp *ipn.Prefs) {
 }
 
 // setPrefsLockedOnEntry requires b.mu be held to call it, but it
-// unlocks b.mu when done.
+// unlocks b.mu when done. newp ownership passes to this function.
 func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 	netMap := b.netMap
 	stateKey := b.stateKey
@@ -1608,6 +1706,10 @@ func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 	oldp := b.prefs
 	newp.Persist = oldp.Persist // caller isn't allowed to override this
 	b.prefs = newp
+	// findExitNodeIDLocked returns whether it updated b.prefs, but
+	// everything in this function treats b.prefs as completely new
+	// anyway. No-op if no exit node resolution is needed.
+	b.findExitNodeIDLocked(netMap)
 	b.inServerMode = newp.ForceDaemon
 	// We do this to avoid holding the lock while doing everything else.
 	newp = b.prefs.Clone()
@@ -1643,7 +1745,7 @@ func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 				// notified (to update its prefs/persist) on
 				// account switch.  Log this while we figure it
 				// out.
-				b.logf("active login: %s ([unexpected] corp#461, not %s)", newp.Persist.LoginName)
+				b.logf("active login: %q ([unexpected] corp#461, not %q)", newp.Persist.LoginName, login)
 			}
 		}
 	}
@@ -1685,15 +1787,24 @@ func (b *LocalBackend) getPeerAPIPortForTSMPPing(ip netaddr.IP) (port uint16, ok
 
 func (b *LocalBackend) peerAPIServicesLocked() (ret []tailcfg.Service) {
 	for _, pln := range b.peerAPIListeners {
-		proto := tailcfg.ServiceProto("peerapi4")
+		proto := tailcfg.PeerAPI4
 		if pln.ip.Is6() {
-			proto = "peerapi6"
+			proto = tailcfg.PeerAPI6
 		}
 		ret = append(ret, tailcfg.Service{
 			Proto: proto,
 			Port:  uint16(pln.port),
 		})
 	}
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd", "illumos", "darwin", "windows":
+		// These are the platforms currently supported by
+		// net/dns/resolver/tsdns.go:Resolver.HandleExitNodeDNSQuery.
+		ret = append(ret, tailcfg.Service{
+			Proto: tailcfg.PeerAPIDNS,
+			Port:  1, // version
+		})
+	}
 	return ret
 }
 
@@ -1738,6 +1849,12 @@ func (b *LocalBackend) NetMap() *netmap.NetworkMap {
 	return b.netMap
 }
 
+func (b *LocalBackend) isEngineBlocked() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.blocked
+}
+
 // blockEngineUpdate sets b.blocked to block, while holding b.mu. Its
 // indirect effect is to turn b.authReconfig() into a no-op if block
 // is true.
@@ -1788,6 +1905,15 @@ func (b *LocalBackend) authReconfig() {
 		}
 	}
 
+	// Keep the dialer updated about whether we're supposed to use
+	// an exit node's DNS server (so SOCKS5/HTTP outgoing dials
+	// can use it for name resolution)
+	if dohURL, ok := exitNodeCanProxyDNS(nm, prefs.ExitNodeID); ok {
+		b.dialer.SetExitDNSDoH(dohURL)
+	} else {
+		b.dialer.SetExitDNSDoH("")
+	}
+
 	cfg, err := nmcfg.WGCfg(nm, b.logf, flags, prefs.ExitNodeID)
 	if err != nil {
 		b.logf("wgcfg: %v", err)
@@ -1886,12 +2012,32 @@ func dnsConfigForNetmap(nm *netmap.NetworkMap, prefs *ipn.Prefs, logf logger.Log
 		return dcfg
 	}
 
+	for _, dom := range nm.DNS.Domains {
+		fqdn, err := dnsname.ToFQDN(dom)
+		if err != nil {
+			logf("[unexpected] non-FQDN search domain %q", dom)
+		}
+		dcfg.SearchDomains = append(dcfg.SearchDomains, fqdn)
+	}
+	if nm.DNS.Proxied { // actually means "enable MagicDNS"
+		for _, dom := range magicDNSRootDomains(nm) {
+			dcfg.Routes[dom] = nil // resolve internally with dcfg.Hosts
+		}
+	}
+
 	addDefault := func(resolvers []dnstype.Resolver) {
 		for _, r := range resolvers {
 			dcfg.DefaultResolvers = append(dcfg.DefaultResolvers, normalizeResolver(r))
 		}
 	}
 
+	// If we're using an exit node and that exit node is new enough (1.19.x+)
+	// to run a DoH DNS proxy, then send all our DNS traffic through it.
+	if dohURL, ok := exitNodeCanProxyDNS(nm, prefs.ExitNodeID); ok {
+		addDefault([]dnstype.Resolver{{Addr: dohURL}})
+		return dcfg
+	}
+
 	addDefault(nm.DNS.Resolvers)
 	for suffix, resolvers := range nm.DNS.Routes {
 		fqdn, err := dnsname.ToFQDN(suffix)
@@ -1913,18 +2059,6 @@ func dnsConfigForNetmap(nm *netmap.NetworkMap, prefs *ipn.Prefs, logf logger.Log
 			dcfg.Routes[fqdn] = append(dcfg.Routes[fqdn], normalizeResolver(r))
 		}
 	}
-	for _, dom := range nm.DNS.Domains {
-		fqdn, err := dnsname.ToFQDN(dom)
-		if err != nil {
-			logf("[unexpected] non-FQDN search domain %q", dom)
-		}
-		dcfg.SearchDomains = append(dcfg.SearchDomains, fqdn)
-	}
-	if nm.DNS.Proxied { // actually means "enable MagicDNS"
-		for _, dom := range magicDNSRootDomains(nm) {
-			dcfg.Routes[dom] = nil // resolve internally with dcfg.Hosts
-		}
-	}
 
 	// Set FallbackResolvers as the default resolvers in the
 	// scenarios that can't handle a purely split-DNS config. See
@@ -1948,9 +2082,6 @@ func dnsConfigForNetmap(nm *netmap.NetworkMap, prefs *ipn.Prefs, logf logger.Log
 		addDefault(nm.DNS.FallbackResolvers)
 	case len(dcfg.Routes) == 0:
 		// No settings requiring split DNS, no problem.
-	case versionOS == "android":
-		// We don't support split DNS at all on Android yet.
-		addDefault(nm.DNS.FallbackResolvers)
 	}
 
 	return dcfg
@@ -1966,34 +2097,29 @@ func normalizeResolver(cfg dnstype.Resolver) dnstype.Resolver {
 	return cfg
 }
 
+// SetVarRoot sets the root directory of Tailscale's writable
+// storage area . (e.g. "/var/lib/tailscale")
+//
+// It should only be called before the LocalBackend is used.
+func (b *LocalBackend) SetVarRoot(dir string) {
+	b.varRoot = dir
+}
+
 // TailscaleVarRoot returns the root directory of Tailscale's writable
 // storage area. (e.g. "/var/lib/tailscale")
 //
 // It returns an empty string if there's no configured or discovered
 // location.
 func (b *LocalBackend) TailscaleVarRoot() string {
+	if b.varRoot != "" {
+		return b.varRoot
+	}
 	switch runtime.GOOS {
 	case "ios", "android":
 		dir, _ := paths.AppSharedDir.Load().(string)
 		return dir
 	}
-	// Temporary (2021-09-27) transitional fix for #2927 (Synology
-	// cert dir) on the way towards a more complete fix
-	// (#2932). It fixes any case where the state file is provided
-	// to tailscaled explicitly when it's not in the default
-	// location.
-	if fs, ok := b.store.(*ipn.FileStore); ok {
-		if fp := fs.Path(); fp != "" {
-			if dir := filepath.Dir(fp); strings.EqualFold(filepath.Base(dir), "tailscale") {
-				return dir
-			}
-		}
-	}
-	stateFile := paths.DefaultTailscaledStateFile()
-	if stateFile == "" {
-		return ""
-	}
-	return filepath.Dir(stateFile)
+	return ""
 }
 
 func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
@@ -2002,7 +2128,7 @@ func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
 	}
 	varRoot := b.TailscaleVarRoot()
 	if varRoot == "" {
-		b.logf("peerapi disabled; no state directory")
+		b.logf("Taildrop disabled; no state directory")
 		return ""
 	}
 	baseDir := fmt.Sprintf("%s-uid-%d",
@@ -2010,7 +2136,7 @@ func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
 		uid)
 	dir := filepath.Join(varRoot, "files", baseDir)
 	if err := os.MkdirAll(dir, 0700); err != nil {
-		b.logf("peerapi disabled; error making directory: %v", err)
+		b.logf("Taildrop disabled; error making directory: %v", err)
 		return ""
 	}
 	return dir
@@ -2073,22 +2199,20 @@ func (b *LocalBackend) initPeerAPIListener() {
 
 	fileRoot := b.fileRootLocked(selfNode.User)
 	if fileRoot == "" {
-		return
-	}
-
-	var tunName string
-	if ge, ok := b.e.(wgengine.InternalsGetter); ok {
-		if tunWrap, _, ok := ge.GetInternals(); ok {
-			tunName, _ = tunWrap.Name()
-		}
+		b.logf("peerapi starting without Taildrop directory configured")
 	}
 
 	ps := &peerAPIServer{
-		b:              b,
-		rootDir:        fileRoot,
-		tunName:        tunName,
-		selfNode:       selfNode,
-		directFileMode: b.directFileRoot != "",
+		b:                       b,
+		rootDir:                 fileRoot,
+		selfNode:                selfNode,
+		directFileMode:          b.directFileRoot != "",
+		directFileDoFinalRename: b.directFileDoFinalRename,
+	}
+	if re, ok := b.e.(wgengine.ResolvingEngine); ok {
+		if r, ok := re.GetResolver(); ok {
+			ps.resolver = r
+		}
 	}
 	b.peerAPIServer = ps
 
@@ -2365,6 +2489,7 @@ func (b *LocalBackend) nextState() ipn.State {
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 		st          = b.engineStatus
+		keyExpired  = b.keyExpired
 	)
 	b.mu.Unlock()
 
@@ -2397,7 +2522,9 @@ func (b *LocalBackend) nextState() ipn.State {
 		}
 	case !wantRunning:
 		return ipn.Stopped
-	case !netMap.Expiry.IsZero() && time.Until(netMap.Expiry) <= 0:
+	case keyExpired:
+		// NetMap must be non-nil for us to get here.
+		// The node key expired, need to relogin.
 		return ipn.NeedsLogin
 	case netMap.MachineStatus != tailcfg.MachineAuthorized:
 		// TODO(crawshaw): handle tailcfg.MachineInvalid
@@ -2478,6 +2605,7 @@ func (b *LocalBackend) ResetForClientDisconnect() {
 	b.userID = ""
 	b.setNetMapLocked(nil)
 	b.prefs = new(ipn.Prefs)
+	b.keyExpired = false
 	b.authURL = ""
 	b.authURLSticky = ""
 	b.activeLogin = ""
@@ -2562,6 +2690,7 @@ func hasCapability(nm *netmap.NetworkMap, cap string) bool {
 }
 
 func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
+	b.dialer.SetNetMap(nm)
 	var login string
 	if nm != nil {
 		login = nm.UserProfiles[nm.User].LoginName
@@ -2647,7 +2776,7 @@ func (b *LocalBackend) OperatorUserID() string {
 // TestOnlyPublicKeys returns the current machine and node public
 // keys. Used in tests only to facilitate automated node authorization
 // in the test harness.
-func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey tailcfg.NodeKey) {
+func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey key.NodePublic) {
 	b.mu.Lock()
 	prefs := b.prefs
 	machinePrivKey := b.machinePrivKey
@@ -2659,16 +2788,13 @@ func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeK
 
 	mk := machinePrivKey.Public()
 	nk := prefs.Persist.PrivateNodeKey.Public()
-	return mk, tailcfg.NodeKey(nk)
+	return mk, nk
 }
 
 func (b *LocalBackend) WaitingFiles() ([]apitype.WaitingFile, error) {
 	b.mu.Lock()
 	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
-	if apiSrv == nil {
-		return nil, errors.New("peerapi disabled")
-	}
 	return apiSrv.WaitingFiles()
 }
 
@@ -2676,9 +2802,6 @@ func (b *LocalBackend) DeleteFile(name string) error {
 	b.mu.Lock()
 	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
-	if apiSrv == nil {
-		return errors.New("peerapi disabled")
-	}
 	return apiSrv.DeleteFile(name)
 }
 
@@ -2686,9 +2809,6 @@ func (b *LocalBackend) OpenFile(name string) (rc io.ReadCloser, size int64, err
 	b.mu.Lock()
 	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
-	if apiSrv == nil {
-		return nil, 0, errors.New("peerapi disabled")
-	}
 	return apiSrv.OpenFile(name)
 }
 
@@ -2749,7 +2869,7 @@ func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
 	b.mu.Lock()
 	cc := b.cc
 	if prefs := b.prefs; prefs != nil {
-		req.NodeKey = tailcfg.NodeKey(prefs.Persist.PrivateNodeKey.Public())
+		req.NodeKey = prefs.Persist.PrivateNodeKey.Public()
 	}
 	b.mu.Unlock()
 	if cc == nil {
@@ -2802,9 +2922,9 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 	var p4, p6 uint16
 	for _, s := range peer.Hostinfo.Services {
 		switch s.Proto {
-		case "peerapi4":
+		case tailcfg.PeerAPI4:
 			p4 = s.Port
-		case "peerapi6":
+		case tailcfg.PeerAPI6:
 			p6 = s.Port
 		}
 	}
@@ -2838,48 +2958,97 @@ func (b *LocalBackend) CheckIPForwarding() error {
 	if wgengine.IsNetstackRouter(b.e) {
 		return nil
 	}
-	if isBSD(runtime.GOOS) {
+
+	switch {
+	case isBSD(runtime.GOOS):
 		return fmt.Errorf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS)
+	case runtime.GOOS == "linux":
+		return checkIPForwardingLinux()
+	default:
+		// TODO: subnet routing and exit nodes probably don't work
+		// correctly on non-linux, non-netstack OSes either. Warn
+		// instead of being silent?
+		return nil
+	}
+}
+
+// checkIPForwardingLinux checks if IP forwarding is enabled correctly
+// for subnet routing and exit node functionality. Returns an error
+// describing configuration issues if the configuration is not
+// definitely good.
+func checkIPForwardingLinux() error {
+	const kbLink = "\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
+
+	disabled, err := disabledSysctls("net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding")
+	if err != nil {
+		return fmt.Errorf("Couldn't check system's IP forwarding configuration, subnet routing/exit nodes may not work: %w%s", err, kbLink)
 	}
 
-	var keys []string
-
-	if runtime.GOOS == "linux" {
-		keys = append(keys, "net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding")
-	} else if isBSD(runtime.GOOS) {
-		keys = append(keys, "net.inet.ip.forwarding")
-	} else {
+	if len(disabled) == 0 {
+		// IP forwarding is enabled systemwide, all is well.
 		return nil
 	}
 
-	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
-	for _, key := range keys {
-		bs, err := exec.Command("sysctl", "-n", key).Output()
-		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
+	// IP forwarding isn't enabled globally, but it might be enabled
+	// on a per-interface basis. Check if it's on for all interfaces,
+	// and warn appropriately if it's not.
+	ifaces, err := interfaces.GetList()
+	if err != nil {
+		return fmt.Errorf("Couldn't enumerate network interfaces, subnet routing/exit nodes may not work: %w%s", err, kbLink)
+	}
+
+	var (
+		warnings   []string
+		anyEnabled bool
+	)
+	for _, iface := range ifaces {
+		if iface.Name == "lo" {
+			continue
 		}
-		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+		disabled, err = disabledSysctls(fmt.Sprintf("net.ipv4.conf.%s.forwarding", iface.Name), fmt.Sprintf("net.ipv6.conf.%s.forwarding", iface.Name))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
+			return fmt.Errorf("Couldn't check system's IP forwarding configuration, subnet routing/exit nodes may not work: %w%s", err, kbLink)
 		}
-		if !on {
-			return fmt.Errorf("%s is disabled.%s", key, suffix)
+		if len(disabled) > 0 {
+			warnings = append(warnings, fmt.Sprintf("Traffic received on %s won't be forwarded (%s disabled)", iface.Name, strings.Join(disabled, ", ")))
+		} else {
+			anyEnabled = true
 		}
 	}
+	if !anyEnabled {
+		// IP forwarding is compeltely disabled, just say that rather
+		// than enumerate all the interfaces on the system.
+		return fmt.Errorf("IP forwarding is disabled, subnet routing/exit nodes will not work.%s", kbLink)
+	}
+	if len(warnings) > 0 {
+		// If partially enabled, enumerate the bits that won't work.
+		return fmt.Errorf("%s\nSubnet routes and exit nodes may not work correctly.%s", strings.Join(warnings, "\n"), kbLink)
+	}
+
 	return nil
 }
 
-// peerDialControlFunc is non-nil on platforms that require a way to
-// bind to dial out to other peers.
-var peerDialControlFunc func(*LocalBackend) func(network, address string, c syscall.RawConn) error
-
-// PeerDialControlFunc returns a net.Dialer.Control func (possibly nil) to use to
-// dial other Tailscale peers from the current environment.
-func (b *LocalBackend) PeerDialControlFunc() func(network, address string, c syscall.RawConn) error {
-	if peerDialControlFunc != nil {
-		return peerDialControlFunc(b)
+// disabledSysctls checks if the given sysctl keys are off, according
+// to strconv.ParseBool. Returns a list of keys that are disabled, or
+// err if something went wrong which prevented the lookups from
+// completing.
+func disabledSysctls(sysctls ...string) (disabled []string, err error) {
+	for _, k := range sysctls {
+		// TODO: on linux, we can get at these values via /proc/sys,
+		// rather than fork subcommands that may not be installed.
+		bs, err := exec.Command("sysctl", "-n", k).Output()
+		if err != nil {
+			return nil, fmt.Errorf("couldn't check %s (%v)", k, err)
+		}
+		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+		if err != nil {
+			return nil, fmt.Errorf("couldn't parse %s (%v)", k, err)
+		}
+		if !on {
+			disabled = append(disabled, k)
+		}
 	}
-	return nil
+	return disabled, nil
 }
 
 // DERPMap returns the current DERPMap in use, or nil if not connected.
@@ -2891,3 +3060,77 @@ func (b *LocalBackend) DERPMap() *tailcfg.DERPMap {
 	}
 	return b.netMap.DERPMap
 }
+
+// OfferingExitNode reports whether b is currently offering exit node
+// access.
+func (b *LocalBackend) OfferingExitNode() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.prefs == nil {
+		return false
+	}
+	var def4, def6 bool
+	for _, r := range b.prefs.AdvertiseRoutes {
+		if r.Bits() != 0 {
+			continue
+		}
+		if r.IP().Is4() {
+			def4 = true
+		} else if r.IP().Is6() {
+			def6 = true
+		}
+	}
+	return def4 && def6
+}
+
+// allowExitNodeDNSProxyToServeName reports whether the Exit Node DNS
+// proxy is allowed to serve responses for the provided DNS name.
+func (b *LocalBackend) allowExitNodeDNSProxyToServeName(name string) bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	nm := b.netMap
+	if nm == nil {
+		return false
+	}
+	name = strings.ToLower(name)
+	for _, bad := range nm.DNS.ExitNodeFilteredSet {
+		if bad == "" {
+			// Invalid, ignore.
+			continue
+		}
+		if bad[0] == '.' {
+			// Entries beginning with a dot are suffix matches.
+			if dnsname.HasSuffix(name, bad) {
+				return false
+			}
+			continue
+		}
+		// Otherwise entries are exact matches. They're
+		// guaranteed to be lowercase already.
+		if name == bad {
+			return false
+		}
+	}
+	return true
+}
+
+// exitNodeCanProxyDNS reports the DoH base URL ("http://foo/dns-query") without query parameters
+// to exitNodeID's DoH service, if available.
+//
+// If exitNodeID is the zero valid, it returns "", false.
+func exitNodeCanProxyDNS(nm *netmap.NetworkMap, exitNodeID tailcfg.StableNodeID) (dohURL string, ok bool) {
+	if exitNodeID.IsZero() {
+		return "", false
+	}
+	for _, p := range nm.Peers {
+		if p.StableID != exitNodeID {
+			continue
+		}
+		for _, s := range p.Hostinfo.Services {
+			if s.Proto == tailcfg.PeerAPIDNS && s.Port >= 1 {
+				return peerAPIBase(nm, p) + "/dns-query", true
+			}
+		}
+	}
+	return "", false
+}

ipn/ipnlocal/local_test.go

@@ -6,6 +6,7 @@ package ipnlocal
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"reflect"
 	"testing"
@@ -91,14 +92,14 @@ func TestNetworkMapCompare(t *testing.T) {
 		},
 		{
 			"Node names identical",
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "A"}}},
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "A"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "A"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "A"}}},
 			true,
 		},
 		{
 			"Node names differ",
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "A"}}},
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "B"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "A"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "B"}}},
 			false,
 		},
 		{
@@ -116,8 +117,8 @@ func TestNetworkMapCompare(t *testing.T) {
 		{
 			"Node Users differ",
 			// User field is not checked.
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{User: 0}}},
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{User: 1}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{User: 0}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{User: 1}}},
 			true,
 		},
 	}
@@ -444,7 +445,7 @@ func TestLazyMachineKeyGeneration(t *testing.T) {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}
 	t.Cleanup(eng.Close)
-	lb, err := NewLocalBackend(logf, "logid", store, eng)
+	lb, err := NewLocalBackend(logf, "logid", store, nil, eng)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
@@ -494,3 +495,103 @@ func TestFileTargets(t *testing.T) {
 	}
 	// (other cases handled by TestPeerAPIBase above)
 }
+
+func TestInternalAndExternalInterfaces(t *testing.T) {
+	type interfacePrefix struct {
+		i   interfaces.Interface
+		pfx netaddr.IPPrefix
+	}
+
+	masked := func(ips ...interfacePrefix) (pfxs []netaddr.IPPrefix) {
+		for _, ip := range ips {
+			pfxs = append(pfxs, ip.pfx.Masked())
+		}
+		return pfxs
+	}
+	iList := func(ips ...interfacePrefix) (il interfaces.List) {
+		for _, ip := range ips {
+			il = append(il, ip.i)
+		}
+		return il
+	}
+	newInterface := func(name, pfx string, wsl2, loopback bool) interfacePrefix {
+		ippfx := netaddr.MustParseIPPrefix(pfx)
+		ip := interfaces.Interface{
+			Interface: &net.Interface{},
+			AltAddrs: []net.Addr{
+				ippfx.IPNet(),
+			},
+		}
+		if loopback {
+			ip.Flags = net.FlagLoopback
+		}
+		if wsl2 {
+			ip.HardwareAddr = []byte{0x00, 0x15, 0x5d, 0x00, 0x00, 0x00}
+		}
+		return interfacePrefix{i: ip, pfx: ippfx}
+	}
+	var (
+		en0      = newInterface("en0", "10.20.2.5/16", false, false)
+		en1      = newInterface("en1", "192.168.1.237/24", false, false)
+		wsl      = newInterface("wsl", "192.168.5.34/24", true, false)
+		loopback = newInterface("lo0", "127.0.0.1/8", false, true)
+	)
+
+	tests := []struct {
+		name    string
+		goos    string
+		il      interfaces.List
+		wantInt []netaddr.IPPrefix
+		wantExt []netaddr.IPPrefix
+	}{
+		{
+			name: "single-interface",
+			goos: "linux",
+			il: iList(
+				en0,
+				loopback,
+			),
+			wantInt: masked(loopback),
+			wantExt: masked(en0),
+		},
+		{
+			name: "multiple-interfaces",
+			goos: "linux",
+			il: iList(
+				en0,
+				en1,
+				wsl,
+				loopback,
+			),
+			wantInt: masked(loopback),
+			wantExt: masked(en0, en1, wsl),
+		},
+		{
+			name: "wsl2",
+			goos: "windows",
+			il: iList(
+				en0,
+				en1,
+				wsl,
+				loopback,
+			),
+			wantInt: masked(loopback, wsl),
+			wantExt: masked(en0, en1),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			gotInt, gotExt, err := internalAndExternalInterfacesFrom(tc.il, tc.goos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(gotInt, tc.wantInt) {
+				t.Errorf("unexpected internal prefixes\ngot %v\nwant %v", gotInt, tc.wantInt)
+			}
+			if !reflect.DeepEqual(gotExt, tc.wantExt) {
+				t.Errorf("unexpected external prefixes\ngot %v\nwant %v", gotExt, tc.wantExt)
+			}
+		})
+	}
+}

ipn/ipnlocal/loglines_test.go

@@ -54,7 +54,7 @@ func TestLocalLogLines(t *testing.T) {
 	}
 	t.Cleanup(e.Close)
 
-	lb, err := NewLocalBackend(logf, idA.String(), store, e)
+	lb, err := NewLocalBackend(logf, idA.String(), store, nil, e)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -90,7 +90,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()
@@ -105,7 +105,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       11,
 			RxBytes:       12,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()

ipn/ipnlocal/peerapi.go

@@ -6,6 +6,8 @@ package ipnlocal
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -27,32 +29,48 @@ import (
 	"unicode"
 	"unicode/utf8"
 
+	"golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/logtail/backoff"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/filter"
 )
 
 var initListenConfig func(*net.ListenConfig, netaddr.IP, *interfaces.State, string) error
 
+// addH2C is non-nil on platforms where we want to add H2C
+// ("cleartext" HTTP/2) support to the peerAPI.
+var addH2C func(*http.Server)
+
 type peerAPIServer struct {
 	b          *LocalBackend
-	rootDir    string
-	tunName    string
+	rootDir    string // empty means file receiving unavailable
 	selfNode   *tailcfg.Node
 	knownEmpty syncs.AtomicBool
+	resolver   *resolver.Resolver
 
 	// directFileMode is whether we're writing files directly to a
 	// download directory (as *.partial files), rather than making
 	// the frontend retrieve it over localapi HTTP and write it
-	// somewhere itself. This is used on GUI macOS version.
+	// somewhere itself. This is used on the GUI macOS versions
+	// and on Synology.
 	// In directFileMode, the peerapi doesn't do the final rename
-	// from "foo.jpg.partial" to "foo.jpg".
+	// from "foo.jpg.partial" to "foo.jpg" unless
+	// directFileDoFinalRename is set.
 	directFileMode bool
+
+	// directFileDoFinalRename is whether in directFileMode we
+	// additionally move the *.direct file to its final name after
+	// it's received.
+	directFileDoFinalRename bool
 }
 
 const (
@@ -69,6 +87,10 @@ const (
 	deletedSuffix = ".deleted"
 )
 
+func (s *peerAPIServer) canReceiveFiles() bool {
+	return s != nil && s.rootDir != ""
+}
+
 func validFilenameRune(r rune) bool {
 	switch r {
 	case '/':
@@ -115,7 +137,7 @@ func (s *peerAPIServer) diskPath(baseName string) (fullPath string, ok bool) {
 // hasFilesWaiting reports whether any files are buffered in the
 // tailscaled daemon storage.
 func (s *peerAPIServer) hasFilesWaiting() bool {
-	if s.rootDir == "" || s.directFileMode {
+	if s == nil || s.rootDir == "" || s.directFileMode {
 		return false
 	}
 	if s.knownEmpty.Get() {
@@ -175,8 +197,11 @@ func (s *peerAPIServer) hasFilesWaiting() bool {
 // As a side effect, it also does any lazy deletion of files as
 // required by Windows.
 func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
+	if s == nil {
+		return nil, errNilPeerAPIServer
+	}
 	if s.rootDir == "" {
-		return nil, errors.New("peerapi disabled; no storage configured")
+		return nil, errNoTaildrop
 	}
 	if s.directFileMode {
 		return nil, nil
@@ -240,6 +265,11 @@ func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 	return ret, nil
 }
 
+var (
+	errNilPeerAPIServer = errors.New("peerapi unavailable; not listening")
+	errNoTaildrop       = errors.New("Taildrop disabled; no storage directory")
+)
+
 // tryDeleteAgain tries to delete path (and path+deletedSuffix) after
 // it failed earlier.  This happens on Windows when various anti-virus
 // tools hook into filesystem operations and have the file open still
@@ -255,8 +285,11 @@ func tryDeleteAgain(fullPath string) {
 }
 
 func (s *peerAPIServer) DeleteFile(baseName string) error {
+	if s == nil {
+		return errNilPeerAPIServer
+	}
 	if s.rootDir == "" {
-		return errors.New("peerapi disabled; no storage configured")
+		return errNoTaildrop
 	}
 	if s.directFileMode {
 		return errors.New("deletes not allowed in direct mode")
@@ -321,8 +354,11 @@ func touchFile(path string) error {
 }
 
 func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) {
+	if s == nil {
+		return nil, 0, errNilPeerAPIServer
+	}
 	if s.rootDir == "" {
-		return nil, 0, errors.New("peerapi disabled; no storage configured")
+		return nil, 0, errNoTaildrop
 	}
 	if s.directFileMode {
 		return nil, 0, errors.New("opens not allowed in direct mode")
@@ -355,7 +391,7 @@ func (s *peerAPIServer) listen(ip netaddr.IP, ifState *interfaces.State) (ln net
 		// On iOS/macOS, this sets the lc.Control hook to
 		// setsockopt the interface index to bind to, to get
 		// out of the network sandbox.
-		if err := initListenConfig(&lc, ip, ifState, s.tunName); err != nil {
+		if err := initListenConfig(&lc, ip, ifState, s.b.dialer.TUNName()); err != nil {
 			return nil, err
 		}
 		if runtime.GOOS == "darwin" || runtime.GOOS == "ios" {
@@ -460,6 +496,9 @@ func (pln *peerAPIListener) serve() {
 		httpServer := &http.Server{
 			Handler: h,
 		}
+		if addH2C != nil {
+			addH2C(httpServer)
+		}
 		go httpServer.Serve(&oneConnListener{Listener: pln.ln, conn: c})
 	}
 }
@@ -500,9 +539,20 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handlePeerPut(w, r)
 		return
 	}
-	if r.URL.Path == "/v0/goroutines" {
+	if strings.HasPrefix(r.URL.Path, "/dns-query") {
+		h.handleDNSQuery(w, r)
+		return
+	}
+	switch r.URL.Path {
+	case "/v0/goroutines":
 		h.handleServeGoroutines(w, r)
 		return
+	case "/v0/env":
+		h.handleServeEnv(w, r)
+		return
+	case "/v0/metrics":
+		h.handleServeMetrics(w, r)
+		return
 	}
 	who := h.peerUser.DisplayName
 	fmt.Fprintf(w, `<html>
@@ -589,7 +639,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if h.ps.rootDir == "" {
-		http.Error(w, "no rootdir", http.StatusInternalServerError)
+		http.Error(w, errNoTaildrop.Error(), http.StatusInternalServerError)
 		return
 	}
 	rawPath := r.URL.EscapedPath()
@@ -661,7 +711,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if h.ps.directFileMode {
+	if h.ps.directFileMode && !h.ps.directFileDoFinalRename {
 		if inFile != nil { // non-zero length; TODO: notify even for zero length
 			inFile.markAndNotifyDone()
 		}
@@ -710,3 +760,247 @@ func (h *peerAPIHandler) handleServeGoroutines(w http.ResponseWriter, r *http.Re
 	}
 	w.Write(buf)
 }
+
+func (h *peerAPIHandler) handleServeEnv(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	var data struct {
+		Hostinfo *tailcfg.Hostinfo
+		Uid      int
+		Args     []string
+		Env      []string
+	}
+	data.Hostinfo = hostinfo.New()
+	data.Uid = os.Getuid()
+	data.Args = os.Args
+	data.Env = os.Environ()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}
+
+func (h *peerAPIHandler) handleServeMetrics(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
+func (h *peerAPIHandler) replyToDNSQueries() bool {
+	if h.isSelf {
+		// If the peer is owned by the same user, just allow it
+		// without further checks.
+		return true
+	}
+	b := h.ps.b
+	if !b.OfferingExitNode() {
+		// If we're not an exit node, there's no point to
+		// being a DNS server for somebody.
+		return false
+	}
+	if !h.remoteAddr.IsValid() {
+		// This should never be the case if the peerAPIHandler
+		// was wired up correctly, but just in case.
+		return false
+	}
+	// Otherwise, we're an exit node but the peer is not us, so
+	// we need to check if they're allowed access to the internet.
+	// As peerapi bypasses wgengine/filter checks, we need to check
+	// ourselves. As a proxy for autogroup:internet access, we see
+	// if we would've accepted a packet to 0.0.0.0:53. We treat
+	// the IP 0.0.0.0 as being "the internet".
+	f, ok := b.filterAtomic.Load().(*filter.Filter)
+	if !ok {
+		return false
+	}
+	// Note: we check TCP here because the Filter type already had
+	// a CheckTCP method (for unit tests), but it's pretty
+	// arbitrary. DNS runs over TCP and UDP, so sure... we check
+	// TCP.
+	dstIP := netaddr.IPv4(0, 0, 0, 0)
+	remoteIP := h.remoteAddr.IP()
+	if remoteIP.Is6() {
+		// autogroup:internet for IPv6 is defined to start with 2000::/3,
+		// so use 2000::0 as the probe "the internet" address.
+		dstIP = netaddr.MustParseIP("2000::")
+	}
+	verdict := f.CheckTCP(remoteIP, dstIP, 53)
+	return verdict == filter.Accept
+}
+
+// handleDNSQuery implements a DoH server (RFC 8484) over the peerapi.
+// It's not over HTTPS as the spec dictates, but rather HTTP-over-WireGuard.
+func (h *peerAPIHandler) handleDNSQuery(w http.ResponseWriter, r *http.Request) {
+	if h.ps.resolver == nil {
+		http.Error(w, "DNS not wired up", http.StatusNotImplemented)
+		return
+	}
+	if !h.replyToDNSQueries() {
+		http.Error(w, "DNS access denied", http.StatusForbidden)
+		return
+	}
+	pretty := false // non-DoH debug mode for humans
+	q, publicError := dohQuery(r)
+	if publicError != "" && r.Method == "GET" {
+		if name := r.FormValue("q"); name != "" {
+			pretty = true
+			publicError = ""
+			q = dnsQueryForName(name, r.FormValue("t"))
+		}
+	}
+	if publicError != "" {
+		http.Error(w, publicError, http.StatusBadRequest)
+		return
+	}
+
+	// Some timeout that's short enough to be noticed by humans
+	// but long enough that it's longer than real DNS timeouts.
+	const arbitraryTimeout = 5 * time.Second
+
+	ctx, cancel := context.WithTimeout(r.Context(), arbitraryTimeout)
+	defer cancel()
+	res, err := h.ps.resolver.HandleExitNodeDNSQuery(ctx, q, h.remoteAddr, h.ps.b.allowExitNodeDNSProxyToServeName)
+	if err != nil {
+		h.logf("handleDNS fwd error: %v", err)
+		if err := ctx.Err(); err != nil {
+			http.Error(w, err.Error(), 500)
+		} else {
+			http.Error(w, "DNS forwarding error", 500)
+		}
+		return
+	}
+	if pretty {
+		// Non-standard response for interactive debugging.
+		w.Header().Set("Content-Type", "application/json")
+		writePrettyDNSReply(w, res)
+		return
+	}
+	w.Header().Set("Content-Type", "application/dns-message")
+	w.Header().Set("Content-Length", strconv.Itoa(len(res)))
+	w.Write(res)
+}
+
+func dohQuery(r *http.Request) (dnsQuery []byte, publicErr string) {
+	const maxQueryLen = 256 << 10
+	switch r.Method {
+	default:
+		return nil, "bad HTTP method"
+	case "GET":
+		q64 := r.FormValue("dns")
+		if q64 == "" {
+			return nil, "missing 'dns' parameter"
+		}
+		if base64.RawURLEncoding.DecodedLen(len(q64)) > maxQueryLen {
+			return nil, "query too large"
+		}
+		q, err := base64.RawURLEncoding.DecodeString(q64)
+		if err != nil {
+			return nil, "invalid 'dns' base64 encoding"
+		}
+		return q, ""
+	case "POST":
+		if r.Header.Get("Content-Type") != "application/dns-message" {
+			return nil, "unexpected Content-Type"
+		}
+		q, err := io.ReadAll(io.LimitReader(r.Body, maxQueryLen+1))
+		if err != nil {
+			return nil, "error reading post body with DNS query"
+		}
+		if len(q) > maxQueryLen {
+			return nil, "query too large"
+		}
+		return q, ""
+	}
+}
+
+func dnsQueryForName(name, typStr string) []byte {
+	typ := dnsmessage.TypeA
+	switch strings.ToLower(typStr) {
+	case "aaaa":
+		typ = dnsmessage.TypeAAAA
+	case "txt":
+		typ = dnsmessage.TypeTXT
+	}
+	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{
+		OpCode:           0, // query
+		RecursionDesired: true,
+		ID:               0,
+	})
+	if !strings.HasSuffix(name, ".") {
+		name += "."
+	}
+	b.StartQuestions()
+	b.Question(dnsmessage.Question{
+		Name:  dnsmessage.MustNewName(name),
+		Type:  typ,
+		Class: dnsmessage.ClassINET,
+	})
+	msg, _ := b.Finish()
+	return msg
+}
+
+func writePrettyDNSReply(w io.Writer, res []byte) (err error) {
+	defer func() {
+		if err != nil {
+			j, _ := json.Marshal(struct {
+				Error string
+			}{err.Error()})
+			j = append(j, '\n')
+			w.Write(j)
+			return
+		}
+	}()
+	var p dnsmessage.Parser
+	hdr, err := p.Start(res)
+	if err != nil {
+		return err
+	}
+	if hdr.RCode != dnsmessage.RCodeSuccess {
+		return fmt.Errorf("DNS RCode = %v", hdr.RCode)
+	}
+	if err := p.SkipAllQuestions(); err != nil {
+		return err
+	}
+
+	var gotIPs []string
+	for {
+		h, err := p.AnswerHeader()
+		if err == dnsmessage.ErrSectionDone {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if h.Class != dnsmessage.ClassINET {
+			continue
+		}
+		switch h.Type {
+		case dnsmessage.TypeA:
+			r, err := p.AResource()
+			if err != nil {
+				return err
+			}
+			gotIPs = append(gotIPs, net.IP(r.A[:]).String())
+		case dnsmessage.TypeAAAA:
+			r, err := p.AAAAResource()
+			if err != nil {
+				return err
+			}
+			gotIPs = append(gotIPs, net.IP(r.AAAA[:]).String())
+		case dnsmessage.TypeTXT:
+			r, err := p.TXTResource()
+			if err != nil {
+				return err
+			}
+			gotIPs = append(gotIPs, r.TXT...)
+		}
+	}
+	j, _ := json.Marshal(gotIPs)
+	j = append(j, '\n')
+	w.Write(j)
+	return nil
+}

ipn/ipnlocal/peerapi_h2c.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !ios && !android
+// +build !ios,!android
+
+package ipnlocal
+
+import (
+	"net/http"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+)
+
+func init() {
+	addH2C = func(s *http.Server) {
+		h2s := &http2.Server{}
+		s.Handler = h2c.NewHandler(s.Handler, h2s)
+	}
+}

ipn/ipnlocal/peerapi_macios_ext.go

@@ -9,10 +9,8 @@
 package ipnlocal
 
 import (
-	"errors"
 	"fmt"
 	"net"
-	"syscall"
 
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
@@ -21,7 +19,6 @@ import (
 
 func init() {
 	initListenConfig = initListenConfigNetworkExtension
-	peerDialControlFunc = peerDialControlFuncNetworkExtension
 }
 
 // initListenConfigNetworkExtension configures nc for listening on IP
@@ -34,24 +31,3 @@ func initListenConfigNetworkExtension(nc *net.ListenConfig, ip netaddr.IP, st *i
 	}
 	return netns.SetListenConfigInterfaceIndex(nc, tunIf.Index)
 }
-
-func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address string, c syscall.RawConn) error {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	st := b.prevIfState
-	pas := b.peerAPIServer
-	index := -1
-	if st != nil && pas != nil && pas.tunName != "" {
-		if tunIf, ok := st.Interface[pas.tunName]; ok {
-			index = tunIf.Index
-		}
-	}
-	var lc net.ListenConfig
-	netns.SetListenConfigInterfaceIndex(&lc, index)
-	return func(network, address string, c syscall.RawConn) error {
-		if index == -1 {
-			return errors.New("failed to find TUN interface to bind to")
-		}
-		return lc.Control(network, address, c)
-	}
-}

ipn/ipnlocal/peerapi_test.go

@@ -19,8 +19,13 @@ import (
 	"strings"
 	"testing"
 
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
+	"tailscale.com/types/logger"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/filter"
 )
 
 type peerAPITestEnv struct {
@@ -174,7 +179,7 @@ func TestHandlePeerAPI(t *testing.T) {
 			req:        httptest.NewRequest("PUT", "/v0/put/foo", nil),
 			checks: checks(
 				httpStatus(http.StatusInternalServerError),
-				bodyContains("no rootdir"),
+				bodyContains("Taildrop disabled; no storage directory"),
 			),
 		},
 		{
@@ -568,3 +573,55 @@ func TestDeletedMarkers(t *testing.T) {
 	}
 
 }
+
+func TestPeerAPIReplyToDNSQueries(t *testing.T) {
+	var h peerAPIHandler
+
+	h.isSelf = true
+	if !h.replyToDNSQueries() {
+		t.Errorf("for isSelf = false; want true")
+	}
+	h.isSelf = false
+	h.remoteAddr = netaddr.MustParseIPPort("100.150.151.152:12345")
+
+	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+	h.ps = &peerAPIServer{
+		b: &LocalBackend{
+			e: eng,
+		},
+	}
+	if h.ps.b.OfferingExitNode() {
+		t.Fatal("unexpectedly offering exit node")
+	}
+	h.ps.b.prefs = &ipn.Prefs{
+		AdvertiseRoutes: []netaddr.IPPrefix{
+			netaddr.MustParseIPPrefix("0.0.0.0/0"),
+			netaddr.MustParseIPPrefix("::/0"),
+		},
+	}
+	if !h.ps.b.OfferingExitNode() {
+		t.Fatal("unexpectedly not offering exit node")
+	}
+
+	if h.replyToDNSQueries() {
+		t.Errorf("unexpectedly doing DNS without filter")
+	}
+
+	h.ps.b.setFilter(filter.NewAllowNone(logger.Discard, new(netaddr.IPSet)))
+	if h.replyToDNSQueries() {
+		t.Errorf("unexpectedly doing DNS without filter")
+	}
+
+	f := filter.NewAllowAllForTest(logger.Discard)
+
+	h.ps.b.setFilter(f)
+	if !h.replyToDNSQueries() {
+		t.Errorf("unexpectedly deny; wanted to be a DNS server")
+	}
+
+	// Also test IPv6.
+	h.remoteAddr = netaddr.MustParseIPPort("[fe70::1]:12345")
+	if !h.replyToDNSQueries() {
+		t.Errorf("unexpectedly IPv6 deny; wanted to be a DNS server")
+	}
+}

ipn/ipnlocal/state_test.go

@@ -21,7 +21,6 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
-	"tailscale.com/types/wgkey"
 	"tailscale.com/wgengine"
 )
 
@@ -122,7 +121,7 @@ func (cc *mockControl) populateKeys() (newKeys bool) {
 	if cc.persist.PrivateNodeKey.IsZero() {
 		cc.logf("Generating a new nodekey.")
 		cc.persist.OldPrivateNodeKey = cc.persist.PrivateNodeKey
-		cc.persist.PrivateNodeKey, _ = wgkey.NewPrivate()
+		cc.persist.PrivateNodeKey = key.NewNode()
 		newKeys = true
 	}
 
@@ -137,9 +136,7 @@ func (cc *mockControl) send(err error, url string, loginFinished bool, nm *netma
 			URL:     url,
 			NetMap:  nm,
 			Persist: &cc.persist,
-		}
-		if err != nil {
-			s.Err = err.Error()
+			Err:     err,
 		}
 		if loginFinished {
 			s.LoginFinished = &empty.Message{}
@@ -287,7 +284,7 @@ func TestStateMachine(t *testing.T) {
 	t.Cleanup(e.Close)
 
 	cc := newMockControl(t)
-	b, err := NewLocalBackend(logf, "logid", store, e)
+	b, err := NewLocalBackend(logf, "logid", store, nil, e)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
@@ -870,6 +867,45 @@ func TestStateMachine(t *testing.T) {
 		// change either.
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
+	t.Logf("\n\nExpireKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(-time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsTrue)
+	}
+
+	t.Logf("\n\nExtendKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Starting, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsFalse)
+	}
+	notifies.expect(1)
+	// Fake a DERP connection.
+	b.setWgengineStatus(&wgengine.Status{DERPs: 1}, nil)
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Running, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Running, qt.Equals, b.State())
+	}
 }
 
 type testStateStorage struct {
@@ -905,7 +941,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	c.Assert(err, qt.IsNil)
 	t.Cleanup(eng.Close)
-	b, err := NewLocalBackend(logf, "logid", new(ipn.MemoryStore), eng)
+	b, err := NewLocalBackend(logf, "logid", new(ipn.MemoryStore), nil, eng)
 	c.Assert(err, qt.IsNil)
 
 	cc := newMockControl(t)

ipn/ipnserver/server.go

@@ -20,7 +20,6 @@ import (
 	"os/exec"
 	"os/signal"
 	"os/user"
-	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -36,9 +35,9 @@ import (
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
 	"tailscale.com/ipn/store/aws"
-	"tailscale.com/log/filelogger"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/netstat"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
@@ -49,20 +48,18 @@ import (
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
 )
 
 // Options is the configuration of the Tailscale node agent.
 type Options struct {
-	// SocketPath, on unix systems, is the unix socket path to listen
-	// on for frontend connections.
-	SocketPath string
-
-	// Port, on windows, is the localhost TCP port to listen on for
-	// frontend connections.
-	Port int
-
-	// StatePath is the path to the stored agent state.
-	StatePath string
+	// VarRoot is the the Tailscale daemon's private writable
+	// directory (usually "/var/lib/tailscale" on Linux) that
+	// contains the "tailscaled.state" file, the "certs" directory
+	// for TLS certs, and the "files" directory for incoming
+	// Taildrop files before they're moved to a user directory.
+	// If empty, Taildrop and TLS certs don't function.
+	VarRoot string
 
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
@@ -84,15 +81,11 @@ type Options struct {
 	// the actual definition of "disconnect" is when the
 	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
-
-	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
-	// to register a debug handler.
-	DebugMux *http.ServeMux
 }
 
-// server is an IPN backend and its set of 0 or more active connections
-// talking to an IPN backend.
-type server struct {
+// Server is an IPN backend and its set of 0 or more active localhost
+// TCP or unix socket connections talking to that backend.
+type Server struct {
 	b            *ipnlocal.LocalBackend
 	logf         logger.Logf
 	backendLogID string
@@ -101,7 +94,8 @@ type server struct {
 	// being run in "client mode" that requires an active GUI
 	// connection (such as on Windows by default).  Even if this
 	// is true, the ForceDaemon pref can override this.
-	resetOnZero bool
+	resetOnZero       bool
+	autostartStateKey ipn.StateKey
 
 	bsMu sync.Mutex // lock order: bsMu, then mu
 	bs   *ipn.BackendServer
@@ -114,6 +108,9 @@ type server struct {
 	disconnectSub  map[chan<- struct{}]struct{} // keys are subscribers of disconnects
 }
 
+// LocalBackend returns the server's LocalBackend.
+func (s *Server) LocalBackend() *ipnlocal.LocalBackend { return s.b }
+
 // connIdentity represents the owner of a localhost TCP or unix socket connection.
 type connIdentity struct {
 	Conn       net.Conn
@@ -135,7 +132,7 @@ type connIdentity struct {
 // (pid, userid, user). If it's not Windows (for now), it returns a nil error
 // and a ConnIdentity with NotWindows set true. It's only an error if we expected
 // to be able to map it and couldn't.
-func (s *server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
+func (s *Server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
 	ci = connIdentity{Conn: c}
 	if runtime.GOOS != "windows" { // for now; TODO: expand to other OSes
 		ci.NotWindows = true
@@ -172,7 +169,7 @@ func (s *server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
 		return ci, fmt.Errorf("failed to map connection's pid to a user%s: %w", hint, err)
 	}
 	ci.UserID = uid
-	u, err := s.lookupUserFromID(uid)
+	u, err := lookupUserFromID(s.logf, uid)
 	if err != nil {
 		return ci, fmt.Errorf("failed to look up user from userid: %w", err)
 	}
@@ -180,10 +177,10 @@ func (s *server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
 	return ci, nil
 }
 
-func (s *server) lookupUserFromID(uid string) (*user.User, error) {
+func lookupUserFromID(logf logger.Logf, uid string) (*user.User, error) {
 	u, err := user.LookupId(uid)
 	if err != nil && runtime.GOOS == "windows" && errors.Is(err, syscall.Errno(0x534)) {
-		s.logf("[warning] issue 869: os/user.LookupId failed; ignoring")
+		logf("[warning] issue 869: os/user.LookupId failed; ignoring")
 		// Work around https://github.com/tailscale/tailscale/issues/869 for
 		// now. We don't strictly need the username. It's just a nice-to-have.
 		// So make up a *user.User if their machine is broken in this way.
@@ -199,7 +196,7 @@ func (s *server) lookupUserFromID(uid string) (*user.User, error) {
 // blockWhileInUse blocks while until either a Read from conn fails
 // (i.e. it's closed) or until the server is able to accept ci as a
 // user.
-func (s *server) blockWhileInUse(conn io.Reader, ci connIdentity) {
+func (s *Server) blockWhileInUse(conn io.Reader, ci connIdentity) {
 	s.logf("blocking client while server in use; connIdentity=%v", ci)
 	connDone := make(chan struct{})
 	go func() {
@@ -241,7 +238,7 @@ func bufferHasHTTPRequest(br *bufio.Reader) bool {
 		mem.Contains(mem.B(peek), mem.S(" HTTP/"))
 }
 
-func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
+func (s *Server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
 	// First see if it's an HTTP request.
 	br := bufio.NewReader(c)
 	c.SetReadDeadline(time.Now().Add(time.Second))
@@ -391,7 +388,7 @@ func (e inUseOtherUserError) Unwrap() error { return e.error }
 // The returned error, when non-nil, will be of type inUseOtherUserError.
 //
 // s.mu must be held.
-func (s *server) checkConnIdentityLocked(ci connIdentity) error {
+func (s *Server) checkConnIdentityLocked(ci connIdentity) error {
 	// If clients are already connected, verify they're the same user.
 	// This mostly matters on Windows at the moment.
 	if len(s.allClients) > 0 {
@@ -413,14 +410,17 @@ func (s *server) checkConnIdentityLocked(ci connIdentity) error {
 // the Tailscale local daemon API.
 //
 // s.mu must not be held.
-func (s *server) localAPIPermissions(ci connIdentity) (read, write bool) {
-	if runtime.GOOS == "windows" {
+func (s *Server) localAPIPermissions(ci connIdentity) (read, write bool) {
+	switch runtime.GOOS {
+	case "windows":
 		s.mu.Lock()
 		defer s.mu.Unlock()
 		if s.checkConnIdentityLocked(ci) == nil {
 			return true, true
 		}
 		return false, false
+	case "js":
+		return true, true
 	}
 	if ci.IsUnixSock {
 		return true, !isReadonlyConn(ci, s.b.OperatorUserID(), logger.Discard)
@@ -430,7 +430,7 @@ func (s *server) localAPIPermissions(ci connIdentity) (read, write bool) {
 
 // registerDisconnectSub adds ch as a subscribe to connection disconnect
 // events. If add is false, the subscriber is removed.
-func (s *server) registerDisconnectSub(ch chan<- struct{}, add bool) {
+func (s *Server) registerDisconnectSub(ch chan<- struct{}, add bool) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if add {
@@ -448,7 +448,7 @@ func (s *server) registerDisconnectSub(ch chan<- struct{}, add bool) {
 //
 // If the returned error is of type inUseOtherUserError then the
 // returned connIdentity is also valid.
-func (s *server) addConn(c net.Conn, isHTTP bool) (ci connIdentity, err error) {
+func (s *Server) addConn(c net.Conn, isHTTP bool) (ci connIdentity, err error) {
 	ci, err = s.getConnIdentity(c)
 	if err != nil {
 		return
@@ -492,7 +492,7 @@ func (s *server) addConn(c net.Conn, isHTTP bool) (ci connIdentity, err error) {
 	return ci, nil
 }
 
-func (s *server) removeAndCloseConn(c net.Conn) {
+func (s *Server) removeAndCloseConn(c net.Conn) {
 	s.mu.Lock()
 	delete(s.clients, c)
 	delete(s.allClients, c)
@@ -516,7 +516,7 @@ func (s *server) removeAndCloseConn(c net.Conn) {
 	c.Close()
 }
 
-func (s *server) stopAll() {
+func (s *Server) stopAll() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	for c := range s.clients {
@@ -529,7 +529,7 @@ func (s *server) stopAll() {
 // setServerModeUserLocked is called when we're in server mode but our s.serverModeUser is nil.
 //
 // s.mu must be held
-func (s *server) setServerModeUserLocked() {
+func (s *Server) setServerModeUserLocked() {
 	var ci connIdentity
 	var ok bool
 	for _, ci = range s.allClients {
@@ -553,7 +553,7 @@ func (s *server) setServerModeUserLocked() {
 
 var jsonEscapedZero = []byte(`\u0000`)
 
-func (s *server) writeToClients(n ipn.Notify) {
+func (s *Server) writeToClients(n ipn.Notify) {
 	inServerMode := s.b.InServerMode()
 
 	s.mu.Lock()
@@ -602,27 +602,63 @@ func tryWindowsAppDataMigration(logf logger.Logf, path string) string {
 		// what they are doing.
 		return path
 	}
-	oldFile := filepath.Join(os.Getenv("LocalAppData"), "Tailscale", "server-state.conf")
+	oldFile := paths.LegacyStateFilePath()
 	return paths.TryConfigFileMigration(logf, oldFile, path)
 }
 
+// StateStore returns a StateStore from path.
+//
+// The path should be an absolute path to a file.
+//
+// Special cases:
+//
+//   * empty string means to use an in-memory store
+//   * if the string begins with "kube:", the suffix
+//     is a Kubernetes secret name
+//   * if the string begins with "arn:", the value is
+//     an AWS ARN for an SSM.
+func StateStore(path string, logf logger.Logf) (ipn.StateStore, error) {
+	if path == "" {
+		return &ipn.MemoryStore{}, nil
+	}
+	const kubePrefix = "kube:"
+	const arnPrefix = "arn:"
+	switch {
+	case strings.HasPrefix(path, kubePrefix):
+		secretName := strings.TrimPrefix(path, kubePrefix)
+		store, err := ipn.NewKubeStore(secretName)
+		if err != nil {
+			return nil, fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
+		}
+		return store, nil
+	case strings.HasPrefix(path, arnPrefix):
+		store, err := aws.NewStore(path)
+		if err != nil {
+			return nil, fmt.Errorf("aws.NewStore(%q): %v", path, err)
+		}
+		return store, nil
+	}
+	if runtime.GOOS == "windows" {
+		path = tryWindowsAppDataMigration(logf, path)
+	}
+	store, err := ipn.NewFileStore(path)
+	if err != nil {
+		return nil, fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
+	}
+	return store, nil
+}
+
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
-func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+//
+// Deprecated: use New and Server.Run instead.
+func Run(ctx context.Context, logf logger.Logf, ln net.Listener, store ipn.StateStore, linkMon *monitor.Mon, dialer *tsdial.Dialer, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
 	getEngine = getEngineUntilItWorksWrapper(getEngine)
 	runDone := make(chan struct{})
 	defer close(runDone)
 
-	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
-	if err != nil {
-		return fmt.Errorf("safesocket.Listen: %v", err)
-	}
-
-	server := &server{
-		backendLogID: logid,
-		logf:         logf,
-		resetOnZero:  !opts.SurviveDisconnects,
-	}
+	var serverMu sync.Mutex
+	var serverOrNil *Server
 
 	// When the context is closed or when we return, whichever is first, close our listener
 	// and all open connections.
@@ -631,57 +667,33 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 		case <-ctx.Done():
 		case <-runDone:
 		}
-		server.stopAll()
-		listen.Close()
+		serverMu.Lock()
+		if s := serverOrNil; s != nil {
+			s.stopAll()
+		}
+		serverMu.Unlock()
+		ln.Close()
 	}()
-	logf("Listening on %v", listen.Addr())
+	logf("Listening on %v", ln.Addr())
 
-	var store ipn.StateStore
-	if opts.StatePath != "" {
-		const kubePrefix = "kube:"
-		const arnPrefix = "arn:"
-		path := opts.StatePath
-		switch {
-		case strings.HasPrefix(path, kubePrefix):
-			secretName := strings.TrimPrefix(path, kubePrefix)
-			store, err = ipn.NewKubeStore(secretName)
-			if err != nil {
-				return fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
-			}
-		case strings.HasPrefix(path, arnPrefix):
-			store, err = aws.NewStore(path)
-			if err != nil {
-				return fmt.Errorf("aws.NewStore(%q): %v", path, err)
-			}
-		default:
-			if runtime.GOOS == "windows" {
-				path = tryWindowsAppDataMigration(logf, path)
-			}
-			store, err = ipn.NewFileStore(path)
-			if err != nil {
-				return fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
-			}
+	var serverModeUser *user.User
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return fmt.Errorf("calling ReadState on state store: %w", err)
 		}
-		if opts.AutostartStateKey == "" {
-			autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
-			if err != nil && err != ipn.ErrStateNotExist {
-				return fmt.Errorf("calling ReadState on %s: %w", path, err)
-			}
-			key := string(autoStartKey)
-			if strings.HasPrefix(key, "user-") {
-				uid := strings.TrimPrefix(key, "user-")
-				u, err := server.lookupUserFromID(uid)
-				if err != nil {
-					logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
-				} else {
-					logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
-					server.serverModeUser = u
-				}
-				opts.AutostartStateKey = ipn.StateKey(key)
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
 			}
+			opts.AutostartStateKey = ipn.StateKey(key)
 		}
-	} else {
-		store = &ipn.MemoryStore{}
 	}
 
 	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
@@ -691,7 +703,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	if err != nil {
 		logf("ipnserver: initial getEngine call: %v", err)
 		for i := 1; ctx.Err() == nil; i++ {
-			c, err := listen.Accept()
+			c, err := ln.Accept()
 			if err != nil {
 				logf("%d: Accept: %v", i, err)
 				bo.BackOff(ctx, err)
@@ -717,54 +729,131 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			return err
 		}
 	}
-
-	b, err := ipnlocal.NewLocalBackend(logf, logid, store, eng)
-	if err != nil {
-		return fmt.Errorf("NewLocalBackend: %v", err)
+	if unservedConn != nil {
+		ln = &listenerWithReadyConn{
+			Listener: ln,
+			c:        unservedConn,
+		}
 	}
-	defer b.Shutdown()
+
+	server, err := New(logf, logid, store, eng, dialer, serverModeUser, opts)
+	if err != nil {
+		return err
+	}
+	serverMu.Lock()
+	serverOrNil = server
+	serverMu.Unlock()
+	return server.Run(ctx, ln)
+}
+
+// New returns a new Server.
+//
+// To start it, use the Server.Run method.
+func New(logf logger.Logf, logid string, store ipn.StateStore, eng wgengine.Engine, dialer *tsdial.Dialer, serverModeUser *user.User, opts Options) (*Server, error) {
+	b, err := ipnlocal.NewLocalBackend(logf, logid, store, dialer, eng)
+	if err != nil {
+		return nil, fmt.Errorf("NewLocalBackend: %v", err)
+	}
+	b.SetVarRoot(opts.VarRoot)
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)
 	})
 
-	if opts.DebugMux != nil {
-		opts.DebugMux.HandleFunc("/debug/ipn", func(w http.ResponseWriter, r *http.Request) {
-			serveHTMLStatus(w, b)
-		})
+	dg := distro.Get()
+	switch dg {
+	case distro.Synology, distro.TrueNAS:
+		// See if they have a "Taildrop" share.
+		// See https://github.com/tailscale/tailscale/issues/2179#issuecomment-982821319
+		path, err := findTaildropDir(dg)
+		if err != nil {
+			logf("%s Taildrop support: %v", dg, err)
+		} else {
+			logf("%s Taildrop: using %v", dg, path)
+			b.SetDirectFileRoot(path)
+			b.SetDirectFileDoFinalRename(true)
+		}
+
 	}
 
-	server.b = b
-	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return nil, fmt.Errorf("calling ReadState on store: %w", err)
+		}
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
+			}
+			opts.AutostartStateKey = ipn.StateKey(key)
+		}
+	}
 
-	if opts.AutostartStateKey != "" {
-		server.bs.GotCommand(context.TODO(), &ipn.Command{
+	server := &Server{
+		b:                 b,
+		backendLogID:      logid,
+		logf:              logf,
+		resetOnZero:       !opts.SurviveDisconnects,
+		serverModeUser:    serverModeUser,
+		autostartStateKey: opts.AutostartStateKey,
+	}
+	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
+	return server, nil
+}
+
+// Run runs the server, accepting connections from ln forever.
+//
+// If the context is done, the listener is closed.
+func (s *Server) Run(ctx context.Context, ln net.Listener) error {
+	defer s.b.Shutdown()
+
+	runDone := make(chan struct{})
+	defer close(runDone)
+
+	// When the context is closed or when we return, whichever is first, close our listener
+	// and all open connections.
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-runDone:
+		}
+		s.stopAll()
+		ln.Close()
+	}()
+
+	if s.autostartStateKey != "" {
+		s.bs.GotCommand(ctx, &ipn.Command{
 			Version: version.Long,
 			Start: &ipn.StartArgs{
-				Opts: ipn.Options{StateKey: opts.AutostartStateKey},
+				Opts: ipn.Options{StateKey: s.autostartStateKey},
 			},
 		})
 	}
 
 	systemd.Ready()
-	for i := 1; ctx.Err() == nil; i++ {
-		var c net.Conn
-		var err error
-		if unservedConn != nil {
-			c = unservedConn
-			unservedConn = nil
-		} else {
-			c, err = listen.Accept()
+	bo := backoff.NewBackoff("ipnserver", s.logf, 30*time.Second)
+	var connNum int
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
 		}
+		c, err := ln.Accept()
 		if err != nil {
-			if ctx.Err() == nil {
-				logf("ipnserver: Accept: %v", err)
-				bo.BackOff(ctx, err)
+			if ctx.Err() != nil {
+				return ctx.Err()
 			}
+			s.logf("ipnserver: Accept: %v", err)
+			bo.BackOff(ctx, err)
 			continue
 		}
-		go server.serveConn(ctx, c, logger.WithPrefix(logf, fmt.Sprintf("ipnserver: conn%d: ", i)))
+		connNum++
+		go s.serveConn(ctx, c, logger.WithPrefix(s.logf, fmt.Sprintf("ipnserver: conn%d: ", connNum)))
 	}
-	return ctx.Err()
 }
 
 // BabysitProc runs the current executable as a child process with the
@@ -779,14 +868,6 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		panic("cannot determine executable: " + err.Error())
 	}
 
-	if runtime.GOOS == "windows" {
-		if len(args) != 2 && args[0] != "/subproc" {
-			panic(fmt.Sprintf("unexpected arguments %q", args))
-		}
-		logID := args[1]
-		logf = filelogger.New("tailscale-service", logID, logf)
-	}
-
 	var proc struct {
 		mu sync.Mutex
 		p  *os.Process
@@ -954,7 +1035,7 @@ func (a dummyAddr) String() string  { return string(a) }
 // HTTP. So we Read from its bufio.Reader. On Close, we we tell the
 // server it's closed, so the server can account the who's connected.
 type protoSwitchConn struct {
-	s *server
+	s *Server
 	net.Conn
 	br        *bufio.Reader
 	closeOnce sync.Once
@@ -966,7 +1047,7 @@ func (psc *protoSwitchConn) Close() error {
 	return nil
 }
 
-func (s *server) localhostHandler(ci connIdentity) http.Handler {
+func (s *Server) localhostHandler(ci connIdentity) http.Handler {
 	lah := localapi.NewHandler(s.b, s.logf, s.backendLogID)
 	lah.PermitRead, lah.PermitWrite = s.localAPIPermissions(ci)
 
@@ -979,13 +1060,13 @@ func (s *server) localhostHandler(ci connIdentity) http.Handler {
 			io.WriteString(w, "<html><title>Tailscale</title><body><h1>Tailscale</h1>This is the local Tailscale daemon.")
 			return
 		}
-		serveHTMLStatus(w, s.b)
+		s.ServeHTMLStatus(w, r)
 	})
 }
 
-func serveHTMLStatus(w http.ResponseWriter, b *ipnlocal.LocalBackend) {
+func (s *Server) ServeHTMLStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	st := b.Status()
+	st := s.b.Status()
 	// TODO(bradfitz): add LogID and opts to st?
 	st.WriteHTML(w)
 }
@@ -1020,3 +1101,70 @@ func marshalNotify(n ipn.Notify, logf logger.Logf) (b []byte, ok bool) {
 	}
 	return b, true
 }
+
+// listenerWithReadyConn is a net.Listener wrapper that has
+// one net.Conn ready to be accepted already.
+type listenerWithReadyConn struct {
+	net.Listener
+
+	mu sync.Mutex
+	c  net.Conn // if non-nil, ready to be Accepted
+}
+
+func (ln *listenerWithReadyConn) Accept() (net.Conn, error) {
+	ln.mu.Lock()
+	c := ln.c
+	ln.c = nil
+	ln.mu.Unlock()
+	if c != nil {
+		return c, nil
+	}
+	return ln.Listener.Accept()
+}
+
+func findTaildropDir(dg distro.Distro) (string, error) {
+	const name = "Taildrop"
+	switch dg {
+	case distro.Synology:
+		return findSynologyTaildropDir(name)
+	case distro.TrueNAS:
+		return findTrueNASTaildropDir(name)
+	}
+	return "", fmt.Errorf("%s is an unsupported distro for Taildrop dir", dg)
+}
+
+// findSynologyTaildropDir looks for the first volume containing a
+// "Taildrop" directory.  We'd run "synoshare --get Taildrop" command
+// but on DSM7 at least, we lack permissions to run that.
+func findSynologyTaildropDir(name string) (dir string, err error) {
+	for i := 1; i <= 16; i++ {
+		dir = fmt.Sprintf("/volume%v/%s", i, name)
+		if fi, err := os.Stat(dir); err == nil && fi.IsDir() {
+			return dir, nil
+		}
+	}
+	return "", fmt.Errorf("shared folder %q not found", name)
+}
+
+// findTrueNASTaildropDir returns the first matching directory of
+// /mnt/{name} or /mnt/*/{name}
+func findTrueNASTaildropDir(name string) (dir string, err error) {
+	// If we're running in a jail, a mount point could just be added at /mnt/Taildrop
+	dir = fmt.Sprintf("/mnt/%s", name)
+	if fi, err := os.Stat(dir); err == nil && fi.IsDir() {
+		return dir, nil
+	}
+
+	// but if running on the host, it may be something like /mnt/Primary/Taildrop
+	fis, err := ioutil.ReadDir("/mnt")
+	if err != nil {
+		return "", fmt.Errorf("error reading /mnt: %w", err)
+	}
+	for _, fi := range fis {
+		dir = fmt.Sprintf("/mnt/%s/%s", fi.Name(), name)
+		if fi, err := os.Stat(dir); err == nil && fi.IsDir() {
+			return dir, nil
+		}
+	}
+	return "", fmt.Errorf("shared folder %q not found", name)
+}

ipn/ipnserver/server_test.go

@@ -13,6 +13,7 @@ import (
 
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/safesocket"
 	"tailscale.com/wgengine"
 )
@@ -32,10 +33,11 @@ func TestRunMultipleAccepts(t *testing.T) {
 		t.Logf(format, args...)
 	}
 
+	s := safesocket.DefaultConnectionStrategy(socketPath)
 	connect := func() {
 		for i := 1; i <= 2; i++ {
 			logf("connect %d ...", i)
-			c, err := safesocket.Connect(socketPath, 0)
+			c, err := safesocket.Connect(s)
 			if err != nil {
 				t.Fatalf("safesocket.Connect: %v\n", err)
 			}
@@ -62,10 +64,16 @@ func TestRunMultipleAccepts(t *testing.T) {
 	}
 	t.Cleanup(eng.Close)
 
-	opts := ipnserver.Options{
-		SocketPath: socketPath,
-	}
+	opts := ipnserver.Options{}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
+	store := new(ipn.MemoryStore)
+
+	ln, _, err := safesocket.Listen(socketPath, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	err = ipnserver.Run(ctx, logTriggerTestf, ln, store, nil /* mon */, new(tsdial.Dialer), "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -8,7 +8,6 @@
 package ipnstate
 
 import (
-	"bytes"
 	"fmt"
 	"html"
 	"io"
@@ -57,49 +56,55 @@ type Status struct {
 	// trailing periods, and without any "_acme-challenge." prefix.
 	CertDomains []string
 
-	Peer map[key.Public]*PeerStatus
+	Peer map[key.NodePublic]*PeerStatus
 	User map[tailcfg.UserID]tailcfg.UserProfile
 }
 
-func (s *Status) Peers() []key.Public {
-	kk := make([]key.Public, 0, len(s.Peer))
+func (s *Status) Peers() []key.NodePublic {
+	kk := make([]key.NodePublic, 0, len(s.Peer))
 	for k := range s.Peer {
 		kk = append(kk, k)
 	}
-	sort.Slice(kk, func(i, j int) bool { return bytes.Compare(kk[i][:], kk[j][:]) < 0 })
+	sort.Slice(kk, func(i, j int) bool { return kk[i].Less(kk[j]) })
 	return kk
 }
 
 type PeerStatusLite struct {
+	// TxBytes/RxBytes is the total number of bytes transmitted to/received from this peer.
 	TxBytes, RxBytes int64
-	LastHandshake    time.Time
-	NodeKey          tailcfg.NodeKey
+	// LastHandshake is the last time a handshake succeeded with this peer.
+	// (Or we got key confirmation via the first data message,
+	// which is approximately the same thing.)
+	LastHandshake time.Time
+	// NodeKey is this peer's public node key.
+	NodeKey key.NodePublic
 }
 
 type PeerStatus struct {
 	ID        tailcfg.StableNodeID
-	PublicKey key.Public
+	PublicKey key.NodePublic
 	HostName  string // HostInfo's Hostname (not a DNS name or necessarily unique)
 	DNSName   string
 	OS        string // HostInfo.OS
 	UserID    tailcfg.UserID
 
-	TailAddrDeprecated string       `json:"TailAddr"` // Tailscale IP
-	TailscaleIPs       []netaddr.IP // Tailscale IP(s) assigned to this node
+	TailscaleIPs []netaddr.IP // Tailscale IP(s) assigned to this node
 
 	// Endpoints:
 	Addrs   []string
 	CurAddr string // one of Addrs, or unique if roaming
 	Relay   string // DERP region
 
-	RxBytes       int64
-	TxBytes       int64
-	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
-	LastSeen      time.Time // last seen to tailcontrol
-	LastHandshake time.Time // with local wireguard
-	KeepAlive     bool
-	ExitNode      bool // true if this is the currently selected exit node.
+	RxBytes        int64
+	TxBytes        int64
+	Created        time.Time // time registered with tailcontrol
+	LastWrite      time.Time // time last packet sent
+	LastSeen       time.Time // last seen to tailcontrol; only present if offline
+	LastHandshake  time.Time // with local wireguard
+	Online         bool      // whether node is connected to the control plane
+	KeepAlive      bool
+	ExitNode       bool // true if this is the currently selected exit node.
+	ExitNodeOption bool // true if this node can be an exit node (offered && approved)
 
 	// Active is whether the node was recently active. The
 	// definition is somewhat undefined but has historically and
@@ -201,7 +206,7 @@ func (sb *StatusBuilder) AddTailscaleIP(ip netaddr.IP) {
 // AddPeer adds a peer node to the status.
 //
 // Its PeerStatus is mixed with any previous status already added.
-func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
+func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) {
 	if st == nil {
 		panic("nil PeerStatus")
 	}
@@ -214,7 +219,7 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	}
 
 	if sb.st.Peer == nil {
-		sb.st.Peer = make(map[key.Public]*PeerStatus)
+		sb.st.Peer = make(map[key.NodePublic]*PeerStatus)
 	}
 	e, ok := sb.st.Peer[peer]
 	if !ok {
@@ -238,9 +243,6 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.UserID; v != 0 {
 		e.UserID = v
 	}
-	if v := st.TailAddrDeprecated; v != "" {
-		e.TailAddrDeprecated = v
-	}
 	if v := st.TailscaleIPs; v != nil {
 		e.TailscaleIPs = v
 	}
@@ -271,6 +273,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.LastWrite; !v.IsZero() {
 		e.LastWrite = v
 	}
+	if st.Online {
+		e.Online = true
+	}
 	if st.InNetworkMap {
 		e.InNetworkMap = true
 	}
@@ -286,6 +291,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if st.ExitNode {
 		e.ExitNode = true
 	}
+	if st.ExitNodeOption {
+		e.ExitNodeOption = true
+	}
 	if st.ShareeNode {
 		e.ShareeNode = true
 	}
@@ -478,5 +486,6 @@ func sortKey(ps *PeerStatus) string {
 	if len(ps.TailscaleIPs) > 0 {
 		return ps.TailscaleIPs[0].String()
 	}
-	return string(ps.PublicKey[:])
+	raw := ps.PublicKey.Raw32()
+	return string(raw[:])
 }

ipn/localapi/localapi.go

@@ -12,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -20,7 +19,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"inet.af/netaddr"
@@ -28,9 +26,9 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/net/netknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/version"
 )
 
@@ -113,6 +111,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveSetDNS(w, r)
 	case "/localapi/v0/derpmap":
 		h.serveDERPMap(w, r)
+	case "/localapi/v0/metrics":
+		h.serveMetrics(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -184,6 +184,17 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+func (h *Handler) serveMetrics(w http.ResponseWriter, r *http.Request) {
+	// Require write access out of paranoia that the metrics
+	// might contain something sensitive.
+	if !h.PermitWrite {
+		http.Error(w, "metric access denied", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 // serveProfileFunc is the implementation of Handler.serveProfile, after auth,
 // for platforms where we want to link it in.
 var serveProfileFunc func(http.ResponseWriter, *http.Request)
@@ -362,6 +373,25 @@ func (h *Handler) serveFileTargets(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(fts)
 }
 
+// serveFilePut sends a file to another node.
+//
+// It's sometimes possible for clients to do this themselves, without
+// tailscaled, except in the case of tailscaled running in
+// userspace-networking ("netstack") mode, in which case tailscaled
+// needs to a do a netstack dial out.
+//
+// Instead, the CLI also goes through tailscaled so it doesn't need to be
+// aware of the network mode in use.
+//
+// macOS/iOS have always used this localapi method to simplify the GUI
+// clients.
+//
+// The Windows client currently (2021-11-30) uses the peerapi (/v0/put/)
+// directly, as the Windows GUI always runs in tun mode anyway.
+//
+// URL format:
+//
+//    * PUT /localapi/v0/file-put/:stableID/:escaped-filename
 func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitWrite {
 		http.Error(w, "file access denied", http.StatusForbidden)
@@ -409,7 +439,7 @@ func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
 	outReq.ContentLength = r.ContentLength
 
 	rp := httputil.NewSingleHostReverseProxy(dstURL)
-	rp.Transport = getDialPeerTransport(h.b)
+	rp.Transport = h.b.Dialer().PeerAPITransport()
 	rp.ServeHTTP(w, outReq)
 }
 
@@ -443,26 +473,6 @@ func (h *Handler) serveDERPMap(w http.ResponseWriter, r *http.Request) {
 	e.Encode(h.b.DERPMap())
 }
 
-var dialPeerTransportOnce struct {
-	sync.Once
-	v *http.Transport
-}
-
-func getDialPeerTransport(b *ipnlocal.LocalBackend) *http.Transport {
-	dialPeerTransportOnce.Do(func() {
-		t := http.DefaultTransport.(*http.Transport).Clone()
-		t.Dial = nil
-		dialer := net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: netknob.PlatformTCPKeepAlive(),
-			Control:   b.PeerDialControlFunc(),
-		}
-		t.DialContext = dialer.DialContext
-		dialPeerTransportOnce.v = t
-	})
-	return dialPeerTransportOnce.v
-}
-
 func defBool(a string, def bool) bool {
 	if a == "" {
 		return def

ipn/policy/policy.go

@@ -14,7 +14,8 @@ import (
 // system (a version.OS value) is an interesting enough port to report
 // to our peer nodes for discovery purposes.
 func IsInterestingService(s tailcfg.Service, os string) bool {
-	if s.Proto == "peerapi4" || s.Proto == "peerapi6" {
+	switch s.Proto {
+	case tailcfg.PeerAPI4, tailcfg.PeerAPI6, tailcfg.PeerAPIDNS:
 		return true
 	}
 	if s.Proto != tailcfg.TCP {

ipn/prefs_clone.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by the following command; DO NOT EDIT.
-//   tailscale.com/cmd/cloner -type Prefs
+// Code generated by tailscale.com/cmd/cloner; DO NOT EDIT.
+//go:generate go run tailscale.com/cmd/cloner -type=Prefs -output=prefs_clone.go
 
 package ipn
 

ipn/prefs_test.go

@@ -15,12 +15,13 @@ import (
 	"testing"
 	"time"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
+	"tailscale.com/types/key"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
-	"tailscale.com/types/wgkey"
 )
 
 func fieldsOf(t reflect.Type) (fields []string) {
@@ -404,7 +405,7 @@ func TestPrefsPretty(t *testing.T) {
 		{
 			Prefs{
 				Persist: &persist.Persist{
-					PrivateNodeKey: wgkey.Private{1: 1},
+					PrivateNodeKey: key.NodePrivateFromRaw32(mem.B([]byte{1: 1, 31: 0})),
 				},
 			},
 			"linux",

ipn/store.go

@@ -28,7 +28,7 @@ var ErrStateNotExist = errors.New("no state with given ID")
 
 const (
 	// MachineKeyStateKey is the key under which we store the machine key,
-	// in its wgkey.Private.MarshalText representation.
+	// in its key.NodePrivate.MarshalText representation.
 	MachineKeyStateKey = StateKey("_machinekey")
 
 	// GlobalDaemonStateKey is the ipn.StateKey that tailscaled

ipn/store/aws/store_aws.go

@@ -23,7 +23,7 @@ import (
 )
 
 const (
-	parameterNameRxStr = `^parameter/(.*)`
+	parameterNameRxStr = `^parameter(/.*)`
 )
 
 var parameterNameRx = regexp.MustCompile(parameterNameRxStr)

log/filelogger/log.go

@@ -9,6 +9,7 @@ package filelogger
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -26,30 +27,30 @@ const (
 	maxFiles = 50
 )
 
-// New returns a logf wrapper that appends to local disk log
+// New returns a Writer that appends to local disk log
 // files on Windows, rotating old log files as needed to stay under
 // file count & byte limits.
-func New(fileBasePrefix, logID string, logf logger.Logf) logger.Logf {
+func New(fileBasePrefix, logID string, inner *log.Logger) io.Writer {
 	if runtime.GOOS != "windows" {
 		panic("not yet supported on any platform except Windows")
 	}
-	if logf == nil {
-		panic("nil logf")
+	if inner == nil {
+		panic("nil inner logger")
 	}
 	dir := filepath.Join(os.Getenv("ProgramData"), "Tailscale", "Logs")
 
 	if err := os.MkdirAll(dir, 0700); err != nil {
-		log.Printf("failed to create local log directory; not writing logs to disk: %v", err)
-		return logf
+		inner.Printf("failed to create local log directory; not writing logs to disk: %v", err)
+		return inner.Writer()
 	}
-	logf("local disk logdir: %v", dir)
+	inner.Printf("local disk logdir: %v", dir)
 	lfw := &logFileWriter{
 		fileBasePrefix: fileBasePrefix,
 		logID:          logID,
 		dir:            dir,
-		wrappedLogf:    logf,
+		wrappedLogf:    inner.Printf,
 	}
-	return lfw.Logf
+	return logger.FuncWriter(lfw.Logf)
 }
 
 // logFileWriter is the state for the log writer & rotator.
@@ -106,6 +107,7 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	if w.fday != day {
 		w.startNewFileLocked()
 	}
+	out = removeDatePrefix(out)
 	if w.f != nil {
 		// RFC3339Nano but with a fixed number (3) of nanosecond digits:
 		const formatPre = "2006-01-02T15:04:05"
@@ -118,6 +120,30 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	}
 }
 
+func isNum(b byte) bool { return '0' <= b && b <= '9' }
+
+// removeDatePrefix returns a subslice of v with the log package's
+// standard datetime prefix format removed, if present.
+func removeDatePrefix(v []byte) []byte {
+	const format = "2009/01/23 01:23:23 "
+	if len(v) < len(format) {
+		return v
+	}
+	for i, b := range v[:len(format)] {
+		fb := format[i]
+		if isNum(fb) {
+			if !isNum(b) {
+				return v
+			}
+			continue
+		}
+		if b != fb {
+			return v
+		}
+	}
+	return v[len(format):]
+}
+
 // startNewFileLocked opens a new log file for writing
 // and also cleans up any old files.
 //

log/filelogger/log_test.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package filelogger
+
+import "testing"
+
+func TestRemoveDatePrefix(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"\n", "\n"},
+		{"2009/01/23 01:23:23", "2009/01/23 01:23:23"},
+		{"2009/01/23 01:23:23 \n", "\n"},
+		{"2009/01/23 01:23:23 foo\n", "foo\n"},
+		{"9999/01/23 01:23:23 foo\n", "foo\n"},
+		{"2009_01/23 01:23:23 had an underscore\n", "2009_01/23 01:23:23 had an underscore\n"},
+	}
+	for i, tt := range tests {
+		got := removeDatePrefix([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Logf("[%d] removeDatePrefix(%q) = %q; want %q", i, tt.in, got, tt.want)
+		}
+	}
+
+}